diff --git a/CMakeLists.txt b/CMakeLists.txt
index 566d4ad29d..fb2b306f65 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -10,6 +10,8 @@ if (NOT DEFINED TD_SOURCE_DIR)
set( TD_SOURCE_DIR ${PROJECT_SOURCE_DIR} )
endif()
+SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR})
+
set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake")
set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib")
diff --git a/cmake/cmake.version b/cmake/cmake.version
index 29f560148e..5150ee3b75 100644
--- a/cmake/cmake.version
+++ b/cmake/cmake.version
@@ -16,7 +16,7 @@ find_program(HAVE_GIT NAMES git)
IF (DEFINED GITINFO)
SET(TD_VER_GIT ${GITINFO})
ELSEIF (HAVE_GIT)
- execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID)
+ execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${TD_COMMUNITY_DIR} OUTPUT_VARIABLE GIT_COMMITID)
#message(STATUS "git log result:${GIT_COMMITID}")
IF (GIT_COMMITID)
string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
@@ -30,6 +30,23 @@ ELSE ()
SET(TD_VER_GIT "no git commit id")
ENDIF ()
+IF (DEFINED GITINFOI)
+ SET(TD_VER_GIT_INTERNAL ${GITINFOI})
+ELSEIF (HAVE_GIT)
+ execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID)
+ message(STATUS "git log result:${GIT_COMMITID}")
+ IF (GIT_COMMITID)
+ string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
+ SET(TD_VER_GIT_INTERNAL ${GIT_COMMITID})
+ ELSE ()
+ message(STATUS "not a git repository")
+ SET(TD_VER_GIT "no git commit id")
+ ENDIF ()
+ELSE ()
+ message(STATUS "no git cmd")
+ SET(TD_VER_GIT_INTERNAL "no git commit id")
+ENDIF ()
+
IF (DEFINED VERDATE)
SET(TD_VER_DATE ${VERDATE})
ELSE ()
diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx
index 5ee2053210..7465cc0a12 100644
--- a/docs/en/07-develop/07-tmq.mdx
+++ b/docs/en/07-develop/07-tmq.mdx
@@ -368,7 +368,6 @@ conf := &tmq.ConfigMap{
"td.connect.port": "6030",
"client.id": "test_tmq_c",
"enable.auto.commit": "false",
- "enable.heartbeat.background": "true",
"experimental.snapshot.enable": "true",
"msg.with.table.name": "true",
}
diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go
index 01bf5e6421..cb24e351ab 100644
--- a/docs/examples/go/sub/main.go
+++ b/docs/examples/go/sub/main.go
@@ -35,7 +35,6 @@ func main() {
"td.connect.port": "6030",
"client.id": "test_tmq_client",
"enable.auto.commit": "false",
- "enable.heartbeat.background": "true",
"experimental.snapshot.enable": "true",
"msg.with.table.name": "true",
})
diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx
index b82972ab3e..036c9aa86e 100644
--- a/docs/zh/07-develop/07-tmq.mdx
+++ b/docs/zh/07-develop/07-tmq.mdx
@@ -367,7 +367,6 @@ conf := &tmq.ConfigMap{
"td.connect.port": "6030",
"client.id": "test_tmq_c",
"enable.auto.commit": "false",
- "enable.heartbeat.background": "true",
"experimental.snapshot.enable": "true",
"msg.with.table.name": "true",
}
@@ -419,7 +418,6 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"})
| `auto.commit.interval.ms` | string | 以毫秒为单位的自动提交时间间隔 | 默认值：5000 ms |
| `auto.offset.reset` | string | 消费组订阅的初始位置 | 可选：`earliest`(default), `latest`, `none` |
| `experimental.snapshot.enable` | string | 是否允许从 TSDB 消费数据 | 合法值：`true`, `false` |
-| `enable.heartbeat.background` | string | 启用后台心跳，启用后即使长时间不 poll 消息也不会造成离线 | 合法值：`true`, `false` |
diff --git a/examples/JDBC/taosdemo/pom.xml b/examples/JDBC/taosdemo/pom.xml
index 68224bbad5..4731d8e237 100644
--- a/examples/JDBC/taosdemo/pom.xml
+++ b/examples/JDBC/taosdemo/pom.xml
@@ -10,7 +10,7 @@
Demo project for TDengine
- 5.3.20
+ 5.3.26
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 9e928a79ac..51a714c792 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -185,7 +185,7 @@ typedef struct SBlockID {
typedef struct SDataBlockInfo {
STimeWindow window;
int32_t rowSize;
- int32_t rows; // todo hide this attribute
+ int64_t rows; // todo hide this attribute
uint32_t capacity;
SBlockID id;
int16_t hasVarCol;
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 84d696e518..99fffa2cf1 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -178,7 +178,7 @@ int32_t getJsonValueLen(const char* data);
int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
-int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows);
+int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity,
const SColumnInfoData* pSource, int32_t numOfRow2);
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h
index eca8740d28..9b709272b2 100644
--- a/include/common/tmsgcb.h
+++ b/include/common/tmsgcb.h
@@ -34,7 +34,7 @@ typedef enum {
WRITE_QUEUE,
APPLY_QUEUE,
SYNC_QUEUE,
- SYNC_CTRL_QUEUE,
+ SYNC_RD_QUEUE,
STREAM_QUEUE,
QUEUE_MAX,
} EQueueType;
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index e53d8992e8..6cf6140815 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -260,7 +260,7 @@ enum {
TD_NEW_MSG_SEG(TDMT_SYNC_MSG)
TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
- TD_DEF_MSG_TYPE(TDMT_SYNC_PING, "sync-ping", NULL, NULL) // no longer used
+ TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT_ELECTION, "sync-elect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) // no longer used
TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST, "sync-client-request", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST_BATCH, "sync-client-request-batch", NULL, NULL)
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index e76422ee34..91b6fa51a2 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -78,7 +78,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v
* @param SReadHandle
* @return
*/
-qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, SSchemaWrapper** pSchema);
+qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols,
+ uint64_t id);
/**
* set the task Id, usually used by message queue process
@@ -89,6 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3
void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
+
/**
* Set multiple input data blocks for the stream scan.
* @param tinfo
@@ -149,7 +151,6 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
* @param handle
* @return
*/
-
int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal);
int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds);
@@ -162,6 +163,7 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo);
* @return
*/
int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode);
+int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode);
bool qTaskIsExecuting(qTaskInfo_t qinfo);
@@ -171,14 +173,6 @@ bool qTaskIsExecuting(qTaskInfo_t qinfo);
*/
void qDestroyTask(qTaskInfo_t tinfo);
-/**
- * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks.
- *
- * @param iter the table iterator to traverse all tables belongs to a super table, or an invert index
- * @return
- */
-int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList);
-
void qProcessRspMsg(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList /*,int32_t* resNum, SExplainExecInfo** pRes*/);
diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index fb6ef26a8a..aa5c78195a 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -112,7 +112,7 @@ typedef struct SResultDataInfo {
typedef struct SInputColumnInfoData {
int32_t totalRows; // total rows in current columnar data
int32_t startRowIndex; // handle started row index
- int32_t numOfRows; // the number of rows needs to be handled
+ int64_t numOfRows; // the number of rows needs to be handled
int32_t numOfInputCols; // PTS is not included
bool colDataSMAIsSet; // if agg is set or not
SColumnInfoData *pPTS; // primary timestamp column
diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h
index 7b65c06b85..60a4a8c605 100644
--- a/include/libs/function/functionMgt.h
+++ b/include/libs/function/functionMgt.h
@@ -241,6 +241,7 @@ int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
int32_t fmSetInvertFunc(int32_t funcId, SFuncExecFuncs* pFpSet);
int32_t fmSetNormalFunc(int32_t funcId, SFuncExecFuncs* pFpSet);
bool fmIsInvertible(int32_t funcId);
+char* fmGetFuncName(int32_t funcId);
#ifdef __cplusplus
}
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index cb547ee6b3..b6ada5a0c7 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -33,6 +33,7 @@ typedef enum {
JOB_TASK_STATUS_INIT,
JOB_TASK_STATUS_EXEC,
JOB_TASK_STATUS_PART_SUCC,
+ JOB_TASK_STATUS_FETCH,
JOB_TASK_STATUS_SUCC,
JOB_TASK_STATUS_FAIL,
JOB_TASK_STATUS_DROP,
diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h
index 6b09bf4899..fd5cec2931 100644
--- a/include/libs/stream/streamState.h
+++ b/include/libs/stream/streamState.h
@@ -23,20 +23,19 @@ extern "C" {
#ifndef _STREAM_STATE_H_
#define _STREAM_STATE_H_
-typedef struct SStreamTask SStreamTask;
-
typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
typedef struct STdbState {
- SStreamTask* pOwner;
- TDB* db;
- TTB* pStateDb;
- TTB* pFuncStateDb;
- TTB* pFillStateDb; // todo refactor
- TTB* pSessionStateDb;
- TTB* pParNameDb;
- TTB* pParTagDb;
- TXN* txn;
+ struct SStreamTask* pOwner;
+
+ TDB* db;
+ TTB* pStateDb;
+ TTB* pFuncStateDb;
+ TTB* pFillStateDb; // todo refactor
+ TTB* pSessionStateDb;
+ TTB* pParNameDb;
+ TTB* pParTagDb;
+ TXN* txn;
} STdbState;
// incremental state storage
@@ -45,7 +44,7 @@ typedef struct {
int32_t number;
} SStreamState;
-SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages);
+SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages);
void streamStateClose(SStreamState* pState);
int32_t streamStateBegin(SStreamState* pState);
int32_t streamStateCommit(SStreamState* pState);
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 1d301623b1..cdcf54bc61 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -295,7 +295,7 @@ typedef struct {
SEpSet epSet;
} SStreamChildEpInfo;
-typedef struct SStreamTask {
+struct SStreamTask {
int64_t streamId;
int32_t taskId;
int32_t totalLevel;
@@ -362,8 +362,7 @@ typedef struct SStreamTask {
int64_t checkpointingId;
int32_t checkpointAlignCnt;
-
-} SStreamTask;
+};
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
@@ -587,7 +586,7 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg);
-int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
+// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
int32_t streamTryExec(SStreamTask* pTask);
int32_t streamSchedExec(SStreamTask* pTask);
diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h
index a86e97ea78..fdb4506cf3 100644
--- a/include/libs/sync/sync.h
+++ b/include/libs/sync/sync.h
@@ -48,7 +48,7 @@ extern "C" {
#define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500
#define SYNC_SNAP_RESEND_MS 1000 * 60
-#define SYNC_VND_COMMIT_MIN_MS 1000
+#define SYNC_VND_COMMIT_MIN_MS 3000
#define SYNC_MAX_BATCH_SIZE 1
#define SYNC_INDEX_BEGIN 0
diff --git a/include/util/tarray.h b/include/util/tarray.h
index a8510e4bc8..4bf24b46b9 100644
--- a/include/util/tarray.h
+++ b/include/util/tarray.h
@@ -69,14 +69,6 @@ void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles);
*/
void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*));
-/**
- *
- * @param pArray
- * @param comparFn
- * @param fp
- */
-void taosArrayRemoveDuplicateP(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*));
-
/**
* add all element from the source array list into the destination
* @param pArray
@@ -216,6 +208,8 @@ void taosArrayDestroyP(SArray* pArray, FDelete fp);
void taosArrayDestroyEx(SArray* pArray, FDelete fp);
+void taosArraySwap(SArray* a, SArray* b);
+
/**
* sort the array
* @param pArray
@@ -252,14 +246,6 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par
int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
-/**
- * swap array
- * @param a
- * @param b
- * @return
- */
-void taosArraySwap(SArray* a, SArray* b);
-
#ifdef __cplusplus
}
#endif
diff --git a/include/util/tsimplehash.h b/include/util/tsimplehash.h
index c9df911476..987a9fe2a8 100644
--- a/include/util/tsimplehash.h
+++ b/include/util/tsimplehash.h
@@ -48,6 +48,13 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn);
*/
int32_t tSimpleHashGetSize(const SSHashObj *pHashObj);
+/**
+ * set the free function pointer
+ * @param pHashObj
+ * @param freeFp
+ */
+void tSimpleHashSetFreeFp(SSHashObj* pHashObj, _hash_free_fn_t freeFp);
+
int32_t tSimpleHashPrint(const SSHashObj *pHashObj);
/**
diff --git a/include/util/version.h b/include/util/version.h
index bac29e5baf..b241dd248b 100644
--- a/include/util/version.h
+++ b/include/util/version.h
@@ -23,6 +23,7 @@ extern "C" {
extern char version[];
extern char compatible_version[];
extern char gitinfo[];
+extern char gitinfoOfInternal[];
extern char buildinfo[];
#ifdef __cplusplus
diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec
index e9d86219cb..c21063e6a4 100644
--- a/packaging/rpm/tdengine.spec
+++ b/packaging/rpm/tdengine.spec
@@ -90,45 +90,33 @@ cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepat
cp -r %{_compiledir}/../examples/* %{buildroot}%{homepath}/examples
if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then
- mkdir -p %{buildroot}%{userlocalpath}/bin
- mkdir -p %{buildroot}%{userlocalpath}/lib
- mkdir -p %{buildroot}%{userlocalpath}/lib/pkgconfig
- mkdir -p %{buildroot}%{userlocalpath}/include
- mkdir -p %{buildroot}%{userlocalpath}/include/jemalloc
- mkdir -p %{buildroot}%{userlocalpath}/share
- mkdir -p %{buildroot}%{userlocalpath}/share/doc
- mkdir -p %{buildroot}%{userlocalpath}/share/doc/jemalloc
- mkdir -p %{buildroot}%{userlocalpath}/share/man
- mkdir -p %{buildroot}%{userlocalpath}/share/man/man3
+ mkdir -p %{buildroot}%{homepath}/jemalloc/ ||:
+ mkdir -p %{buildroot}%{homepath}/jemalloc/include/jemalloc/ ||:
+ mkdir -p %{buildroot}%{homepath}/jemalloc/lib/ ||:
+ mkdir -p %{buildroot}%{homepath}/jemalloc/lib/pkgconfig ||:
- cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{userlocalpath}/bin/
+ cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{homepath}/jemalloc/bin
if [ -f %{_compiledir}/build/bin/jemalloc.sh ]; then
- cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{userlocalpath}/bin/
+ cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{homepath}/jemalloc/bin
fi
if [ -f %{_compiledir}/build/bin/jeprof ]; then
- cp %{_compiledir}/build/bin/jeprof %{buildroot}%{userlocalpath}/bin/
+ cp %{_compiledir}/build/bin/jeprof %{buildroot}%{homepath}/jemalloc/bin
fi
if [ -f %{_compiledir}/build/include/jemalloc/jemalloc.h ]; then
- cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{userlocalpath}/include/jemalloc/
+ cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{homepath}/jemalloc/include/jemalloc/
fi
if [ -f %{_compiledir}/build/lib/libjemalloc.so.2 ]; then
- cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/
- ln -sf libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/libjemalloc.so
+ cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib
+ ln -sf libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib/libjemalloc.so
fi
if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then
- cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{userlocalpath}/lib/
+ cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{homepath}/jemalloc/lib
fi
if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then
- cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{userlocalpath}/lib/
+ cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{homepath}/jemalloc/lib
fi
if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then
- cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{userlocalpath}/lib/pkgconfig/
- fi
- if [ -f %{_compiledir}/build/share/doc/jemalloc/jemalloc.html ]; then
- cp %{_compiledir}/build/share/doc/jemalloc/jemalloc.html %{buildroot}%{userlocalpath}/share/doc/jemalloc/
- fi
- if [ -f %{_compiledir}/build/share/man/man3/jemalloc.3 ]; then
- cp %{_compiledir}/build/share/man/man3/jemalloc.3 %{buildroot}%{userlocalpath}/share/man/man3/
+ cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig
fi
fi
diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh
index 10093bdf49..1a1622cb93 100755
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@@ -51,6 +51,8 @@ fi
if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then
cd ${top_dir}/tools/taos-tools/packaging/deb
+
+ taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1)
[ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1)
diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh
index d2712d8365..3a013ade2c 100755
--- a/packaging/tools/post.sh
+++ b/packaging/tools/post.sh
@@ -145,6 +145,59 @@ function install_include() {
log_print "install include success"
}
+function install_jemalloc() {
+ jemalloc_dir=${script_dir}/../jemalloc
+
+ if [ -d ${jemalloc_dir} ]; then
+ ${csudo}/usr/bin/install -c -d /usr/local/bin
+
+ if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then
+ ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin
+ fi
+ if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then
+ ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin
+ fi
+ if [ -f ${jemalloc_dir}/bin/jeprof ]; then
+ ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin
+ fi
+ if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then
+ ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc
+ ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc
+ fi
+ if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then
+ ${csudo}/usr/bin/install -c -d /usr/local/lib
+ ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib
+ ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
+ ${csudo}/usr/bin/install -c -d /usr/local/lib
+ if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
+ ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
+ fi
+ if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+ ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
+ fi
+ if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+ ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig
+ ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig
+ fi
+ fi
+ if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then
+ ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc
+ ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc
+ fi
+ if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then
+ ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3
+ ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
+ fi
+
+ if [ -d /etc/ld.so.conf.d ]; then
+ echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf"
+ ${csudo}ldconfig
+ else
+ echo "/etc/ld.so.conf.d not found!"
+ fi
+ fi
+}
+
function install_lib() {
log_print "start install lib from ${lib_dir} to ${lib_link_dir}"
${csudo}rm -f ${lib_link_dir}/libtaos* || :
@@ -667,6 +720,7 @@ function install_TDengine() {
# Install include, lib, binary and service
install_include &&
install_lib &&
+ install_jemalloc
install_bin
if [[ "$?" != 0 ]];then
diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c
index 21590022b1..a0146cfa39 100644
--- a/source/client/src/clientMsgHandler.c
+++ b/source/client/src/clientMsgHandler.c
@@ -76,7 +76,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
goto End;
}
- if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 2)) != 0) {
+ if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 3)) != 0) {
setErrno(pRequest, code);
tsem_post(&pRequest->body.rspSem);
goto End;
@@ -506,7 +506,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
}
- if(code != 0){
+ if (code != 0) {
taosMemoryFree(pRes);
}
tFreeSShowVariablesRsp(&rsp);
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 6bd8b01842..a09780dc15 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -25,9 +25,7 @@
#include "tref.h"
#include "ttimer.h"
-static tb_uid_t processSuid(tb_uid_t suid, char* db){
- return suid + MurmurHash3_32(db, strlen(db));
-}
+static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
int8_t t) {
@@ -363,7 +361,8 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) {
if (pCreateReq->type == TSDB_CHILD_TABLE) {
string = buildCreateCTableJson(req.pReqs, req.nReqs);
} else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
- string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE);
+ string =
+ buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE);
}
uDebug("processCreateTable :%s", string);
}
@@ -709,7 +708,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
pReq.source = TD_REQ_FROM_TAOX;
pReq.igExists = true;
- uDebug("taosCreateStb name:%s suid:%"PRId64" processSuid:%"PRId64, req.name, req.suid, pReq.suid);
+ uDebug("taosCreateStb name:%s suid:%" PRId64 " processSuid:%" PRId64, req.name, req.suid, pReq.suid);
STscObj* pTscObj = pRequest->pTscObj;
SName tableName;
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
@@ -776,20 +775,20 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
goto end;
}
- SCatalog* pCatalog = NULL;
+ SCatalog* pCatalog = NULL;
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
- .requestId = pRequest->requestId,
- .requestObjRefId = pRequest->self,
- .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
- SName pName = {0};
+ .requestId = pRequest->requestId,
+ .requestObjRefId = pRequest->self,
+ .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
+ SName pName = {0};
toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName);
- STableMeta *pTableMeta = NULL;
+ STableMeta* pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
- if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
+ if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
code = TSDB_CODE_SUCCESS;
taosMemoryFreeClear(pTableMeta);
goto end;
@@ -804,9 +803,9 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
// build drop stable
pReq.igNotExists = true;
pReq.source = TD_REQ_FROM_TAOX;
-// pReq.suid = processSuid(req.suid, pRequest->pDb);
+ // pReq.suid = processSuid(req.suid, pRequest->pDb);
- uDebug("taosDropStb name:%s suid:%"PRId64" new suid:%"PRId64, req.name, req.suid, pReq.suid);
+ uDebug("taosDropStb name:%s suid:%" PRId64 " new suid:%" PRId64, req.name, req.suid, pReq.suid);
STscObj* pTscObj = pRequest->pTscObj;
SName tableName = {0};
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
@@ -831,7 +830,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
launchQueryImpl(pRequest, &pQuery, true, NULL);
if (pRequest->code == TSDB_CODE_SUCCESS) {
-// SCatalog* pCatalog = NULL;
+ // SCatalog* pCatalog = NULL;
catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
catalogRemoveTableMeta(pCatalog, &tableName);
}
@@ -924,8 +923,8 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
if (pCreateReq->type == TSDB_CHILD_TABLE) {
STableMeta* pTableMeta = NULL;
SName sName = {0};
- tb_uid_t oldSuid = pCreateReq->ctb.suid;
-// pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
+ tb_uid_t oldSuid = pCreateReq->ctb.suid;
+ // pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName);
code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta);
if (code != TSDB_CODE_SUCCESS) {
@@ -933,7 +932,8 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
goto end;
}
pCreateReq->ctb.suid = pTableMeta->uid;
- uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" new suid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid);
+ uDebug("taosCreateTable name:%s sname:%s suid:%" PRId64 " new suid:%" PRId64, pCreateReq->name,
+ pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid);
for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) {
char* tName = taosArrayGet(pCreateReq->ctb.tagName, i);
@@ -1066,7 +1066,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
pDropReq = req.pReqs + iReq;
pDropReq->igNotExists = true;
-// pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
+ // pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
SVgroupInfo pInfo = {0};
SName pName = {0};
@@ -1076,9 +1076,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
goto end;
}
- STableMeta *pTableMeta = NULL;
+ STableMeta* pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
- if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
+ if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
code = TSDB_CODE_SUCCESS;
taosMemoryFreeClear(pTableMeta);
continue;
@@ -1090,7 +1090,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
tb_uid_t oldSuid = pDropReq->suid;
pDropReq->suid = pTableMeta->suid;
taosMemoryFreeClear(pTableMeta);
- uDebug("taosDropTable name:%s suid:%"PRId64" new suid:%"PRId64, pDropReq->name, oldSuid, pDropReq->suid);
+ uDebug("taosDropTable name:%s suid:%" PRId64 " new suid:%" PRId64, pDropReq->name, oldSuid, pDropReq->suid);
taosArrayPush(pRequest->tableList, &pName);
SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId));
@@ -1106,7 +1106,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
}
}
- if (taosHashGetSize(pVgroupHashmap) == 0){
+ if (taosHashGetSize(pVgroupHashmap) == 0) {
goto end;
}
SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap);
@@ -1373,7 +1373,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname);
goto end;
}
-// uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid);
+ // uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid);
pQuery = smlInitHandle();
if (pQuery == NULL) {
@@ -1382,7 +1382,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData));
-// uError("td23101 1vgId:%d, numEps:%d, name:%s, uid:%"PRIu64, vgData.vgId, vgData.epSet.numOfEps, tbname, pTableMeta->uid);
+ // uError("td23101 1vgId:%d, numEps:%d, name:%s, uid:%"PRIu64, vgData.vgId, vgData.epSet.numOfEps, tbname,
+ // pTableMeta->uid);
code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false);
if (code != TSDB_CODE_SUCCESS) {
@@ -1601,6 +1602,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
uError("WriteRaw:rawBlockBindData failed");
goto end;
}
+ taosMemoryFreeClear(pTableMeta);
}
code = smlBuildOutput(pQuery, pVgHash);
@@ -1707,6 +1709,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) {
tDecoderClear(&decoderTmp);
+ tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
uError("WriteRaw: tDecodeSVCreateTbReq error");
code = TSDB_CODE_TMQ_INVALID_MSG;
goto end;
@@ -1715,15 +1718,19 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
if (pCreateReq.type != TSDB_CHILD_TABLE) {
uError("WriteRaw:pCreateReq.type != TSDB_CHILD_TABLE. table name: %s", tbName);
code = TSDB_CODE_TSC_INVALID_VALUE;
+ tDecoderClear(&decoderTmp);
+ tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
goto end;
}
if (strcmp(tbName, pCreateReq.name) == 0) {
cloneSVreateTbReq(&pCreateReq, &pCreateReqDst);
-// pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
+ // pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
tDecoderClear(&decoderTmp);
+ tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
break;
}
tDecoderClear(&decoderTmp);
+ tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
}
SVgroupInfo vg;
@@ -1774,6 +1781,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
goto end;
}
pCreateReqDst = NULL;
+ taosMemoryFreeClear(pTableMeta);
}
code = smlBuildOutput(pQuery, pVgHash);
diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index d93566d7fa..ef7bd546c1 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -256,20 +256,15 @@ cleanup:
kvVal->type = TSDB_DATA_TYPE_FLOAT; \
kvVal->f = (float)result;
-#define SET_BIGINT \
- if (smlDoubleToInt64OverFlow(result)) { \
- errno = 0; \
- int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \
- if (errno == ERANGE) { \
- smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
- return false; \
- } \
- kvVal->type = TSDB_DATA_TYPE_BIGINT; \
- kvVal->i = tmp; \
- return true; \
- } \
- kvVal->type = TSDB_DATA_TYPE_BIGINT; \
- kvVal->i = (int64_t)result;
+#define SET_BIGINT \
+ errno = 0; \
+ int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \
+ if (errno == ERANGE) { \
+ smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
+ return false; \
+ } \
+ kvVal->type = TSDB_DATA_TYPE_BIGINT; \
+ kvVal->i = tmp;
#define SET_INT \
if (!IS_VALID_INT(result)) { \
@@ -287,20 +282,15 @@ cleanup:
kvVal->type = TSDB_DATA_TYPE_SMALLINT; \
kvVal->i = result;
-#define SET_UBIGINT \
- if (result >= (double)UINT64_MAX || result < 0) { \
- errno = 0; \
- uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \
- if (errno == ERANGE || result < 0) { \
- smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
- return false; \
- } \
- kvVal->type = TSDB_DATA_TYPE_UBIGINT; \
- kvVal->u = tmp; \
- return true; \
- } \
- kvVal->type = TSDB_DATA_TYPE_UBIGINT; \
- kvVal->u = result;
+#define SET_UBIGINT \
+ errno = 0; \
+ uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \
+ if (errno == ERANGE || result < 0) { \
+ smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
+ return false; \
+ } \
+ kvVal->type = TSDB_DATA_TYPE_UBIGINT; \
+ kvVal->u = tmp;
#define SET_UINT \
if (!IS_VALID_UINT(result)) { \
@@ -637,8 +627,8 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
for (int j = 0; j < taosArrayGetSize(cols); ++j) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j);
ESchemaAction action = SCHEMA_ACTION_NULL;
- int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info);
- if(code != 0){
+ int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info);
+ if (code != 0) {
return code;
}
if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) {
@@ -649,11 +639,11 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
taosArrayPush(results, &field);
} else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen);
- if(index == NULL){
+ if (index == NULL) {
uError("smlBuildFieldsList get error, key:%s", kv->key);
return TSDB_CODE_SML_INVALID_DATA;
}
- uint16_t newIndex = *index;
+ uint16_t newIndex = *index;
if (isTag) newIndex -= numOfCols;
SField *field = (SField *)taosArrayGet(results, newIndex);
field->bytes = getBytes(kv->type, kv->length);
@@ -750,7 +740,8 @@ end:
}
static int32_t smlModifyDBSchemas(SSmlHandle *info) {
- uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema);
+ uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat,
+ info->needModifySchema);
if (info->dataFormat && !info->needModifySchema) {
return TSDB_CODE_SUCCESS;
}
@@ -820,7 +811,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end;
}
if (action != SCHEMA_ACTION_NULL) {
- uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, action);
+ uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname,
+ action);
SArray *pColumns =
taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray *pTags =
@@ -838,7 +830,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
}
}
code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags,
- pTableMeta->tableInfo.numOfColumns, true);
+ pTableMeta->tableInfo.numOfColumns, true);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname);
goto end;
@@ -872,7 +864,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end;
}
if (action != SCHEMA_ACTION_NULL) {
- uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, action);
+ uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname,
+ action);
SArray *pColumns =
taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray *pTags =
@@ -891,7 +884,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
}
code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns,
- pTableMeta->tableInfo.numOfColumns, false);
+ pTableMeta->tableInfo.numOfColumns, false);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname);
goto end;
@@ -939,10 +932,11 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
}
sTableData->tableMeta = pTableMeta;
- uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, pTableMeta->sversion, pTableMeta->tversion)
- tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
+ uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid,
+ pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
}
- uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema);
+ uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat,
+ info->needModifySchema);
return 0;
@@ -950,7 +944,8 @@ end:
taosHashCleanup(hashTmp);
taosMemoryFreeClear(pTableMeta);
catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
- uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema);
+ uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code,
+ tstrerror(code), info->dataFormat, info->needModifySchema);
return code;
}
@@ -1128,7 +1123,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
}
if (taos != NULL) {
info->taos = acquireTscObj(*(int64_t *)taos);
- if(info->taos == NULL){
+ if (info->taos == NULL) {
goto cleanup;
}
code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
@@ -1171,7 +1166,9 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
for (size_t i = 0; i < taosArrayGetSize(cols); i++) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES);
- if(terrno == TSDB_CODE_DUP_KEY){return terrno;}
+ if (terrno == TSDB_CODE_DUP_KEY) {
+ return terrno;
+ }
}
taosArrayPush(colsArray, &kvHash);
@@ -1179,23 +1176,25 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
}
static int32_t smlParseLineBottom(SSmlHandle *info) {
- uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
+ uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat,
+ info->lineNum);
if (info->dataFormat) return TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < info->lineNum; i++) {
SSmlLineInfo *elements = info->lines + i;
SSmlTableInfo *tinfo = NULL;
if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
- SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen);
- if(tmp) tinfo = *tmp;
+ SSmlTableInfo **tmp =
+ (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen);
+ if (tmp) tinfo = *tmp;
} else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
- SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
- elements->measureLen + elements->tagsLen);
- if(tmp) tinfo = *tmp;
+ SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
+ elements->measureLen + elements->tagsLen);
+ if (tmp) tinfo = *tmp;
} else {
- SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
- elements->measureLen + elements->tagsLen);
- if(tmp) tinfo = *tmp;
+ SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
+ elements->measureLen + elements->tagsLen);
+ if (tmp) tinfo = *tmp;
}
if (tinfo == NULL) {
@@ -1222,7 +1221,8 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
SSmlSTableMeta **tableMeta =
(SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
if (tableMeta) { // update meta
- uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
+ uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat,
+ info->lineNum);
ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf);
if (ret == TSDB_CODE_SUCCESS) {
ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf);
@@ -1237,10 +1237,13 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
// uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
// return ret;
// }
- uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
+ uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
+ info->lineNum);
SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
- if(terrno == TSDB_CODE_DUP_KEY){return terrno;}
+ if (terrno == TSDB_CODE_DUP_KEY) {
+ return terrno;
+ }
smlInsertMeta(meta->colHash, meta->cols, elements->colArray);
taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
}
@@ -1254,11 +1257,12 @@ static int32_t smlInsertData(SSmlHandle *info) {
int32_t code = TSDB_CODE_SUCCESS;
uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat);
- if(info->pRequest->dbList == NULL){
+ if (info->pRequest->dbList == NULL) {
info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN);
}
- void* data = taosArrayReserve(info->pRequest->dbList, 1);
- memcpy(data, info->pRequest->pDb, TSDB_DB_FNAME_LEN > strlen(info->pRequest->pDb) ? strlen(info->pRequest->pDb) : TSDB_DB_FNAME_LEN);
+ void *data = taosArrayReserve(info->pRequest->dbList, 1);
+ memcpy(data, info->pRequest->pDb,
+ TSDB_DB_FNAME_LEN > strlen(info->pRequest->pDb) ? strlen(info->pRequest->pDb) : TSDB_DB_FNAME_LEN);
SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
while (oneTable) {
@@ -1268,7 +1272,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName));
- if(info->pRequest->tableList == NULL){
+ if (info->pRequest->tableList == NULL) {
info->pRequest->tableList = taosArrayInit(1, sizeof(SName));
}
taosArrayPush(info->pRequest->tableList, &pName);
@@ -1297,7 +1301,8 @@ static int32_t smlInsertData(SSmlHandle *info) {
// use tablemeta of stable to save vgid and uid of child table
(*pMeta)->tableMeta->vgId = vg.vgId;
(*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid
- uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, tableData->uid, info->dataFormat);
+ uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname,
+ tableData->uid, info->dataFormat);
code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
(*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
@@ -1320,7 +1325,8 @@ static int32_t smlInsertData(SSmlHandle *info) {
atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
launchQueryImpl(info->pRequest, info->pQuery, true, NULL);
- uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code));
+ uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code,
+ tstrerror(info->pRequest->code));
return info->pRequest->code;
}
@@ -1331,11 +1337,11 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
" smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
"",
- info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables,
- info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables,
- info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime,
- info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.endTime - info->cost.insertRpcTime,
- info->cost.endTime - info->cost.parseTime);
+ info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables,
+ info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables,
+ info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime,
+ info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
+ info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
}
int32_t smlClearForRerun(SSmlHandle *info) {
@@ -1412,14 +1418,15 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
}
}
- char cTmp = 0; // for print tmp if is raw
- if(info->isRawLine){
+ char cTmp = 0; // for print tmp if is raw
+ if (info->isRawLine) {
cTmp = tmp[len - 1];
tmp[len - 1] = '\0';
}
- uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, info->isRawLine, numLines, info->protocol, len, tmp);
- if(info->isRawLine){
+ uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id,
+ info->isRawLine, numLines, info->protocol, len, tmp);
+ if (info->isRawLine) {
tmp[len - 1] = cTmp;
}
@@ -1515,9 +1522,9 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
return NULL;
}
SRequestObj *request = NULL;
- SSmlHandle *info = NULL;
- int cnt = 0;
- while(1){
+ SSmlHandle *info = NULL;
+ int cnt = 0;
+ while (1) {
request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid);
if (request == NULL) {
uError("SML:taos_schemaless_insert error request is null");
@@ -1571,15 +1578,16 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
request->code = code;
info->cost.endTime = taosGetTimestampUs();
info->cost.code = code;
- if(code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING
- || code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT){
- if(cnt++ >= 10){
- uInfo("SML:%"PRIx64" retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
+ if (code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING ||
+ code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT) {
+ if (cnt++ >= 10) {
+ uInfo("SML:%" PRIx64 " retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
break;
}
taosMsleep(100);
refreshMeta(request->pTscObj, request);
- uInfo("SML:%"PRIx64" retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
+ uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code,
+ tstrerror(code));
smlDestroyInfo(info);
info = NULL;
taos_free_result(request);
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 77bbd0be1a..9e43765b5b 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -24,8 +24,10 @@
#include "tref.h"
#include "ttimer.h"
-#define EMPTY_BLOCK_POLL_IDLE_DURATION 10
-#define DEFAULT_AUTO_COMMIT_INTERVAL 5000
+#define EMPTY_BLOCK_POLL_IDLE_DURATION 10
+#define DEFAULT_AUTO_COMMIT_INTERVAL 5000
+
+typedef void (*__tmq_askep_fn_t)(tmq_t* pTmq, int32_t code, SDataBuf* pBuf, void* pParam);
struct SMqMgmt {
int8_t inited;
@@ -92,8 +94,8 @@ struct tmq_t {
int32_t epSkipCnt;
#endif
// poll info
- int64_t pollCnt;
- int64_t totalRows;
+ int64_t pollCnt;
+ int64_t totalRows;
// timer
tmr_h hbLiveTimer;
@@ -104,11 +106,16 @@ struct tmq_t {
SArray* clientTopics; // SArray
STaosQueue* mqueue; // queue of rsp
STaosQall* qall;
- STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit
- TdThreadMutex lock; // used to protect the operation on each topic, when updating the epsets.
+ STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit
+ TdThreadMutex lock; // used to protect the operation on each topic, when updating the epsets.
tsem_t rspSem;
};
+typedef struct SAskEpInfo {
+ int32_t code;
+ tsem_t sem;
+} SAskEpInfo;
+
enum {
TMQ_VG_STATUS__IDLE = 0,
TMQ_VG_STATUS__WAIT,
@@ -135,7 +142,7 @@ typedef struct {
int32_t vgId;
int32_t vgStatus;
int32_t vgSkipCnt;
- int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data
+ int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data
SEpSet epSet;
} SMqClientVg;
@@ -148,7 +155,7 @@ typedef struct {
typedef struct {
int8_t tmqRspType;
- int32_t epoch; // epoch can be used to guard the vgHandle
+ int32_t epoch; // epoch can be used to guard the vgHandle
int32_t vgId;
SMqClientVg* vgHandle;
SMqClientTopic* topicHandle;
@@ -169,11 +176,10 @@ typedef struct {
} SMqSubscribeCbParam;
typedef struct {
- int64_t refId;
- int32_t epoch;
- int32_t code;
- int32_t async;
- tsem_t rspSem;
+ int64_t refId;
+ int32_t epoch;
+ void* pParam;
+ __tmq_askep_fn_t pUserFn;
} SMqAskEpCbParam;
typedef struct {
@@ -183,22 +189,19 @@ typedef struct {
SMqClientTopic* pTopic;
int32_t vgId;
tsem_t rspSem;
- uint64_t requestId; // request id for debug purpose
+ uint64_t requestId; // request id for debug purpose
} SMqPollCbParam;
typedef struct {
int64_t refId;
int32_t epoch;
- int8_t automatic;
- int8_t async;
int32_t waitingRspNum;
int32_t totalRspNum;
- int32_t rspErr;
- tmq_commit_cb* userCb;
+ int32_t code;
+ tmq_commit_cb* callbackFn;
/*SArray* successfulOffsets;*/
/*SArray* failedOffsets;*/
- void* userParam;
- tsem_t rspSem;
+ void* userParam;
} SMqCommitCbParamSet;
typedef struct {
@@ -209,12 +212,14 @@ typedef struct {
tmq_t* pTmq;
} SMqCommitCbParam;
-static int32_t tmqAskEp(tmq_t* tmq, bool async);
+static int32_t doAskEp(tmq_t* tmq);
static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg);
static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet);
static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet,
int32_t index, int32_t totalVgroups);
-static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId);
+static void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId);
+static void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param);
+static void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param);
tmq_conf_t* tmq_conf_new() {
tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t));
@@ -320,15 +325,16 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
}
if (strcasecmp(key, "enable.heartbeat.background") == 0) {
- if (strcasecmp(value, "true") == 0) {
- conf->hbBgEnable = true;
- return TMQ_CONF_OK;
- } else if (strcasecmp(value, "false") == 0) {
- conf->hbBgEnable = false;
- return TMQ_CONF_OK;
- } else {
- return TMQ_CONF_INVALID;
- }
+ // if (strcasecmp(value, "true") == 0) {
+ // conf->hbBgEnable = true;
+ // return TMQ_CONF_OK;
+ // } else if (strcasecmp(value, "false") == 0) {
+ // conf->hbBgEnable = false;
+ // return TMQ_CONF_OK;
+ // } else {
+ tscError("the default value of enable.heartbeat.background is true, can not be seted");
+ return TMQ_CONF_INVALID;
+ // }
}
if (strcasecmp(key, "td.connect.ip") == 0) {
@@ -358,9 +364,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
return TMQ_CONF_UNKNOWN;
}
-tmq_list_t* tmq_list_new() {
- return (tmq_list_t*)taosArrayInit(0, sizeof(void*));
-}
+tmq_list_t* tmq_list_new() { return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); }
int32_t tmq_list_append(tmq_list_t* list, const char* src) {
SArray* container = &list->container;
@@ -388,12 +392,13 @@ char** tmq_list_to_c_array(const tmq_list_t* list) {
return container->pData;
}
-static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t vgId, int32_t* index, int32_t* numOfVgroups) {
+static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t vgId, int32_t* index,
+ int32_t* numOfVgroups) {
int32_t numOfTopics = taosArrayGetSize(pTopicList);
*index = -1;
*numOfVgroups = 0;
- for(int32_t i = 0; i < numOfTopics; ++i) {
+ for (int32_t i = 0; i < numOfTopics; ++i) {
SMqClientTopic* pTopic = taosArrayGet(pTopicList, i);
if (strcmp(pTopic->topicName, pName) != 0) {
continue;
@@ -419,41 +424,42 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) {
SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params;
-// if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again
-// taosThreadMutexLock(&pParam->pTmq->lock);
-// int32_t numOfVgroups, index;
-// SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups);
-// if (pVg == NULL) {
-// tscDebug("consumer:0x%" PRIx64
-// " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry ordinal:%d/%d",
-// pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), index + 1, numOfVgroups);
-// } else { // let's retry the commit
-// int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups);
-// if (code1 != TSDB_CODE_SUCCESS) { // retry failed.
-// tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64
-// " retry failed, ignore this commit. code:%s ordinal:%d/%d",
-// pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->committedOffset.version,
-// tstrerror(terrno), index + 1, numOfVgroups);
-// }
-// }
-//
-// taosThreadMutexUnlock(&pParam->pTmq->lock);
-//
-// taosMemoryFree(pParam->pOffset);
-// taosMemoryFree(pBuf->pData);
-// taosMemoryFree(pBuf->pEpSet);
-//
-// tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId);
-// return 0;
-// }
-//
-// // todo replace the pTmq with refId
+ // if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again
+ // taosThreadMutexLock(&pParam->pTmq->lock);
+ // int32_t numOfVgroups, index;
+ // SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index,
+ // &numOfVgroups); if (pVg == NULL) {
+ // tscDebug("consumer:0x%" PRIx64
+ // " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry
+ // ordinal:%d/%d", pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code),
+ // index + 1, numOfVgroups);
+ // } else { // let's retry the commit
+ // int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups);
+ // if (code1 != TSDB_CODE_SUCCESS) { // retry failed.
+ // tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64
+ // " retry failed, ignore this commit. code:%s ordinal:%d/%d",
+ // pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->committedOffset.version,
+ // tstrerror(terrno), index + 1, numOfVgroups);
+ // }
+ // }
+ //
+ // taosThreadMutexUnlock(&pParam->pTmq->lock);
+ //
+ // taosMemoryFree(pParam->pOffset);
+ // taosMemoryFree(pBuf->pData);
+ // taosMemoryFree(pBuf->pEpSet);
+ //
+ // commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId);
+ // return 0;
+ // }
+ //
+ // // todo replace the pTmq with refId
taosMemoryFree(pParam->pOffset);
taosMemoryFree(pBuf->pData);
taosMemoryFree(pBuf->pEpSet);
- tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId);
+ commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId);
return 0;
}
@@ -461,8 +467,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN
int32_t index, int32_t totalVgroups) {
STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset));
if (pOffset == NULL) {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- return -1;
+ return TSDB_CODE_OUT_OF_MEMORY;
}
pOffset->val = pVg->currentOffset;
@@ -476,13 +481,13 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN
int32_t code = 0;
tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
if (code < 0) {
- return -1;
+ return TSDB_CODE_INVALID_PARA;
}
void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
if (buf == NULL) {
taosMemoryFree(pOffset);
- return -1;
+ return TSDB_CODE_OUT_OF_MEMORY;
}
((SMsgHead*)buf)->vgId = htonl(pVg->vgId);
@@ -499,7 +504,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN
if (pParam == NULL) {
taosMemoryFree(pOffset);
taosMemoryFree(buf);
- return -1;
+ return TSDB_CODE_OUT_OF_MEMORY;
}
pParam->params = pParamSet;
@@ -515,7 +520,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN
taosMemoryFree(pOffset);
taosMemoryFree(buf);
taosMemoryFree(pParam);
- return -1;
+ return TSDB_CODE_OUT_OF_MEMORY;
}
pMsgSendInfo->msgInfo = (SDataBuf){
@@ -535,136 +540,128 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN
atomic_add_fetch_32(&pParamSet->totalRspNum, 1);
SEp* pEp = GET_ACTIVE_EP(&pVg->epSet);
- tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%" PRId64 " prev:%" PRId64
- ", ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64,
- tmq->consumerId, pOffset->subKey, pVg->vgId, pOffset->val.version, pVg->committedOffset.version, pEp->fqdn,
- pEp->port, index + 1, totalVgroups, pMsgSendInfo->requestId);
+ char offsetBuf[80] = {0};
+ tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffset->val);
+
+ char commitBuf[80] = {0};
+ tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->committedOffset);
+ tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%s prev:%s, ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64,
+ tmq->consumerId, pOffset->subKey, pVg->vgId, offsetBuf, commitBuf, pEp->fqdn, pEp->port, index + 1,
+ totalVgroups, pMsgSendInfo->requestId);
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo);
- return 0;
+
+ return TSDB_CODE_SUCCESS;
}
-static int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) {
- char* topic;
- int32_t vgId;
- if (TD_RES_TMQ(msg)) {
- SMqRspObj* pRspObj = (SMqRspObj*)msg;
- topic = pRspObj->topic;
+static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* pCommitFp, void* userParam) {
+ char* pTopicName = NULL;
+ int32_t vgId = 0;
+ int32_t code = 0;
+
+ if (pRes == NULL || tmq == NULL) {
+ pCommitFp(tmq, TSDB_CODE_INVALID_PARA, userParam);
+ return;
+ }
+
+ if (TD_RES_TMQ(pRes)) {
+ SMqRspObj* pRspObj = (SMqRspObj*)pRes;
+ pTopicName = pRspObj->topic;
vgId = pRspObj->vgId;
- } else if (TD_RES_TMQ_META(msg)) {
- SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg;
- topic = pMetaRspObj->topic;
+ } else if (TD_RES_TMQ_META(pRes)) {
+ SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes;
+ pTopicName = pMetaRspObj->topic;
vgId = pMetaRspObj->vgId;
- } else if (TD_RES_TMQ_METADATA(msg)) {
- SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)msg;
- topic = pRspObj->topic;
+ } else if (TD_RES_TMQ_METADATA(pRes)) {
+ SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes;
+ pTopicName = pRspObj->topic;
vgId = pRspObj->vgId;
} else {
- return TSDB_CODE_TMQ_INVALID_MSG;
+ pCommitFp(tmq, TSDB_CODE_TMQ_INVALID_MSG, userParam);
+ return;
}
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
if (pParamSet == NULL) {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- return -1;
+ pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam);
+ return;
}
pParamSet->refId = tmq->refId;
pParamSet->epoch = tmq->epoch;
- pParamSet->automatic = 0;
- pParamSet->async = async;
- pParamSet->userCb = userCb;
+ pParamSet->callbackFn = pCommitFp;
pParamSet->userParam = userParam;
- tsem_init(&pParamSet->rspSem, 0, 0);
- int32_t code = -1;
-
- taosThreadMutexLock(&tmq->lock);
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
- tscDebug("consumer:0x%" PRIx64 " user invoked commit offset for %d", tmq->consumerId, numOfTopics);
- for (int32_t i = 0; i < numOfTopics; i++) {
+ tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId);
+
+ int32_t i = 0;
+ for (; i < numOfTopics; i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
- if (strcmp(pTopic->topicName, topic) != 0) {
- continue;
- }
-
- int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
- for (int32_t j = 0; j < numOfVgroups; j++) {
- SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
- if (pVg->vgId != vgId) {
- continue;
- }
-
- if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) {
- if (doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups) < 0) {
- tsem_destroy(&pParamSet->rspSem);
- taosMemoryFree(pParamSet);
- goto FAIL;
- }
- goto HANDLE_RSP;
- }
+ if (strcmp(pTopic->topicName, pTopicName) == 0) {
+ break;
}
}
-HANDLE_RSP:
- if (pParamSet->totalRspNum == 0) {
- tsem_destroy(&pParamSet->rspSem);
+ if (i == numOfTopics) {
+ tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId,
+ pTopicName, numOfTopics);
taosMemoryFree(pParamSet);
- taosThreadMutexUnlock(&tmq->lock);
- return 0;
+ pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam);
+ return;
}
- if (!async) {
- taosThreadMutexUnlock(&tmq->lock);
- tsem_wait(&pParamSet->rspSem);
- code = pParamSet->rspErr;
- tsem_destroy(&pParamSet->rspSem);
+ SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
+
+ int32_t j = 0;
+ int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
+ for (j = 0; j < numOfVgroups; j++) {
+ SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
+ if (pVg->vgId == vgId) {
+ break;
+ }
+ }
+
+ if (j == numOfVgroups) {
+ tscWarn("consumer:0x%" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s", tmq->consumerId,
+ vgId, numOfVgroups, pTopicName);
taosMemoryFree(pParamSet);
- return code;
- } else {
- code = 0;
+ pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam);
+ return;
}
-FAIL:
- taosThreadMutexUnlock(&tmq->lock);
- if (code != 0 && async) {
- userCb(tmq, code, userParam);
- }
+ SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
+ if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) {
+ code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups);
- return 0;
+ // failed to commit, callback user function directly.
+ if (code != TSDB_CODE_SUCCESS) {
+ taosMemoryFree(pParamSet);
+ pCommitFp(tmq, code, userParam);
+ }
+ } else { // do not perform commit, callback user function directly.
+ taosMemoryFree(pParamSet);
+ pCommitFp(tmq, code, userParam);
+ }
}
-static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) {
- int32_t code = -1;
-
+static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) {
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
if (pParamSet == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- if (async) {
- if (automatic) {
- tmq->commitCb(tmq, code, tmq->commitCbUserParam);
- } else {
- userCb(tmq, code, userParam);
- }
- }
- return -1;
+ pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam);
+ return;
}
pParamSet->refId = tmq->refId;
pParamSet->epoch = tmq->epoch;
-
- pParamSet->automatic = automatic;
- pParamSet->async = async;
- pParamSet->userCb = userCb;
+ pParamSet->callbackFn = pCommitFp;
pParamSet->userParam = userParam;
- tsem_init(&pParamSet->rspSem, 0, 0);
// init as 1 to prevent concurrency issue
pParamSet->waitingRspNum = 1;
- taosThreadMutexLock(&tmq->lock);
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics);
@@ -678,7 +675,7 @@ static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_comm
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) {
- code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups);
+ int32_t code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups);
if (code != TSDB_CODE_SUCCESS) {
tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 " failed, code:%s ordinal:%d/%d",
tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->committedOffset.version, tstrerror(terrno),
@@ -697,39 +694,16 @@ static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_comm
tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1,
numOfTopics);
- taosThreadMutexUnlock(&tmq->lock);
// no request is sent
if (pParamSet->totalRspNum == 0) {
- tsem_destroy(&pParamSet->rspSem);
taosMemoryFree(pParamSet);
- return 0;
+ pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam);
+ return;
}
// count down since waiting rsp num init as 1
- tmqCommitRspCountDown(pParamSet, tmq->consumerId, "", 0);
-
- if (!async) {
- tsem_wait(&pParamSet->rspSem);
- code = pParamSet->rspErr;
- tsem_destroy(&pParamSet->rspSem);
- taosMemoryFree(pParamSet);
-#if 0
- taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree);
- taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree);
-#endif
- }
-
- return code;
-}
-
-static int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb,
- void* userParam) {
- if (msg) { // user invoked commit
- return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam);
- } else { // this for auto commit
- return doAutoCommit(tmq, automatic, async, userCb, userParam);
- }
+ commitRspCountDown(pParamSet, tmq->consumerId, "", 0);
}
static void generateTimedTask(int64_t refId, int32_t type) {
@@ -780,7 +754,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
void tmqSendHbReq(void* param, void* tmrId) {
int64_t refId = *(int64_t*)param;
- tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
+ tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) {
taosMemoryFree(param);
return;
@@ -836,6 +810,12 @@ OVER:
taosReleaseRef(tmqMgmt.rsetId, refId);
}
+static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) {
+ if (code != 0) {
+ tscDebug("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code));
+ }
+}
+
int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
STaosQall* qall = taosAllocateQall();
taosReadAllQitems(pTmq->delayedTask, qall);
@@ -851,7 +831,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
while (pTaskType != NULL) {
if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) {
- tmqAskEp(pTmq, true);
+ asyncAskEp(pTmq, addToQueueCallbackFn, NULL);
int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
*pRefId = pTmq->refId;
@@ -859,12 +839,13 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
taosTmrReset(tmqAssignAskEpTask, 1000, pRefId, tmqMgmt.timer, &pTmq->epTimer);
} else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
- tmqCommitInner(pTmq, NULL, 1, 1, pTmq->commitCb, pTmq->commitCbUserParam);
+ tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn;
+ asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
*pRefId = pTmq->refId;
- tscDebug("consumer:0x%" PRIx64 " commit to vnode(s) in %.2fs", pTmq->consumerId,
+ tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId,
pTmq->autoCommitInterval / 1000.0);
taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, pRefId, tmqMgmt.timer, &pTmq->commitTimer);
} else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) {
@@ -1047,8 +1028,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL ||
conf->groupId[0] == 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
- tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(),
- pTmq->groupId);
+ tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
goto _failed;
}
@@ -1056,8 +1036,6 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->status = TMQ_CONSUMER_STATUS__INIT;
pTmq->pollCnt = 0;
pTmq->epoch = 0;
- /*pTmq->epStatus = 0;*/
- /*pTmq->epSkipCnt = 0;*/
// set conf
strcpy(pTmq->clientId, conf->clientId);
@@ -1104,9 +1082,10 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
char buf[80] = {0};
STqOffsetVal offset = {.type = pTmq->resetOffsetCfg};
tFormatOffset(buf, tListLen(buf), &offset);
- tscInfo("consumer:0x%" PRIx64 " is setup, refId:%"PRId64", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s, backgroudHB:%d",
- pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf,
- pTmq->hbBgEnable);
+ tscInfo("consumer:0x%" PRIx64 " is setup, refId:%" PRId64
+ ", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s, backgroudHB:%d",
+ pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval,
+ buf, pTmq->hbBgEnable);
return pTmq;
@@ -1116,6 +1095,7 @@ _failed:
}
int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
+ const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most
const SArray* container = &topic_list->container;
int32_t sz = taosArrayGetSize(container);
void* buf = NULL;
@@ -1208,8 +1188,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
}
int32_t retryCnt = 0;
- while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) {
- if (retryCnt++ > 40) {
+ while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) {
+ if (retryCnt++ > MAX_RETRY_COUNT) {
goto FAIL;
}
@@ -1278,7 +1258,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) {
taosMsleep(500);
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER);
- tscDebug("consumer:0x%" PRIx64" wait for the re-balance, wait for 500ms and set status to be RECOVER", tmq->consumerId);
+ tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, wait for 500ms and set status to be RECOVER",
+ tmq->consumerId);
} else if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0);
if (pRspWrapper == NULL) {
@@ -1298,7 +1279,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
if (msgEpoch < tmqEpoch) {
// do not write into queue since updating epoch reset
- tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%"PRIx64,
+ tscWarn("consumer:0x%" PRIx64
+ " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64,
tmq->consumerId, vgId, msgEpoch, tmqEpoch, requestId);
tsem_post(&tmq->rspSem);
@@ -1321,7 +1303,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (pRspWrapper == NULL) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
- tscWarn("consumer:0x%"PRIx64" msg discard from vgId:%d, epoch %d since out of memory", tmq->consumerId, vgId, epoch);
+ tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, epoch %d since out of memory", tmq->consumerId, vgId,
+ epoch);
goto CREATE_MSG_FAIL;
}
@@ -1342,7 +1325,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
char buf[80];
tFormatOffset(buf, 80, &pRspWrapper->dataRsp.rspOffset);
- tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64,
+ tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64,
tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId);
} else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SDecoder decoder;
@@ -1356,8 +1339,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp);
tDecoderClear(&decoder);
memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead));
- } else { // invalid rspType
- tscError("consumer:0x%"PRIx64" invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
+ } else { // invalid rspType
+ tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
}
taosMemoryFree(pMsg->pData);
@@ -1393,7 +1376,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
pTopicEp->schema.nCols = 0;
pTopicEp->schema.pSchema = NULL;
- char vgKey[TSDB_TOPIC_FNAME_LEN + 22];
+ char vgKey[TSDB_TOPIC_FNAME_LEN + 22];
int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs);
tstrncpy(pTopic->topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN);
@@ -1408,8 +1391,8 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
makeTopicVgroupKey(vgKey, pTopic->topicName, pVgEp->vgId);
SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey));
- int64_t numOfRows = 0;
- STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg};
+ int64_t numOfRows = 0;
+ STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg};
if (pInfo != NULL) {
offsetNew = pInfo->offset;
numOfRows = pInfo->numOfRows;
@@ -1473,8 +1456,8 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) {
char buf[80];
tFormatOffset(buf, 80, &pVgCur->currentOffset);
- tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch,
- pVgCur->vgId, vgKey, buf);
+ tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId,
+ vgKey, buf);
SVgroupSaveInfo info = {.offset = pVgCur->currentOffset, .numOfRows = pVgCur->numOfRows};
taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo));
@@ -1500,7 +1483,7 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) {
tmq->clientTopics = newTopics;
taosThreadMutexUnlock(&tmq->lock);
- int8_t flag = (topicNumGet == 0)? TMQ_CONSUMER_STATUS__NO_TOPIC:TMQ_CONSUMER_STATUS__READY;
+ int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY;
atomic_store_8(&tmq->status, flag);
atomic_store_32(&tmq->epoch, epoch);
@@ -1508,28 +1491,30 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) {
return set;
}
-static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
+int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
- int8_t async = pParam->async;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
if (tmq == NULL) {
- if (!async) {
- tsem_destroy(&pParam->rspSem);
- } else {
- taosMemoryFree(pParam);
- }
+ terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED;
+ pParam->pUserFn(tmq, terrno, NULL, pParam->pParam);
+
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
- terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED;
- return -1;
+ taosMemoryFree(pParam);
+ return terrno;
}
- pParam->code = code;
if (code != TSDB_CODE_SUCCESS) {
- tscError("consumer:0x%" PRIx64 ", get topic endpoint error, async:%d, code:%s", tmq->consumerId, pParam->async,
- tstrerror(code));
- goto END;
+ tscError("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code));
+ pParam->pUserFn(tmq, code, NULL, pParam->pParam);
+
+ taosReleaseRef(tmqMgmt.rsetId, pParam->refId);
+
+ taosMemoryFree(pMsg->pData);
+ taosMemoryFree(pMsg->pEpSet);
+ taosMemoryFree(pParam);
+ return code;
}
// tmq's epoch is monotonically increase,
@@ -1540,6 +1525,7 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
if (head->epoch <= epoch) {
tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep",
tmq->consumerId, head->epoch, epoch);
+
if (tmq->status == TMQ_CONSUMER_STATUS__RECOVER) {
SMqAskEpRsp rsp;
tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
@@ -1548,45 +1534,17 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
tDeleteSMqAskEpRsp(&rsp);
}
- goto END;
- }
-
- tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId,
- head->epoch, epoch);
-
- if (!async) {
- SMqAskEpRsp rsp;
- tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
- tmqUpdateEp(tmq, head->epoch, &rsp);
- tDeleteSMqAskEpRsp(&rsp);
} else {
- SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0);
- if (pWrapper == NULL) {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- code = -1;
- goto END;
- }
-
- pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
- pWrapper->epoch = head->epoch;
- memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
- tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg);
-
- taosWriteQitem(tmq->mqueue, pWrapper);
- tsem_post(&tmq->rspSem);
+ tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId,
+ head->epoch, epoch);
+ pParam->pUserFn(tmq, code, pMsg, pParam->pParam);
}
-END:
taosReleaseRef(tmqMgmt.rsetId, pParam->refId);
- if (!async) {
- tsem_post(&pParam->rspSem);
- } else {
- taosMemoryFree(pParam);
- }
-
taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pMsg->pData);
+ taosMemoryFree(pParam);
return code;
}
@@ -1638,9 +1596,9 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg,
}
// extract the rows in this data packet
- for(int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
+ for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i);
- int64_t rows = htobe64(pRetrieve->numOfRows);
+ int64_t rows = htobe64(pRetrieve->numOfRows);
pVg->numOfRows += rows;
(*numOfRows) += rows;
}
@@ -1727,8 +1685,8 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
char offsetFormatBuf[80];
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->currentOffset);
- tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64,
- pTmq->consumerId, pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId);
+ tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId,
+ pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId);
asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
pVg->pollCnt++;
@@ -1748,9 +1706,9 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
for (int j = 0; j < numOfVg; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
- if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms
- tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch,
- pVg->vgId);
+ if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms
+ tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId,
+ tmq->epoch, pVg->vgId);
continue;
}
@@ -1811,13 +1769,12 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
if (pRspWrapper == NULL) {
taosReadAllQitems(tmq->mqueue, tmq->qall);
taosGetQitem(tmq->qall, (void**)&pRspWrapper);
-
if (pRspWrapper == NULL) {
return NULL;
}
}
- tscDebug("consumer:0x%"PRIx64" handle rsp, type:%d", tmq->consumerId, pRspWrapper->tmqRspType);
+ tscDebug("consumer:0x%" PRIx64 " handle rsp, type:%d", tmq->consumerId, pRspWrapper->tmqRspType);
if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) {
taosFreeQitem(pRspWrapper);
@@ -1827,34 +1784,35 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
- int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
+ int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp;
if (pDataRsp->head.epoch == consumerEpoch) {
- // todo fix it: race condition
SMqClientVg* pVg = pollRspWrapper->vgHandle;
// update the epset
if (pollRspWrapper->pEpset != NULL) {
SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset);
SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet));
- tscDebug("consumer:0x%" PRIx64 " update epset vgId:%d, ep:%s:%d, old ep:%s:%d", tmq->consumerId,
- pVg->vgId, pEp->fqdn, pEp->port, pOld->fqdn, pOld->port);
+ tscDebug("consumer:0x%" PRIx64 " update epset vgId:%d, ep:%s:%d, old ep:%s:%d", tmq->consumerId, pVg->vgId,
+ pEp->fqdn, pEp->port, pOld->fqdn, pOld->port);
pVg->epSet = *pollRspWrapper->pEpset;
}
+ // update the local offset value only for the returned values.
pVg->currentOffset = pDataRsp->rspOffset;
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
char buf[80];
tFormatOffset(buf, 80, &pDataRsp->rspOffset);
if (pDataRsp->blockNum == 0) {
- tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%"PRId64" total:%"PRId64" reqId:0x%" PRIx64, tmq->consumerId,
- pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId);
+ tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64
+ " total:%" PRId64 " reqId:0x%" PRIx64,
+ tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId);
pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pollRspWrapper);
} else { // build rsp
- int64_t numOfRows = 0;
+ int64_t numOfRows = 0;
SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
tmq->totalRows += numOfRows;
@@ -1893,7 +1851,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
}
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
- int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
+ int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) {
SMqClientVg* pVg = pollRspWrapper->vgHandle;
@@ -1908,11 +1866,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
taosFreeQitem(pollRspWrapper);
continue;
} else {
- pVg->emptyBlockReceiveTs = 0; // reset the ts
+ pVg->emptyBlockReceiveTs = 0; // reset the ts
}
// build rsp
- void* pRsp = NULL;
+ void* pRsp = NULL;
int64_t numOfRows = 0;
if (pollRspWrapper->taosxRsp.createTableNum == 0) {
pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
@@ -1925,7 +1883,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
char buf[80];
tFormatOffset(buf, 80, &pVg->currentOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
- ", vg total:%" PRId64 " total:%"PRId64" reqId:0x%" PRIx64,
+ ", vg total:%" PRId64 " total:%" PRId64 " reqId:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows,
tmq->totalRows, pollRspWrapper->reqId);
@@ -1956,7 +1914,8 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
void* rspObj;
int64_t startTime = taosGetTimestampMs();
- tscDebug("consumer:0x%" PRIx64 " start to poll at %"PRId64", timeout:%" PRId64, tmq->consumerId, startTime, timeout);
+ tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime,
+ timeout);
#if 0
tmqHandleAllDelayedTask(tmq);
@@ -1976,7 +1935,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
while (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) {
int32_t retryCnt = 0;
- while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) {
+ while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) {
if (retryCnt++ > 40) {
return NULL;
}
@@ -2039,7 +1998,7 @@ static void displayConsumeStatistics(const tmq_t* pTmq) {
}
int32_t tmq_consumer_close(tmq_t* tmq) {
- tscDebug("consumer:0x%" PRIx64" start to close consumer, status:%d", tmq->consumerId, tmq->status);
+ tscDebug("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status);
displayConsumeStatistics(tmq);
if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
@@ -2065,7 +2024,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
tmq_list_destroy(lst);
} else {
- tscWarn("consumer:0x%" PRIx64" not in ready state, close it directly", tmq->consumerId);
+ tscWarn("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId);
}
taosRemoveRef(tmqMgmt.rsetId, tmq->refId);
@@ -2158,96 +2117,162 @@ const char* tmq_get_table_name(TAOS_RES* res) {
return NULL;
}
-void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) {
- tmqCommitInner(tmq, msg, 0, 1, cb, param);
-}
-
-int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) {
- return tmqCommitInner(tmq, msg, 0, 0, NULL, NULL);
-}
-
-int32_t tmqAskEp(tmq_t* tmq, bool async) {
- int32_t code = TSDB_CODE_SUCCESS;
-#if 0
- int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
- if (epStatus == 1) {
- int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
- tscTrace("consumer:0x%" PRIx64 ", skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
- if (epSkipCnt < 5000) return 0;
+void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* param) {
+ if (pRes == NULL) { // here needs to commit all offsets.
+ asyncCommitAllOffsets(tmq, cb, param);
+ } else { // only commit one offset
+ asyncCommitOffset(tmq, pRes, cb, param);
}
- atomic_store_32(&tmq->epSkipCnt, 0);
-#endif
+}
+typedef struct SSyncCommitInfo {
+ tsem_t sem;
+ int32_t code;
+} SSyncCommitInfo;
+
+static void commitCallBackFn(tmq_t* pTmq, int32_t code, void* param) {
+ SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param;
+ pInfo->code = code;
+ tsem_post(&pInfo->sem);
+}
+
+int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) {
+ int32_t code = 0;
+
+ SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo));
+ tsem_init(&pInfo->sem, 0, 0);
+ pInfo->code = 0;
+
+ if (pRes == NULL) {
+ asyncCommitAllOffsets(tmq, commitCallBackFn, pInfo);
+ } else {
+ asyncCommitOffset(tmq, pRes, commitCallBackFn, pInfo);
+ }
+
+ tsem_wait(&pInfo->sem);
+ code = pInfo->code;
+
+ tsem_destroy(&pInfo->sem);
+ taosMemoryFree(pInfo);
+
+ tscDebug("consumer:0x%" PRIx64 " sync commit done, code:%s", tmq->consumerId, tstrerror(code));
+ return code;
+}
+
+void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) {
+ SAskEpInfo* pInfo = param;
+ pInfo->code = code;
+
+ if (code == TSDB_CODE_SUCCESS) {
+ SMqRspHead* head = pDataBuf->pData;
+
+ SMqAskEpRsp rsp;
+ tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &rsp);
+ tmqUpdateEp(pTmq, head->epoch, &rsp);
+ tDeleteSMqAskEpRsp(&rsp);
+ }
+
+ tsem_post(&pInfo->sem);
+}
+
+void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) {
+ if (code != TSDB_CODE_SUCCESS) {
+ terrno = code;
+ return;
+ }
+
+ SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0);
+ if (pWrapper == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return;
+ }
+
+ SMqRspHead* head = pDataBuf->pData;
+
+ pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
+ pWrapper->epoch = head->epoch;
+ memcpy(&pWrapper->msg, pDataBuf->pData, sizeof(SMqRspHead));
+ tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &pWrapper->msg);
+
+ taosWriteQitem(pTmq->mqueue, pWrapper);
+}
+
+int32_t doAskEp(tmq_t* pTmq) {
+ SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo));
+ tsem_init(&pInfo->sem, 0, 0);
+
+ asyncAskEp(pTmq, updateEpCallbackFn, pInfo);
+ tsem_wait(&pInfo->sem);
+
+ int32_t code = pInfo->code;
+ tsem_destroy(&pInfo->sem);
+ taosMemoryFree(pInfo);
+ return code;
+}
+
+void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param) {
SMqAskEpReq req = {0};
- req.consumerId = tmq->consumerId;
- req.epoch = tmq->epoch;
- strcpy(req.cgroup, tmq->groupId);
+ req.consumerId = pTmq->consumerId;
+ req.epoch = pTmq->epoch;
+ strcpy(req.cgroup, pTmq->groupId);
int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req);
if (tlen < 0) {
- tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", tmq->consumerId);
- return -1;
+ tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", pTmq->consumerId);
+ askEpFn(pTmq, TSDB_CODE_INVALID_PARA, NULL, param);
+ return;
}
void* pReq = taosMemoryCalloc(1, tlen);
if (pReq == NULL) {
- tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", tmq->consumerId, tlen);
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- return -1;
+ tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", pTmq->consumerId, tlen);
+ askEpFn(pTmq, TSDB_CODE_OUT_OF_MEMORY, NULL, param);
+ return;
}
if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) {
- tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", tmq->consumerId, tlen);
+ tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", pTmq->consumerId, tlen);
taosMemoryFree(pReq);
- return -1;
+
+ askEpFn(pTmq, TSDB_CODE_INVALID_PARA, NULL, param);
+ return;
}
SMqAskEpCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqAskEpCbParam));
if (pParam == NULL) {
- tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", tmq->consumerId);
+ tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId);
taosMemoryFree(pReq);
- return -1;
+
+ askEpFn(pTmq, TSDB_CODE_OUT_OF_MEMORY, NULL, param);
+ return;
}
- pParam->refId = tmq->refId;
- pParam->epoch = tmq->epoch;
- pParam->async = async;
- tsem_init(&pParam->rspSem, 0, 0);
+ pParam->refId = pTmq->refId;
+ pParam->epoch = pTmq->epoch;
+ pParam->pUserFn = askEpFn;
+ pParam->pParam = param;
SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
- tsem_destroy(&pParam->rspSem);
taosMemoryFree(pParam);
taosMemoryFree(pReq);
- return -1;
+ askEpFn(pTmq, TSDB_CODE_OUT_OF_MEMORY, NULL, param);
+ return;
}
- sendInfo->msgInfo = (SDataBuf){
- .pData = pReq,
- .len = tlen,
- .handle = NULL,
- };
+ sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL};
sendInfo->requestId = generateRequestId();
sendInfo->requestObjRefId = 0;
sendInfo->param = pParam;
- sendInfo->fp = tmqAskEpCb;
+ sendInfo->fp = askEpCallbackFn;
sendInfo->msgType = TDMT_MND_TMQ_ASK_EP;
- SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
- tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d, reqId:0x%" PRIx64, tmq->consumerId, async,
- sendInfo->requestId);
+ SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp);
+ tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId);
int64_t transporterId = 0;
- asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
-
- if (!async) {
- tsem_wait(&pParam->rspSem);
- code = pParam->code;
- taosMemoryFree(pParam);
- }
-
- return code;
+ asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
}
int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg) {
@@ -2259,38 +2284,20 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) {
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) {
- if (!pParamSet->async) {
- tsem_destroy(&pParamSet->rspSem);
- }
taosMemoryFree(pParamSet);
terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED;
return -1;
}
// if no more waiting rsp
- if (pParamSet->async) {
- // call async cb func
- if (pParamSet->automatic && tmq->commitCb) {
- tmq->commitCb(tmq, pParamSet->rspErr, tmq->commitCbUserParam);
- } else if (!pParamSet->automatic && pParamSet->userCb) { // sem post
- pParamSet->userCb(tmq, pParamSet->rspErr, pParamSet->userParam);
- }
-
- taosMemoryFree(pParamSet);
- } else {
- tsem_post(&pParamSet->rspSem);
- }
-
-#if 0
- taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree);
- taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree);
-#endif
+ pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam);
+ taosMemoryFree(pParamSet);
taosReleaseRef(tmqMgmt.rsetId, refId);
return 0;
}
-void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) {
+void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) {
int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1);
if (waitingRspNum == 0) {
tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic,
diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp
index 5523e1f777..517d8e0221 100644
--- a/source/client/test/clientTests.cpp
+++ b/source/client/test/clientTests.cpp
@@ -52,15 +52,15 @@ void printResult(TAOS_RES* pRes) {
int32_t n = 0;
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
-// int32_t* length = taos_fetch_lengths(pRes);
-// for(int32_t i = 0; i < numOfFields; ++i) {
-// printf("(%d):%d " , i, length[i]);
-// }
-// printf("\n");
-//
-// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-// printf("%s\n", str);
-// memset(str, 0, sizeof(str));
+ // int32_t* length = taos_fetch_lengths(pRes);
+ // for(int32_t i = 0; i < numOfFields; ++i) {
+ // printf("(%d):%d " , i, length[i]);
+ // }
+ // printf("\n");
+ //
+ // int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+ // printf("%s\n", str);
+ // memset(str, 0, sizeof(str));
}
}
@@ -112,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
}
taos_free_result(pRes);
- for(int32_t i = 0; i < 100; i += 20) {
+ for (int32_t i = 0; i < 100; i += 20) {
char sql[1024] = {0};
sprintf(sql,
"insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
@@ -131,7 +131,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
}
}
-void *queryThread(void *arg) {
+void* queryThread(void* arg) {
TAOS* pConn = taos_connect("192.168.0.209", "root", "taosdata", NULL, 0);
if (pConn == NULL) {
printf("failed to connect to db, reason:%s", taos_errstr(pConn));
@@ -141,9 +141,10 @@ void *queryThread(void *arg) {
int64_t el = 0;
for (int32_t i = 0; i < 5000000; ++i) {
- int64_t st = taosGetTimestampUs();
+ int64_t st = taosGetTimestampUs();
TAOS_RES* pRes = taos_query(pConn,
- "SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;");
+ "SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= "
+ "1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;");
if (taos_errno(pRes) != 0) {
printf("failed, reason:%s\n", taos_errstr(pRes));
} else {
@@ -153,7 +154,7 @@ void *queryThread(void *arg) {
taos_free_result(pRes);
el += (taosGetTimestampUs() - st);
if (i % 1000 == 0 && i != 0) {
- printf("total:%d, avg time:%.2fms\n", i, el/(double)(i*1000));
+ printf("total:%d, avg time:%.2fms\n", i, el / (double)(i * 1000));
}
}
@@ -163,9 +164,7 @@ void *queryThread(void *arg) {
static int32_t numOfThreads = 1;
-void tmq_commit_cb_print(tmq_t *pTmq, int32_t code, void *param) {
- printf("success, code:%d\n", code);
-}
+void tmq_commit_cb_print(tmq_t* pTmq, int32_t code, void* param) { printf("success, code:%d\n", code); }
void* doConsumeData(void* param) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -205,7 +204,7 @@ void* doConsumeData(void* param) {
while (1) {
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
if (pRes) {
- char buf[1024];
+ char buf[1024];
const char* topicName = tmq_get_topic_name(pRes);
const char* dbName = tmq_get_db_name(pRes);
@@ -226,7 +225,7 @@ void* doConsumeData(void* param) {
precision = taos_result_precision(pRes);
taos_print_row(buf, row, fields, numOfFields);
totalRows += 1;
-// printf("precision: %d, row content: %s\n", precision, buf);
+ // printf("precision: %d, row content: %s\n", precision, buf);
}
taos_free_result(pRes);
@@ -491,17 +490,17 @@ TEST(clientCase, create_stable_Test) {
ASSERT_EQ(numOfFields, 0);
taos_free_result(pRes);
-// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)");
-// if (taos_errno(pRes) != 0) {
-// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
-// }
-//
-// pRes = taos_query(pConn, "use abc1");
-// taos_free_result(pRes);
-// pRes = taos_query(pConn, "drop stable `123_$^)`");
-// if (taos_errno(pRes) != 0) {
-// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes));
-// }
+ // pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)");
+ // if (taos_errno(pRes) != 0) {
+ // printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
+ // }
+ //
+ // pRes = taos_query(pConn, "use abc1");
+ // taos_free_result(pRes);
+ // pRes = taos_query(pConn, "drop stable `123_$^)`");
+ // if (taos_errno(pRes) != 0) {
+ // printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes));
+ // }
taos_close(pConn);
}
@@ -666,8 +665,7 @@ TEST(clientCase, create_multiple_tables) {
for (int32_t i = 0; i < 500; i += 2) {
char sql[512] = {0};
- snprintf(sql, tListLen(sql),
- "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5)", i, i + 1);
+ snprintf(sql, tListLen(sql), "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5)", i, i + 1);
TAOS_RES* pres = taos_query(pConn, sql);
if (taos_errno(pres) != 0) {
printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres));
@@ -701,7 +699,7 @@ TEST(clientCase, show_table_Test) {
int32_t numOfFields = taos_num_fields(pRes);
int32_t count = 0;
- char str[512] = {0};
+ char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
@@ -712,30 +710,30 @@ TEST(clientCase, show_table_Test) {
taos_close(pConn);
}
-//TEST(clientCase, drop_stable_Test) {
-// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
-// assert(pConn != nullptr);
+// TEST(clientCase, drop_stable_Test) {
+// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+// assert(pConn != nullptr);
//
-// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1");
-// if (taos_errno(pRes) != 0) {
-// printf("error in creating db, reason:%s\n", taos_errstr(pRes));
-// }
-// taos_free_result(pRes);
+// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1");
+// if (taos_errno(pRes) != 0) {
+// printf("error in creating db, reason:%s\n", taos_errstr(pRes));
+// }
+// taos_free_result(pRes);
//
-// pRes = taos_query(pConn, "use abc1");
-// if (taos_errno(pRes) != 0) {
-// printf("error in using db, reason:%s\n", taos_errstr(pRes));
-// }
-// taos_free_result(pRes);
+// pRes = taos_query(pConn, "use abc1");
+// if (taos_errno(pRes) != 0) {
+// printf("error in using db, reason:%s\n", taos_errstr(pRes));
+// }
+// taos_free_result(pRes);
//
-// pRes = taos_query(pConn, "drop stable st1");
-// if (taos_errno(pRes) != 0) {
-// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes));
-// }
+// pRes = taos_query(pConn, "drop stable st1");
+// if (taos_errno(pRes) != 0) {
+// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes));
+// }
//
-// taos_free_result(pRes);
-// taos_close(pConn);
-//}
+// taos_free_result(pRes);
+// taos_close(pConn);
+// }
TEST(clientCase, generated_request_id_test) {
SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
@@ -744,7 +742,7 @@ TEST(clientCase, generated_request_id_test) {
uint64_t v = generateRequestId();
void* result = taosHashGet(phash, &v, sizeof(v));
if (result != nullptr) {
-// printf("0x%llx, index:%d\n", v, i);
+ // printf("0x%llx, index:%d\n", v, i);
}
assert(result == nullptr);
taosHashPut(phash, &v, sizeof(v), NULL, 0);
@@ -832,7 +830,7 @@ TEST(clientCase, projection_query_tables) {
TEST(clientCase, tsbs_perf_test) {
TdThread qid[20] = {0};
- for(int32_t i = 0; i < numOfThreads; ++i) {
+ for (int32_t i = 0; i < numOfThreads; ++i) {
taosThreadCreate(&qid[i], NULL, queryThread, NULL);
}
getchar();
@@ -858,8 +856,8 @@ TEST(clientCase, projection_query_stables) {
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
-// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-// printf("%s\n", str);
+ // int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+ // printf("%s\n", str);
}
taos_free_result(pRes);
@@ -898,7 +896,8 @@ use test;
create table m1(ts timestamp, k int) tags(a int);
create table tm0 using m1 tags(1);
create table tm1 using m1 tags(2);
-insert into tm0 values('2021-1-1 1:1:1.120', 1) ('2021-1-1 1:1:2.9', 2) tm1 values('2021-1-1 1:1:1.120', 11) ('2021-1-1 1:1:2.99', 22);
+insert into tm0 values('2021-1-1 1:1:1.120', 1) ('2021-1-1 1:1:2.9', 2) tm1 values('2021-1-1 1:1:1.120', 11) ('2021-1-1
+1:1:2.99', 22);
*/
TEST(clientCase, async_api_test) {
@@ -912,7 +911,7 @@ TEST(clientCase, async_api_test) {
printf("failed, reason:%s\n", taos_errstr(pRes));
}
- int32_t n = 0;
+ int32_t n = 0;
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
@@ -920,8 +919,8 @@ TEST(clientCase, async_api_test) {
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t* length = taos_fetch_lengths(pRes);
- for(int32_t i = 0; i < numOfFields; ++i) {
- printf("(%d):%d " , i, length[i]);
+ for (int32_t i = 0; i < numOfFields; ++i) {
+ printf("(%d):%d ", i, length[i]);
}
printf("\n");
@@ -963,8 +962,8 @@ TEST(clientCase, update_test) {
taos_free_result(pRes);
- char s[256] = {0};
- for(int32_t i = 0; i < 17000; ++i) {
+ char s[256] = {0};
+ for (int32_t i = 0; i < 17000; ++i) {
sprintf(s, "insert into tup values(now+%da, %d)", i, i);
pRes = taos_query(pConn, s);
taos_free_result(pRes);
@@ -998,7 +997,8 @@ TEST(clientCase, sub_db_test) {
// 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new();
- tmq_list_append(topicList, "topic_db1");
+ tmq_list_append(topicList, "topic_t1");
+ tmq_list_append(topicList, "topic_s2");
// 启动订阅
tmq_subscribe(tmq, topicList);
@@ -1027,7 +1027,7 @@ TEST(clientCase, sub_db_test) {
printf("db: %s\n", dbName);
printf("vgroup id: %d\n", vgroupId);
- if (count ++ > 200) {
+ if (count++ > 200) {
tmq_unsubscribe(tmq);
break;
}
@@ -1091,7 +1091,7 @@ TEST(clientCase, sub_tb_test) {
while (1) {
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
if (pRes) {
- char buf[1024];
+ char buf[1024];
const char* topicName = tmq_get_topic_name(pRes);
const char* dbName = tmq_get_db_name(pRes);
@@ -1116,9 +1116,9 @@ TEST(clientCase, sub_tb_test) {
}
taos_free_result(pRes);
-// if ((++count) > 1) {
-// break;
-// }
+ // if ((++count) > 1) {
+ // break;
+ // }
} else {
break;
}
@@ -1133,11 +1133,11 @@ TEST(clientCase, sub_tb_mt_test) {
taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
TdThread qid[20] = {0};
- for(int32_t i = 0; i < 1; ++i) {
+ for (int32_t i = 0; i < 1; ++i) {
taosThreadCreate(&qid[i], NULL, doConsumeData, NULL);
}
- for(int32_t i = 0; i < 4; ++i) {
+ for (int32_t i = 0; i < 4; ++i) {
taosThreadJoin(qid[i], NULL);
}
}
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 03102b11a3..e021e65549 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -147,9 +147,17 @@ int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) {
return TSDB_CODE_SUCCESS;
}
-static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData,
- int32_t itemLen, int32_t numOfRows) {
- ASSERT(pColumnInfoData->info.bytes >= itemLen);
+static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData,
+ int32_t itemLen, int32_t numOfRows, bool trimValue) {
+ if (pColumnInfoData->info.bytes < itemLen) {
+ uWarn("column/tag actual data len %d is bigger than schema len %d, trim it:%d", itemLen, pColumnInfoData->info.bytes, trimValue);
+ if (trimValue) {
+ itemLen = pColumnInfoData->info.bytes;
+ } else {
+ return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
+ }
+ }
+
size_t start = 1;
// the first item
@@ -178,10 +186,12 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren
pColumnInfoData->varmeta.length += numOfRows * itemLen;
}
+
+ return TSDB_CODE_SUCCESS;
}
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
- uint32_t numOfRows) {
+ uint32_t numOfRows, bool trimValue) {
int32_t len = pColumnInfoData->info.bytes;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
len = varDataTLen(pData);
@@ -193,8 +203,7 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
}
}
- doCopyNItems(pColumnInfoData, currentRow, pData, len, numOfRows);
- return TSDB_CODE_SUCCESS;
+ return doCopyNItems(pColumnInfoData, currentRow, pData, len, numOfRows, trimValue);
}
static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource,
@@ -973,7 +982,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
taosSort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
int64_t p1 = taosGetTimestampUs();
- uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows);
+ uDebug("blockDataSort easy cost:%" PRId64 ", rows:%" PRId64 "\n", p1 - p0, pDataBlock->info.rows);
return TSDB_CODE_SUCCESS;
} else { // var data type
@@ -1026,6 +1035,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
return TSDB_CODE_SUCCESS;
}
+#if 0
typedef struct SHelper {
int32_t index;
union {
@@ -1074,59 +1084,20 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock*
int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) {
const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param;
- // SSDataBlock* pDataBlock = pHelper->pDataBlock;
-
SHelper* left = (SHelper*)p1;
SHelper* right = (SHelper*)p2;
SArray* pInfo = pHelper->orderInfo;
int32_t offset = 0;
- // for(int32_t i = 0; i < pInfo->size; ++i) {
- // SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0);
- // SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex);
-
- // if (pColInfoData->hasNull) {
- // bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg);
- // bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg);
- // if (leftNull && rightNull) {
- // continue; // continue to next slot
- // }
- //
- // if (rightNull) {
- // return pHelper->nullFirst? 1:-1;
- // }
- //
- // if (leftNull) {
- // return pHelper->nullFirst? -1:1;
- // }
- // }
-
- // void* left1 = colDataGetData(pColInfoData, left);
- // void* right1 = colDataGetData(pColInfoData, right);
-
- // switch(pColInfoData->info.type) {
- // case TSDB_DATA_TYPE_INT: {
int32_t leftx = *(int32_t*)left->pData; //*(int32_t*)(left->pData + offset);
int32_t rightx = *(int32_t*)right->pData; //*(int32_t*)(right->pData + offset);
- // offset += pColInfoData->info.bytes;
if (leftx == rightx) {
- // break;
return 0;
} else {
- // if (pOrder->order == TSDB_ORDER_ASC) {
return (leftx < rightx) ? -1 : 1;
- // } else {
- // return (leftx < rightx)? 1:-1;
- // }
}
- // }
- // default:
- // assert(0);
- // }
- // }
-
return 0;
}
@@ -1170,6 +1141,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
// destroyTupleIndex(index);
return 0;
}
+#endif
void blockDataCleanup(SSDataBlock* pDataBlock) {
blockDataEmpty(pDataBlock);
@@ -1180,7 +1152,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
void blockDataEmpty(SSDataBlock* pDataBlock) {
SDataBlockInfo* pInfo = &pDataBlock->info;
- if (pInfo->capacity == 0 || pInfo->rows > pDataBlock->info.capacity) {
+ if (pInfo->capacity == 0) {
return;
}
@@ -1739,14 +1711,14 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
int64_t tbUid = pBlock->info.id.uid;
int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int16_t hasVarCol = pBlock->info.hasVarCol;
- int32_t rows = pBlock->info.rows;
+ int64_t rows = pBlock->info.rows;
int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, tbUid);
tlen += taosEncodeFixedI16(buf, numOfCols);
tlen += taosEncodeFixedI16(buf, hasVarCol);
- tlen += taosEncodeFixedI32(buf, rows);
+ tlen += taosEncodeFixedI64(buf, rows);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
@@ -1777,7 +1749,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
buf = taosDecodeFixedU64(buf, &pBlock->info.id.uid);
buf = taosDecodeFixedI16(buf, &numOfCols);
buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol);
- buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
+ buf = taosDecodeFixedI64(buf, &pBlock->info.rows);
buf = taosDecodeFixedI32(buf, &sz);
pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData));
for (int32_t i = 0; i < sz; i++) {
@@ -1880,6 +1852,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
return buf;
}
+#if 0
void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag) {
SArray* dataBlocks = taosArrayInit(1, sizeof(SSDataBlock*));
taosArrayPush(dataBlocks, &pBlock);
@@ -1972,6 +1945,8 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
}
}
+#endif
+
// for debug
char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) {
int32_t size = 2048;
@@ -1983,7 +1958,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
int32_t len = 0;
len += snprintf(dumpBuf + len, size - len,
"===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64
- "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n",
+ "|rows:%" PRId64 "|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n",
flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId,
pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version,
pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName);
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 2bb8708372..d6ab974c6c 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -2439,6 +2439,12 @@ _exit:
int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap,
char *data) {
int32_t code = 0;
+ if(data == NULL){
+ for (int32_t i = 0; i < nRows; ++i) {
+ code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
+ }
+ goto _exit;
+ }
if (IS_VAR_DATA_TYPE(type)) { // var-length data type
for (int32_t i = 0; i < nRows; ++i) {
@@ -2447,6 +2453,11 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
if (code) goto _exit;
} else {
+ if(ASSERT(varDataTLen(data + offset) <= bytes)){
+ uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes);
+ code = TSDB_CODE_INVALID_PARA;
+ goto _exit;
+ }
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset),
varDataLen(data + offset));
}
diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c
index 892b4a9a40..f0e020edfe 100644
--- a/source/dnode/mgmt/exe/dmMain.c
+++ b/source/dnode/mgmt/exe/dmMain.c
@@ -18,6 +18,7 @@
#include "mnode.h"
#include "tconfig.h"
#include "tglobal.h"
+#include "version.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@@ -91,28 +92,28 @@ void dmLogCrash(int signum, void *sigInfo, void *context) {
taosIgnSignal(SIGINT);
taosIgnSignal(SIGBREAK);
-#ifndef WINDOWS
+#ifndef WINDOWS
taosIgnSignal(SIGBUS);
#endif
taosIgnSignal(SIGABRT);
taosIgnSignal(SIGFPE);
taosIgnSignal(SIGSEGV);
- char *pMsg = NULL;
+ char *pMsg = NULL;
const char *flags = "UTL FATAL ";
ELogLevel level = DEBUG_FATAL;
int32_t dflag = 255;
- int64_t msgLen= -1;
-
+ int64_t msgLen = -1;
+
if (tsEnableCrashReport) {
if (taosGenCrashJsonMsg(signum, &pMsg, dmGetClusterId(), global.startTime)) {
taosPrintLog(flags, level, dflag, "failed to generate crash json msg");
goto _return;
} else {
- msgLen = strlen(pMsg);
+ msgLen = strlen(pMsg);
}
}
-
+
_return:
taosLogCrashInfo("taosd", pMsg, msgLen, signum, sigInfo);
@@ -138,7 +139,7 @@ static void dmSetSignalHandle() {
#ifndef WINDOWS
taosSetSignal(SIGBUS, dmLogCrash);
-#endif
+#endif
taosSetSignal(SIGABRT, dmLogCrash);
taosSetSignal(SIGFPE, dmLogCrash);
taosSetSignal(SIGSEGV, dmLogCrash);
@@ -149,7 +150,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
int32_t cmdEnvIndex = 0;
if (argc < 2) return 0;
-
+
global.envCmd = taosMemoryMalloc((argc - 1) * sizeof(char *));
memset(global.envCmd, 0, (argc - 1) * sizeof(char *));
for (int32_t i = 1; i < argc; ++i) {
@@ -218,6 +219,9 @@ static void dmPrintVersion() {
#endif
printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
printf("gitinfo: %s\n", gitinfo);
+#ifdef TD_ENTERPRISE
+ printf("gitinfoOfInternal: %s\n", gitinfoOfInternal);
+#endif
printf("buildInfo: %s\n", buildinfo);
}
@@ -299,7 +303,7 @@ int mainWindows(int argc, char **argv) {
printf("failed to init memory dbg, error:%s\n", tstrerror(code));
return code;
}
- tsAsyncLog = false;
+ tsAsyncLog = false;
printf("memory dbg enabled\n");
}
#endif
diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
index b47742b4ed..24f75e3c8b 100644
--- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
+++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
@@ -34,7 +34,7 @@ typedef struct SMnodeMgmt {
SSingleWorker readWorker;
SSingleWorker writeWorker;
SSingleWorker syncWorker;
- SSingleWorker syncCtrlWorker;
+ SSingleWorker syncRdWorker;
bool stopped;
int32_t refCount;
TdThreadRwlock lock;
@@ -54,7 +54,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt);
void mmStopWorker(SMnodeMgmt *pMgmt);
int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
-int32_t mmPutMsgToSyncCtrlQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
+int32_t mmPutMsgToSyncRdQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg);
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
index b96a0ffd42..76977dd4a8 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
@@ -189,7 +189,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DISABLE_WRITE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
@@ -199,13 +199,16 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
+
if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
+
code = 0;
_OVER:
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
index b0810d528f..0152e5d0b1 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
@@ -111,8 +111,8 @@ int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg);
}
-int32_t mmPutMsgToSyncCtrlQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
- return mmPutMsgToWorker(pMgmt, &pMgmt->syncCtrlWorker, pMsg);
+int32_t mmPutMsgToSyncRdQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
+ return mmPutMsgToWorker(pMgmt, &pMgmt->syncRdWorker, pMsg);
}
int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
@@ -151,8 +151,8 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
case SYNC_QUEUE:
pWorker = &pMgmt->syncWorker;
break;
- case SYNC_CTRL_QUEUE:
- pWorker = &pMgmt->syncCtrlWorker;
+ case SYNC_RD_QUEUE:
+ pWorker = &pMgmt->syncRdWorker;
break;
default:
terrno = TSDB_CODE_INVALID_PARA;
@@ -238,12 +238,12 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
SSingleWorkerCfg scCfg = {
.min = 1,
.max = 1,
- .name = "mnode-sync-ctrl",
+ .name = "mnode-sync-rd",
.fp = (FItem)mmProcessSyncMsg,
.param = pMgmt,
};
- if (tSingleWorkerInit(&pMgmt->syncCtrlWorker, &scCfg) != 0) {
- dError("failed to start mnode mnode-sync-ctrl worker since %s", terrstr());
+ if (tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg) != 0) {
+ dError("failed to start mnode mnode-sync-rd worker since %s", terrstr());
return -1;
}
@@ -259,6 +259,6 @@ void mmStopWorker(SMnodeMgmt *pMgmt) {
tSingleWorkerCleanup(&pMgmt->readWorker);
tSingleWorkerCleanup(&pMgmt->writeWorker);
tSingleWorkerCleanup(&pMgmt->syncWorker);
- tSingleWorkerCleanup(&pMgmt->syncCtrlWorker);
+ tSingleWorkerCleanup(&pMgmt->syncRdWorker);
dDebug("mnode workers are closed");
}
diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
index e3fa2964b7..4374ae363c 100644
--- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
+++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
@@ -59,7 +59,7 @@ typedef struct {
SVnode *pImpl;
SMultiWorker pWriteW;
SMultiWorker pSyncW;
- SMultiWorker pSyncCtrlW;
+ SMultiWorker pSyncRdW;
SMultiWorker pApplyW;
STaosQueue *pQueryQ;
STaosQueue *pStreamQ;
@@ -107,7 +107,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc);
int32_t vmPutMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
-int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
+int32_t vmPutMsgToSyncRdQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 11ccff3198..d61eb3ec03 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -21,7 +21,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) {
if (pInfo->pVloads == NULL) return;
tfsUpdateSize(pMgmt->pTfs);
-
+
taosThreadRwlockRdlock(&pMgmt->lock);
void *pIter = taosHashIterate(pMgmt->hash, NULL);
@@ -549,7 +549,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
@@ -559,14 +559,15 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
-
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
+
code = 0;
_OVER:
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
index 0524e2713a..4c5b1246e7 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
@@ -98,9 +98,9 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
pVnode->pSyncW.queue->threadId);
tMultiWorkerCleanup(&pVnode->pSyncW);
- dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId,
- pVnode->pSyncCtrlW.queue, pVnode->pSyncCtrlW.queue->threadId);
- tMultiWorkerCleanup(&pVnode->pSyncCtrlW);
+ dInfo("vgId:%d, wait for vnode sync rd queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
+ pVnode->pSyncRdW.queue->threadId);
+ tMultiWorkerCleanup(&pVnode->pSyncRdW);
dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
pVnode->pApplyW.queue->threadId);
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
index 7aa1c9f56a..e4e0d608de 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
@@ -216,9 +216,9 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pSyncW.queue, pMsg);
break;
- case SYNC_CTRL_QUEUE:
- dGTrace("vgId:%d, msg:%p put into vnode-sync-ctrl queue", pVnode->vgId, pMsg);
- taosWriteQitem(pVnode->pSyncCtrlW.queue, pMsg);
+ case SYNC_RD_QUEUE:
+ dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg);
+ taosWriteQitem(pVnode->pSyncRdW.queue, pMsg);
break;
case APPLY_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg);
@@ -234,9 +234,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
return code;
}
-int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
- return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE);
-}
+int32_t vmPutMsgToSyncRdQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_RD_QUEUE); }
int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); }
@@ -327,18 +325,18 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl};
SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
- SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-ctrl", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
+ SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode->pImpl};
(void)tMultiWorkerInit(&pVnode->pWriteW, &wcfg);
(void)tMultiWorkerInit(&pVnode->pSyncW, &scfg);
- (void)tMultiWorkerInit(&pVnode->pSyncCtrlW, &sccfg);
+ (void)tMultiWorkerInit(&pVnode->pSyncRdW, &sccfg);
(void)tMultiWorkerInit(&pVnode->pApplyW, &acfg);
pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
pVnode->pStreamQ = tAutoQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue);
pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue);
- if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncCtrlW.queue == NULL ||
+ if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL ||
pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@@ -348,8 +346,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
pVnode->pWriteW.queue->threadId);
dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
pVnode->pSyncW.queue->threadId);
- dInfo("vgId:%d, sync-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlW.queue,
- pVnode->pSyncCtrlW.queue->threadId);
+ dInfo("vgId:%d, sync-rd-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
+ pVnode->pSyncRdW.queue->threadId);
dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
pVnode->pApplyW.queue->threadId);
dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h
index 801afd562b..876f2651bc 100644
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@@ -393,7 +393,7 @@ typedef struct {
} SSmaObj;
typedef struct {
- char name[TSDB_TABLE_FNAME_LEN];
+ char name[TSDB_INDEX_FNAME_LEN];
char stb[TSDB_TABLE_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN];
char dstTbName[TSDB_TABLE_FNAME_LEN];
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 2f560910d4..8f1cfbff93 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -81,9 +81,7 @@ bool mndRebTryStart() {
return old == 0;
}
-void mndRebEnd() {
- mndRebCntDec();
-}
+void mndRebEnd() { mndRebCntDec(); }
void mndRebCntInc() {
int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1);
@@ -247,6 +245,13 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
return pRebInfo;
}
+static void freeRebalanceItem(void *param) {
+ SMqRebInfo *pInfo = param;
+ taosArrayDestroy(pInfo->lostConsumers);
+ taosArrayDestroy(pInfo->newConsumers);
+ taosArrayDestroy(pInfo->removedConsumers);
+}
+
static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node;
SSdb *pSdb = pMnode->pSdb;
@@ -262,8 +267,21 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
}
SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg));
+ if (pRebMsg == NULL) {
+ mError("failed to create the rebalance msg, size:%d, quit mq timer", (int32_t)sizeof(SMqDoRebalanceMsg));
+ mndRebEnd();
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
- // TODO set cleanfp
+ if (pRebMsg->rebSubHash == NULL) {
+ mError("failed to create rebalance hashmap");
+ rpcFreeCont(pRebMsg);
+ mndRebEnd();
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ taosHashSetFreeFp(pRebMsg->rebSubHash, freeRebalanceItem);
// iterate all consumers, find all modification
while (1) {
@@ -356,7 +374,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
} else {
taosHashCleanup(pRebMsg->rebSubHash);
rpcFreeCont(pRebMsg);
- mDebug("mq rebalance finished, no modification");
+ mDebug("mq timer finished, no need to re-balance");
mndRebEnd();
}
return 0;
@@ -576,7 +594,7 @@ int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj
return 0;
}
-static int32_t validateTopics(const SArray* pTopicList, SMnode* pMnode, const char* pUser) {
+static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser) {
int32_t numOfTopics = taosArrayGetSize(pTopicList);
for (int32_t i = 0; i < numOfTopics; i++) {
@@ -597,8 +615,13 @@ static int32_t validateTopics(const SArray* pTopicList, SMnode* pMnode, const ch
return 0;
}
-static void* topicNameDup(void* p){
- return taosStrdup((char*) p);
+static void *topicNameDup(void *p) { return taosStrdup((char *)p); }
+
+static void freeItem(void *param) {
+ void *pItem = *(void **)param;
+ if (pItem != NULL) {
+ taosMemoryFree(pItem);
+ }
}
int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
@@ -616,7 +639,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
int32_t code = -1;
SArray *pTopicList = subscribe.topicNames;
taosArraySort(pTopicList, taosArrayCompareString);
- taosArrayRemoveDuplicateP(pTopicList, taosArrayCompareString, taosMemoryFree);
+ taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem);
int32_t newTopicNum = taosArrayGetSize(pTopicList);
@@ -633,8 +656,8 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
pExistedConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pExistedConsumer == NULL) {
- mInfo("receive subscribe request from new consumer:0x%" PRIx64" cgroup:%s, numOfTopics:%d", consumerId,
- subscribe.cgroup, (int32_t) taosArrayGetSize(pTopicList));
+ mInfo("receive subscribe request from new consumer:0x%" PRIx64 " cgroup:%s, numOfTopics:%d", consumerId,
+ subscribe.cgroup, (int32_t)taosArrayGetSize(pTopicList));
pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
tstrncpy(pConsumerNew->clientId, subscribe.clientId, tListLen(pConsumerNew->clientId));
@@ -656,8 +679,9 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
/*taosRLockLatch(&pExistedConsumer->lock);*/
int32_t status = atomic_load_32(&pExistedConsumer->status);
- mInfo("receive subscribe request from existed consumer:0x%" PRIx64 " cgroup:%s, current status:%d(%s), subscribe topic num: %d",
- consumerId, subscribe.cgroup, status,mndConsumerStatusName(status), newTopicNum);
+ mInfo("receive subscribe request from existed consumer:0x%" PRIx64
+ " cgroup:%s, current status:%d(%s), subscribe topic num: %d",
+ consumerId, subscribe.cgroup, status, mndConsumerStatusName(status), newTopicNum);
if (status != MQ_CONSUMER_STATUS__READY) {
terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
@@ -674,7 +698,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
taosArrayDestroy(pConsumerNew->assignedTopics);
pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup);
- int32_t oldTopicNum = (pExistedConsumer->currentTopics)? taosArrayGetSize(pExistedConsumer->currentTopics):0;
+ int32_t oldTopicNum = (pExistedConsumer->currentTopics) ? taosArrayGetSize(pExistedConsumer->currentTopics) : 0;
int32_t i = 0, j = 0;
while (i < oldTopicNum || j < newTopicNum) {
@@ -839,12 +863,59 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
}
static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
- mDebug("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId,
- pConsumer->status, mndConsumerStatusName(pConsumer->status));
+ mDebug("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId, pConsumer->status,
+ mndConsumerStatusName(pConsumer->status));
tDeleteSMqConsumerObj(pConsumer);
return 0;
}
+static void updateConsumerStatus(SMqConsumerObj *pConsumer) {
+ int32_t status = pConsumer->status;
+
+ if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) {
+ if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+ pConsumer->status = MQ_CONSUMER_STATUS__READY;
+ } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) {
+ pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
+ }
+ } else {
+ if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+ pConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
+ } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
+ pConsumer->status = MQ_CONSUMER_STATUS__LOST;
+ }
+ }
+}
+
+// remove from new topic
+static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) {
+ int32_t size = taosArrayGetSize(pConsumer->rebNewTopics);
+ for (int32_t i = 0; i < taosArrayGetSize(pConsumer->rebNewTopics); i++) {
+ char *p = taosArrayGetP(pConsumer->rebNewTopics, i);
+ if (strcmp(pTopic, p) == 0) {
+ taosArrayRemove(pConsumer->rebNewTopics, i);
+ taosMemoryFree(p);
+
+ mDebug("consumer:0x%" PRIx64 " remove new topic:%s in the topic list, remain newTopics:%d", pConsumer->consumerId,
+ pTopic, (int)taosArrayGetSize(pConsumer->rebNewTopics));
+ break;
+ }
+ }
+}
+
+// remove from removed topic
+static void removeFromRemoveTopicList(SMqConsumerObj *pConsumer, const char *pTopic) {
+ int32_t size = taosArrayGetSize(pConsumer->rebRemovedTopics);
+ for (int32_t i = 0; i < size; i++) {
+ char *p = taosArrayGetP(pConsumer->rebRemovedTopics, i);
+ if (strcmp(pTopic, p) == 0) {
+ taosArrayRemove(pConsumer->rebRemovedTopics, i);
+ taosMemoryFree(p);
+ break;
+ }
+ }
+}
+
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", uptime:%" PRId64,
pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime);
@@ -855,6 +926,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
/*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/
/*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/
+ // this new consumer has identical topics with one existed consumers.
if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) {
pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
} else {
@@ -871,7 +943,6 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pNewConsumer->assignedTopics = tmp;
pOldConsumer->subscribeTime = pNewConsumer->upTime;
-
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
}
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
@@ -911,71 +982,48 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
- /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);*/
- /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);*/
+ ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);
+ char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
- char *addedTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
// not exist in current topic
-
- bool existing = false;
-#if 1
+ bool existing = false;
int32_t numOfExistedTopics = taosArrayGetSize(pOldConsumer->currentTopics);
for (int32_t i = 0; i < numOfExistedTopics; i++) {
char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
- if (strcmp(topic, addedTopic) == 0) {
+ if (strcmp(topic, pNewTopic) == 0) {
existing = true;
}
}
-#endif
- // remove from new topic
- for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
- char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
- if (strcmp(addedTopic, topic) == 0) {
- taosArrayRemove(pOldConsumer->rebNewTopics, i);
- taosMemoryFree(topic);
- break;
- }
- }
+ removeFromNewTopicList(pOldConsumer, pNewTopic);
// add to current topic
if (!existing) {
- taosArrayPush(pOldConsumer->currentTopics, &addedTopic);
+ taosArrayPush(pOldConsumer->currentTopics, &pNewTopic);
taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString);
} else {
- taosMemoryFree(addedTopic);
+ taosMemoryFree(pNewTopic);
}
// set status
int32_t status = pOldConsumer->status;
- if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
- if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
- pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
- } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) {
- pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
- }
- } else {
- if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
- pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
- } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
- pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
- }
- }
+ updateConsumerStatus(pOldConsumer);
// the re-balance is triggered when the new consumer is launched.
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
- mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
+ mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
+ ", current topics:%d, newTopics:%d, removeTopics:%d",
pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
- mndConsumerStatusName(pOldConsumer->status),
- pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics));
+ mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime,
+ (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics),
+ (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
+
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
/*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/
/*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/
char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
-
- // not exist in new topic
#if 0
for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
@@ -984,14 +1032,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
#endif
// remove from removed topic
- for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) {
- char *topic = taosArrayGetP(pOldConsumer->rebRemovedTopics, i);
- if (strcmp(removedTopic, topic) == 0) {
- taosArrayRemove(pOldConsumer->rebRemovedTopics, i);
- taosMemoryFree(topic);
- break;
- }
- }
+ removeFromRemoveTopicList(pOldConsumer, removedTopic);
// remove from current topic
int32_t i = 0;
@@ -1004,32 +1045,20 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
break;
}
}
- // must find the topic
- /*A(i < sz);*/
// set status
int32_t status = pOldConsumer->status;
- if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
- if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
- pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
- } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) {
- pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
- }
- } else {
- if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
- pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
- } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
- pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
- }
- }
+ updateConsumerStatus(pOldConsumer);
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
- mDebug("consumer:0x%" PRIx64 " state %d(%s) -> %d(%s), new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
+ mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
+ ", current topics:%d, newTopics:%d, removeTopics:%d",
pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
- mndConsumerStatusName(pOldConsumer->status),
- pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics));
+ mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime,
+ (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics),
+ (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
}
taosWUnLockLatch(&pOldConsumer->lock);
@@ -1105,8 +1134,8 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false);
// status
- char status[20 + VARSTR_HEADER_SIZE] = {0};
- const char* pStatusName = mndConsumerStatusName(pConsumer->status);
+ char status[20 + VARSTR_HEADER_SIZE] = {0};
+ const char *pStatusName = mndConsumerStatusName(pConsumer->status);
STR_TO_VARSTR(status, pStatusName);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c
index 8782fd823f..83172acf64 100644
--- a/source/dnode/mnode/impl/src/mndIndex.c
+++ b/source/dnode/mnode/impl/src/mndIndex.c
@@ -138,7 +138,7 @@ static void *mndBuildDropIdxReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStbOb
mInfo("idx: %s start to build drop index req", pIdx->name);
len = tSerializeSDropIdxReq(NULL, 0, &req);
- if (ret < 0) {
+ if (len < 0) {
goto _err;
}
@@ -672,7 +672,7 @@ _OVER:
static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) {
int32_t code = -1;
SIdxObj idxObj = {0};
- memcpy(idxObj.name, req->idxName, TSDB_TABLE_FNAME_LEN);
+ memcpy(idxObj.name, req->idxName, TSDB_INDEX_FNAME_LEN);
memcpy(idxObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN);
memcpy(idxObj.db, pDb->name, TSDB_DB_FNAME_LEN);
memcpy(idxObj.colName, req->colName, TSDB_COL_NAME_LEN);
diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c
index 41dea50731..50e502f4ab 100644
--- a/source/dnode/mnode/impl/src/mndProfile.c
+++ b/source/dnode/mnode/impl/src/mndProfile.c
@@ -226,7 +226,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
goto _OVER;
}
- if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 2)) != 0) {
+ if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 3)) != 0) {
terrno = code;
goto _OVER;
}
diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c
index 924216bcbf..6a745a0a4f 100644
--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@@ -197,24 +197,20 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
return pRebSub;
}
-static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
- int32_t totalVgNum = pOutput->pSub->vgNum;
- const char *sub = pOutput->pSub->key;
- mInfo("sub:%s mq re-balance %d vgroups", sub, pOutput->pSub->vgNum);
+static void doRemoveExistedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) {
+ int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
+ const char *pSubKey = pOutput->pSub->key;
- // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
- SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
-
- // 2. check and get actual removed consumers, put their vg into hash
- int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
int32_t actualRemoved = 0;
- for (int32_t i = 0; i < removedNum; i++) {
+ for (int32_t i = 0; i < numOfRemoved; i++) {
uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
+ // consumer exists till now
if (pConsumerEp) {
actualRemoved++;
+
int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
for (int32_t j = 0; j < consumerVgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
@@ -223,52 +219,67 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.newConsumerId = -1,
.pVgEp = pVgEp,
};
+
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
- mInfo("sub:%s mq re-balance remove vgId:%d from consumer:0x%" PRIx64, sub, pVgEp->vgId, consumerId);
+ mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, pSubKey, pVgEp->vgId, consumerId);
}
+
taosArrayDestroy(pConsumerEp->vgs);
taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
+
// put into removed
taosArrayPush(pOutput->removedConsumers, &consumerId);
}
}
- if (removedNum != actualRemoved) {
- mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved);
+ if (numOfRemoved != actualRemoved) {
+ mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", pSubKey, numOfRemoved, actualRemoved);
+ } else {
+ mInfo("sub:%s removed %d consumers", pSubKey, numOfRemoved);
}
+}
- // if previously no consumer, there are vgs not assigned
- {
- int32_t consumerVgNum = taosArrayGetSize(pOutput->pSub->unassignedVgs);
- for (int32_t i = 0; i < consumerVgNum; i++) {
- SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs);
- SMqRebOutputVg rebOutput = {
- .oldConsumerId = -1,
- .newConsumerId = -1,
- .pVgEp = pVgEp,
- };
- taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
- mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", sub, pVgEp->vgId);
- }
+static void doAddNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) {
+ int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers);
+ const char *pSubKey = pOutput->pSub->key;
+
+ for (int32_t i = 0; i < numOfNewConsumers; i++) {
+ int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
+
+ SMqConsumerEp newConsumerEp;
+ newConsumerEp.consumerId = consumerId;
+ newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
+
+ taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
+ taosArrayPush(pOutput->newConsumers, &consumerId);
+ mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, pSubKey, consumerId);
}
+}
- // 3. calc vg number of each consumer
- int32_t afterRebConsumerNum = pInput->oldConsumerNum + taosArrayGetSize(pInput->pRebInfo->newConsumers) -
- taosArrayGetSize(pInput->pRebInfo->removedConsumers);
- int32_t minVgCnt = 0;
- int32_t imbConsumerNum = 0;
- // calc num
- if (afterRebConsumerNum) {
- minVgCnt = totalVgNum / afterRebConsumerNum;
- imbConsumerNum = totalVgNum % afterRebConsumerNum;
+static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) {
+ const char *pSubKey = pOutput->pSub->key;
+ int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs);
+
+ for (int32_t i = 0; i < numOfVgroups; i++) {
+ SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs);
+ SMqRebOutputVg rebOutput = {
+ .oldConsumerId = -1,
+ .newConsumerId = -1,
+ .pVgEp = pVgEp,
+ };
+
+ taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
+ mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", pSubKey, pVgEp->vgId);
}
+}
- mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has more vgs", sub,
- afterRebConsumerNum, minVgCnt, imbConsumerNum);
+static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt,
+ int32_t imbConsumerNum) {
+ const char *pSubKey = pOutput->pSub->key;
- // 4. first scan: remove consumer more than wanted, put to remove hash
int32_t imbCnt = 0;
void *pIter = NULL;
+
while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
if (pIter == NULL) {
@@ -276,8 +287,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
}
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
+ int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
- int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
// all old consumers still existing are touched
// TODO optimize: touch only consumer whose vgs changed
taosArrayPush(pOutput->touchedConsumers, &pConsumerEp->consumerId);
@@ -296,13 +307,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp,
};
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
- mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId,
+ mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId,
pConsumerEp->consumerId);
}
imbCnt++;
}
} else {
- // pop until equal minVg
+ // all the remain consumers should only have the number of vgroups, which is equalled to the value of minVg
while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt) {
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
SMqRebOutputVg outputVg = {
@@ -311,36 +322,67 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp,
};
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
- mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId,
+ mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId,
pConsumerEp->consumerId);
}
}
}
}
+}
- // 5. add new consumer into sub
- {
- int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers);
- for (int32_t i = 0; i < consumerNum; i++) {
- int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
+static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
+ int32_t totalVgNum = pOutput->pSub->vgNum;
+ const char *pSubKey = pOutput->pSub->key;
- SMqConsumerEp newConsumerEp;
- newConsumerEp.consumerId = consumerId;
- newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
- taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
- taosArrayPush(pOutput->newConsumers, &consumerId);
- mInfo("sub:%s mq rebalance add new consumer:0x%" PRIx64, sub, consumerId);
- }
+ int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
+ int32_t numOfAdded = taosArrayGetSize(pInput->pRebInfo->newConsumers);
+ mInfo("sub:%s mq re-balance %d vgroups, existed consumers:%d, added:%d, removed:%d", pSubKey, totalVgNum,
+ pInput->oldConsumerNum, numOfAdded, numOfRemoved);
+
+ // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
+ SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+
+ // 2. check and get actual removed consumers, put their vg into hash
+ doRemoveExistedConsumers(pOutput, pHash, pInput);
+
+ // 3. if previously no consumer, there are vgs not assigned
+ addUnassignedVgroups(pOutput, pHash);
+
+ // 4. calc vg number of each consumer
+ int32_t numOfFinal = pInput->oldConsumerNum + numOfAdded - numOfRemoved;
+
+ int32_t minVgCnt = 0;
+ int32_t imbConsumerNum = 0;
+
+ // calc num
+ if (numOfFinal) {
+ minVgCnt = totalVgNum / numOfFinal;
+ imbConsumerNum = totalVgNum % numOfFinal;
+ mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has 1 more vgroups than avg value",
+ pSubKey, numOfFinal, minVgCnt, imbConsumerNum);
+ } else {
+ mInfo("sub:%s no consumer subscribe this topic", pSubKey);
}
- // 6. second scan: find consumer do not have enough vg, extract from temporary hash and assign to new consumer.
+ // 5. first scan: remove vgroups from te consumers, who have more vgroups than the threashold value that is
+ // minVgCnt, and then put them into the recycled hash list
+ transferVgroupsForConsumers(pOutput, pHash, minVgCnt, imbConsumerNum);
+
+ // 6. add new consumer into sub
+ doAddNewConsumers(pOutput, pInput);
+
+ // 7. second scan: find consumer do not have enough vgroups, extract from temporary hash and assign to them
// All related vg should be put into rebVgs
SMqRebOutputVg *pRebVg = NULL;
void *pRemovedIter = NULL;
- pIter = NULL;
+ void *pIter = NULL;
+
while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
- if (pIter == NULL) break;
+ if (pIter == NULL) {
+ break;
+ }
+
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
// push until equal minVg
@@ -348,8 +390,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
// iter hash and find one vg
pRemovedIter = taosHashIterate(pHash, pRemovedIter);
if (pRemovedIter == NULL) {
- mError("sub:%s removed iter is null", sub);
- continue;
+ mError("sub:%s removed iter is null", pSubKey);
+ break;
}
pRebVg = (SMqRebOutputVg *)pRemovedIter;
@@ -409,15 +451,15 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
taosArrayPush(pOutput->rebVgs, pRebOutput);
- mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId);
+ mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", pSubKey, pRebOutput->pVgEp->vgId);
}
}
// 8. generate logs
- mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", sub);
+ mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", pSubKey);
for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
- mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, sub,
+ mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, pSubKey,
pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId);
}
{
@@ -427,10 +469,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
if (pIter == NULL) break;
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
- mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", sub, pConsumerEp->consumerId, sz);
+ mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", pSubKey, pConsumerEp->consumerId, sz);
for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
- mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, sub, pVgEp->vgId,
+ mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, pSubKey, pVgEp->vgId,
pConsumerEp->consumerId);
}
}
@@ -555,17 +597,23 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node;
SMqDoRebalanceMsg *pReq = pMsg->pCont;
void *pIter = NULL;
+ bool rebalanceOnce = false; // to ensure only once.
- mInfo("mq re-balance start");
+ mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash));
+ // here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction.
while (1) {
+ if (rebalanceOnce) {
+ break;
+ }
+
pIter = taosHashIterate(pReq->rebSubHash, pIter);
if (pIter == NULL) {
break;
}
- SMqRebInputObj rebInput = {0};
-
+ // todo handle the malloc failure
+ SMqRebInputObj rebInput = {0};
SMqRebOutputObj rebOutput = {0};
rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t));
rebOutput.removedConsumers = taosArrayInit(0, sizeof(int64_t));
@@ -582,9 +630,10 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
char topic[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CGROUP_LEN];
mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true);
+
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
if (pTopic == NULL) {
- mError("mq re-balance %s ignored since topic %s not exist", pRebInfo->key, topic);
+ mError("mq re-balance %s ignored since topic %s doesn't exist", pRebInfo->key, topic);
continue;
}
@@ -604,11 +653,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
mndReleaseTopic(pMnode, pTopic);
rebInput.oldConsumerNum = 0;
+ mInfo("topic:%s has no consumers sub yet", topic);
} else {
taosRLockLatch(&pSub->lock);
rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash);
rebOutput.pSub = tCloneSubscribeObj(pSub);
taosRUnLockLatch(&pSub->lock);
+ mInfo("topic:%s has %d consumers sub till now", pRebInfo->key, rebInput.oldConsumerNum);
mndReleaseSubscribe(pMnode, pSub);
}
@@ -623,16 +674,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
mError("mq re-balance persist output error, possibly vnode splitted or dropped");
}
- taosArrayDestroy(pRebInfo->lostConsumers);
- taosArrayDestroy(pRebInfo->newConsumers);
- taosArrayDestroy(pRebInfo->removedConsumers);
-
taosArrayDestroy(rebOutput.newConsumers);
taosArrayDestroy(rebOutput.touchedConsumers);
taosArrayDestroy(rebOutput.removedConsumers);
taosArrayDestroy(rebOutput.rebVgs);
tDeleteSubscribeObj(rebOutput.pSub);
taosMemoryFree(rebOutput.pSub);
+
+ rebalanceOnce = true;
}
// reset flag
diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c
index 18548db56f..4965d5c34a 100644
--- a/source/dnode/mnode/impl/src/mndSync.c
+++ b/source/dnode/mnode/impl/src/mndSync.c
@@ -33,7 +33,7 @@ static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
return -1;
}
- int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg);
+ int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg);
if (code != 0) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 4aaed29fc3..106eea0313 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -1093,10 +1093,13 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) {
pAction->msgReceived = 1;
pAction->errCode = pRsp->code;
pTrans->lastErrorNo = pRsp->code;
+
+ mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId,
+ mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode);
+ } else {
+ mInfo("trans:%d, invalid action, index:%d, code:0x%x", transId, action, pRsp->code);
}
- mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, mndTransStr(pAction->stage),
- action, pRsp->code, pAction->acceptableCode, pAction->retryCode);
mndTransExecute(pMnode, pTrans, true);
_OVER:
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 8dc3f46ae3..9911752f8e 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -57,7 +57,7 @@ target_sources(
# tq
"src/tq/tq.c"
- "src/tq/tqExec.c"
+ "src/tq/tqScan.c"
"src/tq/tqMeta.c"
"src/tq/tqRead.c"
"src/tq/tqOffset.c"
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 2d053d04ae..9636dc2872 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -178,11 +178,11 @@ typedef struct STsdbReader STsdbReader;
int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
- SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr);
+ SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly);
void tsdbReaderSetId(STsdbReader* pReader, const char* idstr);
void tsdbReaderClose(STsdbReader *pReader);
-bool tsdbNextDataBlock(STsdbReader *pReader);
+int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
void tsdbReleaseDataBlock(STsdbReader *pReader);
SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index c2b38f5cd1..9037644602 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -109,23 +109,18 @@ typedef struct {
} STqPushEntry;
struct STQ {
- SVnode* pVnode;
- char* path;
- int64_t walLogLastVer;
-
- SRWLatch pushLock;
-
- SHashObj* pPushMgr; // consumerId -> STqPushEntry
- SHashObj* pHandle; // subKey -> STqHandle
- SHashObj* pCheckInfo; // topic -> SAlterCheckInfo
-
+ SVnode* pVnode;
+ char* path;
+ int64_t walLogLastVer;
+ SRWLatch lock;
+ SHashObj* pPushMgr; // consumerId -> STqPushEntry
+ SHashObj* pHandle; // subKey -> STqHandle
+ SHashObj* pCheckInfo; // topic -> SAlterCheckInfo
STqOffsetStore* pOffsetStore;
-
- TDB* pMetaDB;
- TTB* pExecStore;
- TTB* pCheckStore;
-
- SStreamMeta* pStreamMeta;
+ TDB* pMetaDB;
+ TTB* pExecStore;
+ TTB* pCheckStore;
+ SStreamMeta* pStreamMeta;
};
typedef struct {
@@ -144,8 +139,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum, uint64_t reqId);
// tqExec
-int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp);
-// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp);
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows);
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type);
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry);
@@ -164,7 +158,7 @@ typedef struct {
int32_t size;
} STqOffsetHead;
-STqOffsetStore* tqOffsetOpen();
+STqOffsetStore* tqOffsetOpen(STQ* pTq);
void tqOffsetClose(STqOffsetStore*);
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey);
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset);
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 0b38ce6d24..0c4ada2cb1 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -224,6 +224,8 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward,
void *tsdbTbDataIterDestroy(STbDataIter *pIter);
void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
bool tsdbTbDataIterNext(STbDataIter *pIter);
+void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum);
+
// STbData
int32_t tsdbGetNRowsInTbData(STbData *pTbData);
// tsdbFile.c ==============================================================================================
@@ -687,6 +689,8 @@ typedef struct SSttBlockLoadInfo {
STSchema *pSchema;
int16_t *colIds;
int32_t numOfCols;
+ bool checkRemainingRow;
+ bool isLast;
bool sttBlockLoaded;
int32_t numOfStt;
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 0fe7f9a773..253d5aebce 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -187,13 +187,14 @@ int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid
int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
// tq
-int tqInit();
-void tqCleanUp();
-STQ* tqOpen(const char* path, SVnode* pVnode);
-void tqClose(STQ*);
-int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
-int tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type);
-int tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer);
+int tqInit();
+void tqCleanUp();
+STQ* tqOpen(const char* path, SVnode* pVnode);
+void tqClose(STQ*);
+int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
+int tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp,
+ int32_t type);
+int tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer);
int tqCommit(STQ*);
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 3ed1b083e4..c75c675ec3 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -669,7 +669,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
#endif
for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) {
SSDataBlock *output = taosArrayGetP(pResList, i);
- smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid,
+ smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRId64, output->info.id.uid,
output->info.id.groupId, output->info.rows);
STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 5360eb2328..75c2bfcce8 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -55,7 +55,7 @@ void tqCleanUp() {
}
}
-static void destroySTqHandle(void* data) {
+static void destroyTqHandle(void* data) {
STqHandle* pData = (STqHandle*)data;
qDestroyTask(pData->execHandle.task);
if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
@@ -93,9 +93,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
pTq->walLogLastVer = pVnode->pWal->vers.lastVer;
pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
- taosHashSetFreeFp(pTq->pHandle, destroySTqHandle);
+ taosHashSetFreeFp(pTq->pHandle, destroyTqHandle);
- taosInitRWLatch(&pTq->pushLock);
+ taosInitRWLatch(&pTq->lock);
pTq->pPushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
taosHashSetFreeFp(pTq->pPushMgr, tqPushEntryFree);
@@ -209,7 +209,7 @@ static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqData
if (type == TMQ_MSG_TYPE__POLL_RSP) {
tEncodeSMqDataRsp(&encoder, pRsp);
} else if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
- tEncodeSTaosxRsp(&encoder, (STaosxRsp*) pRsp);
+ tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp);
}
tEncoderClear(&encoder);
@@ -222,57 +222,11 @@ static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqData
};
tmsgSendRsp(&rsp);
-
return 0;
}
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
SMqDataRsp* pRsp = pPushEntry->pDataRsp;
-
-#if 0
- A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
- A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
-
- A(!pRsp->withSchema);
- A(taosArrayGetSize(pRsp->blockSchema) == 0);
-
- if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
- A(pRsp->rspOffset.version > pRsp->reqOffset.version);
- }
-#endif
-
-// int32_t len = 0;
-// int32_t code = 0;
-// tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
-// if (code < 0) {
-// return -1;
-// }
-//
-// int32_t tlen = sizeof(SMqRspHead) + len;
-// void* buf = rpcMallocCont(tlen);
-// if (buf == NULL) {
-// return -1;
-// }
-//
-// memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead));
-//
-// void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-//
-// SEncoder encoder = {0};
-// tEncoderInit(&encoder, abuf, len);
-// tEncodeSMqDataRsp(&encoder, pRsp);
-// tEncoderClear(&encoder);
-//
-// SRpcMsg rsp = {
-// .info = pPushEntry->pInfo,
-// .pCont = buf,
-// .contLen = tlen,
-// .code = 0,
-// };
-//
-// tmsgSendRsp(&rsp);
-//
-
SMqRspHead* pHeader = &pPushEntry->pDataRsp->head;
doSendDataRsp(&pPushEntry->info, pRsp, pHeader->epoch, pHeader->consumerId, pHeader->mqMsgType);
@@ -294,11 +248,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
A(taosArrayGetSize(pRsp->blockSchema) == 0);
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
- if (pRsp->blockNum > 0) {
- A(pRsp->rspOffset.version > pRsp->reqOffset.version);
- } else {
- A(pRsp->rspOffset.version >= pRsp->reqOffset.version);
- }
+ A(pRsp->rspOffset.version > pRsp->reqOffset.version);
}
#endif
doSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type);
@@ -308,7 +258,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
tFormatOffset(buf1, 80, &pRsp->reqOffset);
tFormatOffset(buf2, 80, &pRsp->rspOffset);
- tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%"PRIx64,
+ tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64,
TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId);
return 0;
@@ -321,7 +271,7 @@ static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOf
int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
STqOffset offset = {0};
- int32_t vgId = TD_VID(pTq->pVnode);
+ int32_t vgId = TD_VID(pTq->pVnode);
SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
@@ -335,8 +285,8 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64,
offset.subKey, vgId, offset.val.uid, offset.val.ts);
} else if (offset.val.type == TMQ_OFFSET__LOG) {
- tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey,
- vgId, offset.val.version);
+ tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey, vgId,
+ offset.val.version);
if (offset.val.version + 1 == sversion) {
offset.val.version += 1;
}
@@ -449,8 +399,9 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
char formatBuf[80];
tFormatOffset(formatBuf, 80, pOffsetVal);
- tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, prev offset found, offset reset to %s and continue.",
- consumerId, pHandle->subKey, vgId, formatBuf);
+ tqDebug("tmq poll: consumer:0x%" PRIx64
+ ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64,
+ consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId);
return 0;
} else {
// no poll occurs in this vnode for this topic, let's seek to the right offset value.
@@ -497,7 +448,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
return code;
}
} else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
- tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed",
+ tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64
+ " in vg %d, subkey %s, reset none failed",
pHandle->subKey, consumerId, vgId, pRequest->subKey);
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
return -1;
@@ -507,7 +459,49 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
return 0;
}
-static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
+#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0)
+
+static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
+ SRpcMsg* pMsg, STqOffsetVal* pOffset) {
+ int32_t code = 0;
+ uint64_t consumerId = pRequest->consumerId;
+ int32_t vgId = TD_VID(pTq->pVnode);
+
+ SMqDataRsp dataRsp = {0};
+ tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
+
+ // lock
+ taosWLockLatch(&pTq->lock);
+
+ qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
+ code = tqScanData(pTq, pHandle, &dataRsp, pOffset);
+ if (code != TSDB_CODE_SUCCESS) {
+ taosWUnLockLatch(&pTq->lock);
+ return code;
+ }
+
+ // till now, all data has been transferred to consumer, new data needs to push client once arrived.
+ if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
+ dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
+ code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
+ taosWUnLockLatch(&pTq->lock);
+ return code;
+ }
+
+ taosWUnLockLatch(&pTq->lock);
+ code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP);
+
+ // NOTE: this pHandle->consumerId may have been changed already.
+ tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64
+ ", ts:%" PRId64 ", reqId:0x%" PRIx64,
+ consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid,
+ dataRsp.rspOffset.ts, pRequest->reqId);
+
+ tDeleteSMqDataRsp(&dataRsp);
+ return code;
+}
+
+static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
int32_t code = -1;
STqOffsetVal offset = {0};
SWalCkHead* pCkHead = NULL;
@@ -517,9 +511,8 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
uint64_t consumerId = pRequest->consumerId;
// 1. reset the offset if needed
- if (reqOffset.type > 0) {
- offset = reqOffset;
- } else { // handle the reset offset cases, according to the consumer's choice.
+ if (IS_OFFSET_RESET_TYPE(reqOffset.type)) {
+ // handle the reset offset cases, according to the consumer's choice.
bool blockReturned = false;
code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned);
if (code != 0) {
@@ -530,44 +523,20 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
if (blockReturned) {
return 0;
}
+ } else { // use the consumer specified offset
+ // the offset value can not be monotonious increase??
+ offset = reqOffset;
}
- // this is a normal subscription requirement
+ // this is a normal subscribe requirement
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
- SMqDataRsp dataRsp = {0};
- tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
-
- // lock
- taosWLockLatch(&pTq->pushLock);
-
- qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
- code = tqScanData(pTq, pHandle, &dataRsp, &offset);
-
- // till now, all data has been transferred to consumer, new data needs to push client once arrived.
- if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
- dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
- code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
- taosWUnLockLatch(&pTq->pushLock);
- return code;
- }
-
- taosWUnLockLatch(&pTq->pushLock);
- code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP);
-
- // NOTE: this pHandle->consumerId may have been changed already.
- tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64
- ", ts:%" PRId64", reqId:0x%"PRIx64,
- consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid,
- dataRsp.rspOffset.ts, pRequest->reqId);
-
- tDeleteSMqDataRsp(&dataRsp);
- return code;
+ return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
}
// todo handle the case where re-balance occurs.
// for taosx
SMqMetaRsp metaRsp = {0};
- STaosxRsp taosxRsp = {0};
+ STaosxRsp taosxRsp = {0};
tqInitTaosxRsp(&taosxRsp, pRequest);
if (offset.type != TMQ_OFFSET__LOG) {
@@ -579,8 +548,7 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp);
tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64
",ts:%" PRId64,
- consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
- metaRsp.rspOffset.ts);
+ consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts);
taosMemoryFree(metaRsp.metaRsp);
tDeleteSTaosxRsp(&taosxRsp);
return code;
@@ -598,9 +566,9 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
",version:%" PRId64,
consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,
taosxRsp.rspOffset.version);
- } else {
+ }
-// if (offset.type == TMQ_OFFSET__LOG) {
+ if (offset.type == TMQ_OFFSET__LOG) {
int64_t fetchVer = offset.version + 1;
pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
if (pCkHead == NULL) {
@@ -608,9 +576,8 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
-
walSetReaderCapacity(pHandle->pWalReader, 2048);
-
+ int totalRows = 0;
while (1) {
// todo refactor: this is not correct.
int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
@@ -633,32 +600,16 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId,
pRequest->epoch, vgId, fetchVer, pHead->msgType);
- if (pHead->msgType == TDMT_VND_SUBMIT) {
- SPackedData submit = {
- .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)),
- .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg),
- .ver = pHead->version,
- };
-
- if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) {
- tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, vgId,
- pRequest->subKey);
- return -1;
- }
-
- if (taosxRsp.blockNum > 0) {
- tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
+ // process meta
+ if (pHead->msgType != TDMT_VND_SUBMIT) {
+ if (totalRows > 0) {
+ tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1);
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
tDeleteSTaosxRsp(&taosxRsp);
taosMemoryFreeClear(pCkHead);
return code;
- } else {
- fetchVer++;
}
- } else {
- /*A(pHandle->fetchMeta);*/
- /*A(IS_META_MSG(pHead->msgType));*/
tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType));
tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer);
metaRsp.resMsgType = pHead->msgType;
@@ -675,6 +626,31 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
tDeleteSTaosxRsp(&taosxRsp);
return code;
}
+
+ // process data
+ SPackedData submit = {
+ .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)),
+ .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg),
+ .ver = pHead->version,
+ };
+
+ if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) {
+ tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, vgId,
+ pRequest->subKey);
+ taosMemoryFreeClear(pCkHead);
+ tDeleteSTaosxRsp(&taosxRsp);
+ return -1;
+ }
+
+ if (totalRows >= 4096 || taosxRsp.createTableNum > 0) {
+ tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
+ code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
+ tDeleteSTaosxRsp(&taosxRsp);
+ taosMemoryFreeClear(pCkHead);
+ return code;
+ } else {
+ fetchVer++;
+ }
}
}
@@ -684,7 +660,7 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
}
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
- SMqPollReq req = {0};
+ SMqPollReq req = {0};
if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen);
terrno = TSDB_CODE_INVALID_MSG;
@@ -705,31 +681,32 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
}
// 2. check re-balance status
- taosRLockLatch(&pTq->pushLock);
+ taosRLockLatch(&pTq->lock);
if (pHandle->consumerId != consumerId) {
tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64,
consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId);
terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
- taosRUnLockLatch(&pTq->pushLock);
+ taosRUnLockLatch(&pTq->lock);
return -1;
}
- taosRUnLockLatch(&pTq->pushLock);
+ taosRUnLockLatch(&pTq->lock);
- taosWLockLatch(&pTq->pushLock);
// 3. update the epoch value
+ taosWLockLatch(&pTq->lock);
int32_t savedEpoch = pHandle->epoch;
if (savedEpoch < reqEpoch) {
- tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, reqEpoch);
+ tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch,
+ reqEpoch);
pHandle->epoch = reqEpoch;
}
- taosWUnLockLatch(&pTq->pushLock);
+ taosWUnLockLatch(&pTq->lock);
char buf[80];
tFormatOffset(buf, 80, &reqOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64,
consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId);
- return extractDataForMq(pTq, pHandle, &req, pMsg);
+ return doPollDataForMq(pTq, pHandle, &req, pMsg);
}
int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
@@ -737,12 +714,12 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
tqDebug("vgId:%d, tq process delete sub req %s", pTq->pVnode->config.vgId, pReq->subKey);
- taosWLockLatch(&pTq->pushLock);
+ taosWLockLatch(&pTq->lock);
int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey));
if (code != 0) {
tqDebug("vgId:%d, tq remove push handle %s", pTq->pVnode->config.vgId, pReq->subKey);
}
- taosWUnLockLatch(&pTq->pushLock);
+ taosWUnLockLatch(&pTq->lock);
STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey));
if (pHandle) {
@@ -806,25 +783,24 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
SVnode* pVnode = pTq->pVnode;
int32_t vgId = TD_VID(pVnode);
- tqDebug("vgId:%d, tq process sub req %s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey,
+ tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey,
req.oldConsumerId, req.newConsumerId);
STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle == NULL) {
if (req.oldConsumerId != -1) {
- tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId is %" PRId64 "",
+ tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64,
req.vgId, req.subKey, req.newConsumerId, req.oldConsumerId);
}
if (req.newConsumerId == -1) {
- tqError("vgId:%d, tq invalid rebalance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId);
+ tqError("vgId:%d, tq invalid re-balance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId);
taosMemoryFree(req.qmsg);
return 0;
}
STqHandle tqHandle = {0};
pHandle = &tqHandle;
- /*taosInitRWLatch(&pExec->lock);*/
uint64_t oldConsumerId = pHandle->consumerId;
memcpy(pHandle->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN);
@@ -858,8 +834,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
pHandle->execHandle.execCol.qmsg = req.qmsg;
req.qmsg = NULL;
- pHandle->execHandle.task =
- qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, &pHandle->execHandle.numOfCols, NULL);
+ pHandle->execHandle.task = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId,
+ &pHandle->execHandle.numOfCols, req.newConsumerId);
void* scanner = NULL;
qExtractStreamScanner(pHandle->execHandle.task, &scanner);
pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner);
@@ -872,7 +848,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
(SSnapContext**)(&handle.sContext));
- pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, NULL);
+ pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId);
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL);
pHandle->execHandle.execTb.suid = req.suid;
@@ -890,7 +866,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta,
(SSnapContext**)(&handle.sContext));
- pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, NULL);
+ pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId);
}
taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
@@ -907,28 +883,34 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
atomic_add_fetch_32(&pHandle->epoch, 1);
taosMemoryFree(req.qmsg);
return tqMetaSaveHandle(pTq, req.subKey, pHandle);
- }
+ } else {
+ tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
+ req.newConsumerId);
- tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
- req.newConsumerId);
+ // kill executing task
+ qTaskInfo_t pTaskInfo = pHandle->execHandle.task;
+ if (pTaskInfo != NULL) {
+ qKillTask(pTaskInfo, TSDB_CODE_SUCCESS);
+ }
- taosWLockLatch(&pTq->pushLock);
- atomic_store_32(&pHandle->epoch, -1);
+ taosWLockLatch(&pTq->lock);
+ atomic_store_32(&pHandle->epoch, -1);
- // remove if it has been register in the push manager, and return one empty block to consumer
- tqRemovePushEntry(pTq, req.subKey, (int32_t) strlen(req.subKey), pHandle->consumerId, true);
+ // remove if it has been register in the push manager, and return one empty block to consumer
+ tqUnregisterPushEntry(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true);
- atomic_store_64(&pHandle->consumerId, req.newConsumerId);
- atomic_add_fetch_32(&pHandle->epoch, 1);
+ atomic_store_64(&pHandle->consumerId, req.newConsumerId);
+ atomic_add_fetch_32(&pHandle->epoch, 1);
- if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
- qStreamCloseTsdbReader(pHandle->execHandle.task);
- }
+ if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+ qStreamCloseTsdbReader(pTaskInfo);
+ }
- taosWUnLockLatch(&pTq->pushLock);
- if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
- taosMemoryFree(req.qmsg);
- return -1;
+ taosWUnLockLatch(&pTq->lock);
+ if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
+ taosMemoryFree(req.qmsg);
+ return -1;
+ }
}
}
@@ -1008,7 +990,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
pTask->tbSink.vnode = pTq->pVnode;
pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2;
- int32_t ver1 = 1;
+ int32_t ver1 = 1;
SMetaInfo info = {0};
int32_t code = metaGetInfo(pTq->pVnode->pMeta, pTask->tbSink.stbUid, &info, NULL);
if (code == TSDB_CODE_SUCCESS) {
@@ -1017,13 +999,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
pTask->tbSink.pTSchema =
tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, ver1);
- if(pTask->tbSink.pTSchema == NULL) {
+ if (pTask->tbSink.pTSchema == NULL) {
return -1;
}
}
streamSetupTrigger(pTask);
- tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", vgId, pTask->taskId, pTask->selfChildId, pTask->taskLevel);
+ tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", vgId, pTask->taskId, pTask->selfChildId,
+ pTask->taskLevel);
return 0;
}
diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c
index b6bca1e4ca..7b0cdab2f8 100644
--- a/source/dnode/vnode/src/tq/tqMeta.c
+++ b/source/dnode/vnode/src/tq/tqMeta.c
@@ -269,7 +269,7 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) {
}
int32_t tqMetaRestoreHandle(STQ* pTq) {
- int code = 0;
+ int code = 0;
TBC* pCur = NULL;
if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) {
return -1;
@@ -307,7 +307,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
handle.execHandle.task =
- qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, NULL);
+ qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0);
if (handle.execHandle.task == NULL) {
tqError("cannot create exec task for %s", handle.subKey);
code = -1;
@@ -332,7 +332,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta,
(SSnapContext**)(&reader.sContext));
- handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, NULL);
+ handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0);
} else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
@@ -341,7 +341,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid);
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
- tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid);
+ tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid);
}
handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode);
tqReaderSetTbUidList(handle.execHandle.pExecReader, tbUidList);
@@ -349,9 +349,9 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType,
handle.fetchMeta, (SSnapContext**)(&reader.sContext));
- handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, NULL);
+ handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0);
}
- tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, TD_VID(pTq->pVnode));
+ tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId);
taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle));
}
diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c
index 01d8e7cf14..d55b533f8a 100644
--- a/source/dnode/vnode/src/tq/tqPush.c
+++ b/source/dnode/vnode/src/tq/tqPush.c
@@ -206,38 +206,110 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
}
#endif
-int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
+typedef struct {
+ void* pKey;
+ int64_t keyLen;
+} SItem;
+
+static void recordPushedEntry(SArray* cachedKey, void* pIter);
+
+static void freeItem(void* param) {
+ SItem* p = (SItem*)param;
+ taosMemoryFree(p->pKey);
+}
+
+static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) {
+ int32_t vgId = TD_VID(pTq->pVnode);
+ int32_t numOfKeys = (int32_t)taosArrayGetSize(pCachedKeys);
+
+ for (int32_t i = 0; i < numOfKeys; i++) {
+ SItem* pItem = taosArrayGet(pCachedKeys, i);
+ if (taosHashRemove(pTq->pPushMgr, pItem->pKey, pItem->keyLen) != 0) {
+ tqError("vgId:%d, tq push hash remove key error, key: %s", vgId, (char*)pItem->pKey);
+ }
+ }
+
+ if (numOfKeys > 0) {
+ tqDebug("vgId:%d, pushed %d items and remain:%d", vgId, numOfKeys, (int32_t)taosHashGetSize(pTq->pPushMgr));
+ }
+}
+
+static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int64_t ver, int32_t vgId, char* pData,
+ int32_t dataLen, SArray* pCachedKey) {
+ STqPushEntry* pPushEntry = *(STqPushEntry**)pIter;
+
+ SMqDataRsp* pRsp = pPushEntry->pDataRsp;
+ if (pRsp->reqOffset.version >= ver) {
+ tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId,
+ pRsp->reqOffset.version, ver);
+ return;
+ }
+
+ qTaskInfo_t pTaskInfo = pExec->task;
+
+ // prepare scan mem data
+ SPackedData submit = {.msgStr = pData, .msgLen = dataLen, .ver = ver};
+
+ if (qStreamSetScanMemData(pTaskInfo, submit) != 0) {
+ return;
+ }
+
+ // here start to scan submit block to extract the subscribed data
+ int32_t totalRows = 0;
+
+ while (1) {
+ SSDataBlock* pDataBlock = NULL;
+ uint64_t ts = 0;
+ if (qExecTask(pTaskInfo, &pDataBlock, &ts) < 0) {
+ tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr());
+ }
+
+ if (pDataBlock == NULL) {
+ break;
+ }
+
+ tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
+ pRsp->blockNum++;
+ totalRows += pDataBlock->info.rows;
+ }
+
+ tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d, rows:%d", vgId, pPushEntry->subKey, pRsp->blockNum,
+ totalRows);
+
+ if (pRsp->blockNum > 0) {
+ tqOffsetResetToLog(&pRsp->rspOffset, ver);
+ tqPushDataRsp(pTq, pPushEntry);
+ recordPushedEntry(pCachedKey, pIter);
+ }
+}
+
+int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg));
int32_t len = msgLen - sizeof(SSubmitReq2Msg);
int32_t vgId = TD_VID(pTq->pVnode);
if (msgType == TDMT_VND_SUBMIT) {
// lock push mgr to avoid potential msg lost
- taosWLockLatch(&pTq->pushLock);
+ taosWLockLatch(&pTq->lock);
int32_t numOfRegisteredPush = taosHashGetSize(pTq->pPushMgr);
if (numOfRegisteredPush > 0) {
tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d",
- vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush);
+ vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush);
- SArray* cachedKeys = taosArrayInit(0, sizeof(void*));
- SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t));
-
- void* data = taosMemoryMalloc(len);
+ void* data = taosMemoryMalloc(len);
if (data == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
- tqError("failed to copy data for stream since out of memory");
- taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree);
- taosArrayDestroy(cachedKeyLens);
-
- // unlock
- taosWUnLockLatch(&pTq->pushLock);
+ tqError("failed to copy data for stream since out of memory, vgId:%d", vgId);
+ taosWUnLockLatch(&pTq->lock);
return -1;
}
memcpy(data, pReq, len);
- void* pIter = NULL;
+ SArray* cachedKey = taosArrayInit(0, sizeof(SItem));
+ void* pIter = NULL;
+
while (1) {
pIter = taosHashIterate(pTq->pPushMgr, pIter);
if (pIter == NULL) {
@@ -248,83 +320,30 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey));
if (pHandle == NULL) {
- tqDebug("vgId:%d, cannot find handle %s", pTq->pVnode->config.vgId, pPushEntry->subKey);
- continue;
- }
-
- SMqDataRsp* pRsp = pPushEntry->pDataRsp;
- if (pRsp->reqOffset.version >= ver) {
- tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId,
- pRsp->reqOffset.version, ver);
+ tqDebug("vgId:%d, failed to find handle %s in pushing data to consumer, ignore", pTq->pVnode->config.vgId,
+ pPushEntry->subKey);
continue;
}
STqExecHandle* pExec = &pHandle->execHandle;
- qTaskInfo_t task = pExec->task;
-
- // prepare scan mem data
- SPackedData submit = {
- .msgStr = data,
- .msgLen = len,
- .ver = ver,
- };
- if(qStreamSetScanMemData(task, submit) != 0){
- continue;
- }
-
- // here start to scan submit block to extract the subscribed data
- while (1) {
- SSDataBlock* pDataBlock = NULL;
- uint64_t ts = 0;
- if (qExecTask(task, &pDataBlock, &ts) < 0) {
- tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr());
- }
-
- if (pDataBlock == NULL) {
- break;
- }
-
- tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
- pRsp->blockNum++;
- }
-
- tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d", vgId, pPushEntry->subKey, pRsp->blockNum);
- if (pRsp->blockNum > 0) {
- // set offset
- tqOffsetResetToLog(&pRsp->rspOffset, ver);
-
- // remove from hash
- size_t kLen;
- void* key = taosHashGetKey(pIter, &kLen);
- void* keyCopy = taosMemoryCalloc(1, kLen + 1);
- memcpy(keyCopy, key, kLen);
-
- taosArrayPush(cachedKeys, &keyCopy);
- taosArrayPush(cachedKeyLens, &kLen);
-
- tqPushDataRsp(pTq, pPushEntry);
- }
+ doPushDataForEntry(pIter, pExec, pTq, ver, vgId, data, len, cachedKey);
}
- // delete entry
- for (int32_t i = 0; i < taosArrayGetSize(cachedKeys); i++) {
- void* key = taosArrayGetP(cachedKeys, i);
- size_t kLen = *(size_t*)taosArrayGet(cachedKeyLens, i);
- if (taosHashRemove(pTq->pPushMgr, key, kLen) != 0) {
- tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key);
- }
- }
-
- taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree);
- taosArrayDestroy(cachedKeyLens);
+ doRemovePushedEntry(cachedKey, pTq);
+ taosArrayDestroyEx(cachedKey, freeItem);
taosMemoryFree(data);
}
+
// unlock
- taosWUnLockLatch(&pTq->pushLock);
+ taosWUnLockLatch(&pTq->lock);
}
+ // push data for stream processing
if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode)) {
- if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0;
+ if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) {
+ return 0;
+ }
+
if (msgType == TDMT_VND_SUBMIT) {
void* data = taosMemoryMalloc(len);
if (data == NULL) {
@@ -332,12 +351,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
tqError("failed to copy data for stream since out of memory");
return -1;
}
+
memcpy(data, pReq, len);
- SPackedData submit = {
- .msgStr = data,
- .msgLen = len,
- .ver = ver,
- };
+ SPackedData submit = {.msgStr = data, .msgLen = len, .ver = ver};
tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq);
tqProcessSubmitReq(pTq, submit);
@@ -351,8 +367,15 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
return 0;
}
-int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg,
- SMqDataRsp* pDataRsp, int32_t type) {
+void recordPushedEntry(SArray* cachedKey, void* pIter) {
+ size_t kLen = 0;
+ void* key = taosHashGetKey(pIter, &kLen);
+ SItem item = {.pKey = strndup(key, kLen), .keyLen = kLen};
+ taosArrayPush(cachedKey, &item);
+}
+
+int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp,
+ int32_t type) {
uint64_t consumerId = pRequest->consumerId;
int32_t vgId = TD_VID(pTq->pVnode);
STqHandle* pTqHandle = pHandle;
@@ -369,7 +392,7 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest,
memcpy(pPushEntry->subKey, pTqHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN);
if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
- pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(STaosxRsp));
+ pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(STaosxRsp));
memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(STaosxRsp));
} else if (type == TMQ_MSG_TYPE__POLL_RSP) {
pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(SMqDataRsp));
@@ -383,13 +406,13 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest,
taosHashPut(pTq->pPushMgr, pTqHandle->subKey, strlen(pTqHandle->subKey), &pPushEntry, sizeof(void*));
- tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr, total:%d", consumerId,
- pTqHandle->subKey, pDataRsp->reqOffset.version, vgId, taosHashGetSize(pTq->pPushMgr));
+ tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr, total:%d",
+ consumerId, pTqHandle->subKey, pDataRsp->reqOffset.version, vgId, taosHashGetSize(pTq->pPushMgr));
return 0;
}
-int32_t tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) {
- int32_t vgId = TD_VID(pTq->pVnode);
+int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) {
+ int32_t vgId = TD_VID(pTq->pVnode);
STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen);
if (pEntry != NULL) {
@@ -399,7 +422,7 @@ int32_t tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t c
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s vgId:%d remove from push mgr, remains:%d", consumerId,
(*pEntry)->subKey, vgId, taosHashGetSize(pTq->pPushMgr) - 1);
- if (rspConsumer) { // rsp the old consumer with empty block.
+ if (rspConsumer) { // rsp the old consumer with empty block.
tqPushDataRsp(pTq, *pEntry);
}
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 7f9563ae5f..90ff1f8a84 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -340,7 +340,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
continue;
}
ret->fetchType = FETCH_TYPE__DATA;
- tqDebug("return data rows %d", ret->data.info.rows);
+ tqDebug("return data rows %" PRId64, ret->data.info.rows);
return 0;
}
diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqScan.c
similarity index 82%
rename from source/dnode/vnode/src/tq/tqExec.c
rename to source/dnode/vnode/src/tq/tqScan.c
index a62101eb47..6528b7c8d2 100644
--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -18,7 +18,9 @@
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) {
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
void* buf = taosMemoryCalloc(1, dataStrLen);
- if (buf == NULL) return -1;
+ if (buf == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
pRetrieve->useconds = 0;
@@ -31,7 +33,8 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t
actualLen += sizeof(SRetrieveTableRsp);
taosArrayPush(pRsp->blockDataLen, &actualLen);
taosArrayPush(pRsp->blockData, &buf);
- return 0;
+
+ return TSDB_CODE_SUCCESS;
}
static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) {
@@ -62,69 +65,84 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in
}
int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
- const STqExecHandle* pExec = &pHandle->execHandle;
+ const int32_t MAX_ROWS_TO_RETURN = 4096;
+ int32_t vgId = TD_VID(pTq->pVnode);
+ int32_t code = 0;
+ int32_t totalRows = 0;
- qTaskInfo_t task = pExec->task;
+ const STqExecHandle* pExec = &pHandle->execHandle;
+ qTaskInfo_t task = pExec->task;
if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
- tqDebug("prepare scan failed, return");
+ tqDebug("prepare scan failed, vgId:%d, consumer:0x%" PRIx64, vgId, pHandle->consumerId);
if (pOffset->type == TMQ_OFFSET__LOG) {
pRsp->rspOffset = *pOffset;
- return 0;
+ return code;
} else {
tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
- tqDebug("prepare scan failed, return");
+ tqDebug("prepare scan failed, vgId:%d, consumer:0x%" PRIx64, vgId, pHandle->consumerId);
pRsp->rspOffset = *pOffset;
- return 0;
+ return code;
}
}
}
- int32_t rowCnt = 0;
while (1) {
SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0;
- tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId);
- if (qExecTask(task, &pDataBlock, &ts) < 0) {
- tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr());
- return -1;
+ tqDebug("vgId:%d, tmq task start to execute, consumer:0x%" PRIx64, vgId, pHandle->consumerId);
+
+ code = qExecTask(task, &pDataBlock, &ts);
+ if (code != TSDB_CODE_SUCCESS) {
+ tqError("vgId:%d, task exec error since %s, consumer:0x%" PRIx64, vgId, terrstr(), pHandle->consumerId);
+ return code;
}
- tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task executed, get %p", pHandle->consumerId, pTq->pVnode->config.vgId, pDataBlock);
-
- // current scan should be stopped asap, since the rebalance occurs.
+ // current scan should be stopped ASAP, since the re-balance occurs.
if (pDataBlock == NULL) {
break;
}
- tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
+ code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
+ if (code != TSDB_CODE_SUCCESS) {
+ tqError("vgId:%d, failed to add block to rsp msg", vgId);
+ return code;
+ }
+
pRsp->blockNum++;
+ tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%" PRId64 ", total blocks:%d", vgId,
+ pHandle->consumerId, pDataBlock->info.rows, pRsp->blockNum);
+
if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
- rowCnt += pDataBlock->info.rows;
- if (rowCnt >= 4096) {
+ totalRows += pDataBlock->info.rows;
+ if (totalRows >= MAX_ROWS_TO_RETURN) {
break;
}
}
}
- if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) {
- return -1;
- }
+ qStreamExtractOffset(task, &pRsp->rspOffset);
if (pRsp->rspOffset.type == 0) {
- tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts,
- pRsp->rspOffset.uid, pRsp->rspOffset.version);
- return -1;
+ code = TSDB_CODE_INVALID_PARA;
+ tqError("vgId:%d, expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, vgId, pRsp->rspOffset.type,
+ pRsp->rspOffset.ts, pRsp->rspOffset.uid, pRsp->rspOffset.version);
+ return code;
}
if (pRsp->withTbName || pRsp->withSchema) {
- tqError("get column should not with meta:%d,%d", pRsp->withTbName, pRsp->withSchema);
- return -1;
+ code = TSDB_CODE_INVALID_PARA;
+ tqError("vgId:%d, get column should not with meta:%d,%d", vgId, pRsp->withTbName, pRsp->withSchema);
+ return code;
}
- return 0;
+
+ tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, total blocks:%d, rows:%d", vgId, pHandle->consumerId,
+ pRsp->blockNum, totalRows);
+
+ return code;
}
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
@@ -230,23 +248,15 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
return 0;
}
-int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) {
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows) {
STqExecHandle* pExec = &pHandle->execHandle;
- /*A(pExec->subType != TOPIC_SUB_TYPE__COLUMN);*/
-
- SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
- SArray* pSchemas = taosArrayInit(0, sizeof(void*));
+ SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
+ SArray* pSchemas = taosArrayInit(0, sizeof(void*));
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
STqReader* pReader = pExec->pExecReader;
- /*tqReaderSetDataMsg(pReader, pReq, 0);*/
tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver);
while (tqNextDataBlock2(pReader)) {
- /*SSDataBlock block = {0};*/
- /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/
- /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/
- /*}*/
-
taosArrayClear(pBlocks);
taosArrayClear(pSchemas);
SSubmitTbData* pSubmitTbDataRet = NULL;
@@ -254,7 +264,6 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
}
if (pRsp->withTbName) {
- /*int64_t uid = pExec->pExecReader->msgIter.uid;*/
int64_t uid = pExec->pExecReader->lastBlkUid;
if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes);
@@ -276,7 +285,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
if (TSDB_CODE_SUCCESS != code) {
continue;
}
- void* createReq = taosMemoryCalloc(1, len);
+ void* createReq = taosMemoryCalloc(1, len);
SEncoder encoder = {0};
tEncoderInit(&encoder, createReq, len);
code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq);
@@ -296,6 +305,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
SSDataBlock* pBlock = taosArrayGet(pBlocks, i);
tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock),
pTq->pVnode->config.tsdbCfg.precision);
+ totalRows += pBlock->info.rows;
blockDataFreeRes(pBlock);
SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i);
taosArrayPush(pRsp->blockSchema, &pSW);
@@ -304,13 +314,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
}
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
STqReader* pReader = pExec->pExecReader;
- /*tqReaderSetDataMsg(pReader, pReq, 0);*/
tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver);
while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) {
- /*SSDataBlock block = {0};*/
- /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/
- /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/
- /*}*/
taosArrayClear(pBlocks);
taosArrayClear(pSchemas);
SSubmitTbData* pSubmitTbDataRet = NULL;
@@ -339,7 +344,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
if (TSDB_CODE_SUCCESS != code) {
continue;
}
- void* createReq = taosMemoryCalloc(1, len);
+ void* createReq = taosMemoryCalloc(1, len);
SEncoder encoder = {0};
tEncoderInit(&encoder, createReq, len);
code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq);
@@ -355,15 +360,11 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
tEncoderClear(&encoder);
}
- /*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/
- /*pTq->pVnode->config.tsdbCfg.precision);*/
- /*blockDataFreeRes(&block);*/
- /*tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);*/
- /*pRsp->blockNum++;*/
for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) {
SSDataBlock* pBlock = taosArrayGet(pBlocks, i);
tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock),
pTq->pVnode->config.tsdbCfg.precision);
+ *totalRows += pBlock->info.rows;
blockDataFreeRes(pBlock);
SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i);
taosArrayPush(pRsp->blockSchema, &pSW);
@@ -373,9 +374,5 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
}
taosArrayDestroy(pBlocks);
taosArrayDestroy(pSchemas);
-// if (pRsp->blockNum == 0) {
-// return -1;
-// }
-
return 0;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index 3d01184e78..48d3371284 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -590,6 +590,7 @@ typedef struct {
SDataFReader **pDataFReader;
TSDBROW row;
+ bool checkRemainingRow;
SMergeTree mergeTree;
SMergeTree *pMergeTree;
SSttBlockLoadInfo *pLoadInfo;
@@ -633,12 +634,25 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
if (code) goto _err;
}
- state->pLoadInfo->colIds = aCols;
- state->pLoadInfo->numOfCols = nCols;
+ for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) {
+ state->pLoadInfo[i].colIds = aCols;
+ state->pLoadInfo[i].numOfCols = nCols;
+ state->pLoadInfo[i].isLast = isLast;
+ }
tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
&(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
state->pMergeTree = &state->mergeTree;
+ state->state = SFSLASTNEXTROW_BLOCKROW;
+ }
+ case SFSLASTNEXTROW_BLOCKROW: {
+ if (nCols != state->pLoadInfo->numOfCols) {
+ for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) {
+ state->pLoadInfo[i].numOfCols = nCols;
+
+ state->pLoadInfo[i].checkRemainingRow = state->checkRemainingRow;
+ }
+ }
bool hasVal = tMergeTreeNext(&state->mergeTree);
if (!hasVal) {
if (tMergeTreeIgnoreEarlierTs(&state->mergeTree)) {
@@ -649,76 +663,23 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
state->state = SFSLASTNEXTROW_FILESET;
goto _next_fileset;
}
- state->state = SFSLASTNEXTROW_BLOCKROW;
- checkRemainingRow = false;
- }
- case SFSLASTNEXTROW_BLOCKROW: {
- bool skipRow = false;
- do {
- bool hasVal = false;
- state->row = tMergeTreeGetRow(&state->mergeTree);
- *ppRow = &state->row;
- if (nCols != state->pLoadInfo->numOfCols) {
- state->pLoadInfo->numOfCols = nCols;
- }
- hasVal = tMergeTreeNext(&state->mergeTree);
- if (TSDBROW_TS(&state->row) <= state->lastTs) {
- *pIgnoreEarlierTs = true;
- *ppRow = NULL;
- return code;
- }
+ state->row = tMergeTreeGetRow(&state->mergeTree);
+ *ppRow = &state->row;
- *pIgnoreEarlierTs = false;
- if (!hasVal) {
- state->state = SFSLASTNEXTROW_FILESET;
- break;
- }
+ if (TSDBROW_TS(&state->row) <= state->lastTs) {
+ *pIgnoreEarlierTs = true;
+ *ppRow = NULL;
+ return code;
+ }
- if (checkRemainingRow) {
- bool skipBlock = true;
-
- SBlockData *pBlockData = state->row.pBlockData;
-
- for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {
- for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
- SColData *pColData = &pBlockData->aColData[colIndex];
- int16_t cid = pColData->cid;
-
- if (cid == aCols[inputColIndex]) {
- if (isLast && (pColData->flag & HAS_VALUE)) {
- skipBlock = false;
- break;
- } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
- skipBlock = false;
- break;
- }
- }
- }
- }
- /*
- for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
- SColData *pColData = &pBlockData->aColData[colIndex];
- int16_t cid = pColData->cid;
-
- if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
- if (isLast && (pColData->flag & HAS_VALUE)) {
- skipBlock = false;
- break;
- } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
- skipBlock = false;
- break;
- }
-
- ++inputColIndex;
- }
- }
- */
- if (skipBlock) {
- skipRow = true;
- }
- }
- } while (skipRow);
+ *pIgnoreEarlierTs = false;
+ if (!hasVal) {
+ state->state = SFSLASTNEXTROW_FILESET;
+ }
+ if (!state->checkRemainingRow) {
+ state->checkRemainingRow = true;
+ }
return code;
}
default:
diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
index d34af9acae..d0ff403bf7 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@@ -282,6 +282,40 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) {
return true;
}
+int64_t tsdbCountTbDataRows(STbData *pTbData) {
+ SMemSkipListNode *pNode = pTbData->sl.pHead;
+ int64_t rowsNum = 0;
+
+ while (NULL != pNode) {
+ pNode = SL_GET_NODE_FORWARD(pNode, 0);
+ if (pNode == pTbData->sl.pTail) {
+ return rowsNum;
+ }
+
+ rowsNum++;
+ }
+
+ return rowsNum;
+}
+
+void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) {
+ taosRLockLatch(&pMemTable->latch);
+ for (int32_t i = 0; i < pMemTable->nBucket; ++i) {
+ STbData *pTbData = pMemTable->aBucket[i];
+ while (pTbData) {
+ void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid));
+ if (p == NULL) {
+ pTbData = pTbData->next;
+ continue;
+ }
+
+ *rowsNum += tsdbCountTbDataRows(pTbData);
+ pTbData = pTbData->next;
+ }
+ }
+ taosRUnLockLatch(&pMemTable->latch);
+}
+
static int32_t tsdbMemTableRehash(SMemTable *pMemTable) {
int32_t code = 0;
@@ -787,4 +821,4 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) {
_exit:
return aTbDataP;
-}
\ No newline at end of file
+}
diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index 943b16116c..fa8870835c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -504,9 +504,34 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
pIter->iRow += step;
while (1) {
+ bool skipBlock = false;
+
findNextValidRow(pIter, idStr);
- if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) {
+ if (pIter->pBlockLoadInfo->checkRemainingRow) {
+ skipBlock = true;
+ int16_t *aCols = pIter->pBlockLoadInfo->colIds;
+ int nCols = pIter->pBlockLoadInfo->numOfCols;
+ bool isLast = pIter->pBlockLoadInfo->isLast;
+ for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {
+ for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
+ SColData *pColData = &pBlockData->aColData[colIndex];
+ int16_t cid = pColData->cid;
+
+ if (cid == aCols[inputColIndex]) {
+ if (isLast && (pColData->flag & HAS_VALUE)) {
+ skipBlock = false;
+ break;
+ } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+ skipBlock = false;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (skipBlock || pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) {
tLDataIterNextBlock(pIter, idStr);
if (pIter->pSttBlk == NULL) { // no more data
goto _exit;
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 30ae83e1f2..480ae7ea64 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -15,6 +15,7 @@
#include "osDef.h"
#include "tsdb.h"
+#include "tsimplehash.h"
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
@@ -24,6 +25,11 @@ typedef enum {
EXTERNAL_ROWS_NEXT = 0x3,
} EContentData;
+typedef enum {
+ READ_MODE_COUNT_ONLY = 0x1,
+ READ_MODE_ALL,
+} EReadMode;
+
typedef struct {
STbDataIter* iter;
int32_t index;
@@ -167,6 +173,8 @@ struct STsdbReader {
uint64_t suid;
int16_t order;
bool freeBlock;
+ EReadMode readMode;
+ uint64_t rowsNum;
STimeWindow window; // the primary query time window that applies to all queries
SSDataBlock* pResBlock;
int32_t capacity;
@@ -176,14 +184,15 @@ struct STsdbReader {
SBlockLoadSuppInfo suppInfo;
STsdbReadSnap* pReadSnap;
SIOCostSummary cost;
- STSchema* pSchema; // the newest version schema
- STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times
- SDataFReader* pFileReader; // the file reader
- SDelFReader* pDelFReader; // the del file reader
- SArray* pDelIdx; // del file block index;
- SBlockInfoBuf blockInfoBuf;
- int32_t step;
- STsdbReader* innerReader[2];
+ STSchema* pSchema; // the newest version schema
+ // STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times
+ SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema
+ SDataFReader* pFileReader; // the file reader
+ SDelFReader* pDelFReader; // the del file reader
+ SArray* pDelIdx; // del file block index;
+ SBlockInfoBuf blockInfoBuf;
+ int32_t step;
+ STsdbReader* innerReader[2];
};
static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter);
@@ -949,14 +958,18 @@ static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int
pDumpInfo->lastKey = maxKey + step;
}
-static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal,
- SBlockLoadSuppInfo* pSup) {
+static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal,
+ SBlockLoadSuppInfo* pSup) {
if (IS_VAR_DATA_TYPE(pColVal->type)) {
if (!COL_VAL_IS_VALUE(pColVal)) {
colDataSetNULL(pColInfoData, rowIndex);
} else {
varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData);
- ASSERT(pColVal->value.nData <= pColInfoData->info.bytes);
+ if (pColVal->value.nData > pColInfoData->info.bytes) {
+ tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData,
+ pColInfoData->info.bytes);
+ return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
+ }
if (pColVal->value.nData > 0) { // pData may be null, if nData is 0
memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData);
}
@@ -966,6 +979,8 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_
} else {
colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value, !COL_VAL_IS_VALUE(pColVal));
}
+
+ return TSDB_CODE_SUCCESS;
}
static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) {
@@ -1167,6 +1182,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) {
SDataBlk* pBlock = getCurrentBlock(pBlockIter);
SSDataBlock* pResBlock = pReader->pResBlock;
int32_t numOfOutputCols = pSupInfo->numOfCols;
+ int32_t code = TSDB_CODE_SUCCESS;
SColVal cv = {0};
int64_t st = taosGetTimestampUs();
@@ -1244,7 +1260,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) {
} else { // varchar/nchar type
for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) {
tColDataGetValue(pData, j, &cv);
- doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo);
+ code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo);
+ if (code) {
+ return code;
+ }
}
}
}
@@ -1766,7 +1785,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
setComposedBlockFlag(pReader, true);
double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
- tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange:%" PRId64
+ tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64
" - %" PRId64 ", uid:%" PRIu64 ", %s",
pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey,
pBlockScanInfo->uid, pReader->idStr);
@@ -1776,23 +1795,29 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
}
static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key,
- SFileBlockDumpInfo* pDumpInfo) {
+ SFileBlockDumpInfo* pDumpInfo, bool* copied) {
// opt version
// 1. it is not a border point
// 2. the direct next point is not an duplicated timestamp
+ int32_t code = TSDB_CODE_SUCCESS;
+
+ *copied = false;
bool asc = (pReader->order == TSDB_ORDER_ASC);
if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) {
int32_t step = pReader->order == TSDB_ORDER_ASC ? 1 : -1;
int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step];
if (nextKey != key) { // merge is not needed
- doAppendRowFromFileBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex);
+ code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex);
+ if (code) {
+ return code;
+ }
pDumpInfo->rowIndex += step;
- return true;
+ *copied = true;
}
}
- return false;
+ return code;
}
static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo,
@@ -1819,20 +1844,35 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
}
static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader,
- STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader) {
+ STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader,
+ bool* copied) {
+ int32_t code = TSDB_CODE_SUCCESS;
+
+ *copied = false;
+
bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo, &pReader->verRange);
if (hasVal) {
int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
if (next1 != ts) {
- doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow);
- return true;
+ code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow);
+ if (code) {
+ return code;
+ }
+
+ *copied = true;
+ return code;
}
} else {
- doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow);
- return true;
+ code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow);
+ if (code) {
+ return code;
+ }
+
+ *copied = true;
+ return code;
}
- return false;
+ return code;
}
static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid) {
@@ -1858,28 +1898,23 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader*
return pReader->pSchema;
}
- if (pReader->pMemSchema == NULL) {
- int32_t code =
- metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema);
- if (code != TSDB_CODE_SUCCESS) {
- terrno = code;
- return NULL;
- } else {
- return pReader->pMemSchema;
- }
+ void** p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion));
+ if (p != NULL) {
+ return *(STSchema**)p;
}
- if (pReader->pMemSchema->version == sversion) {
- return pReader->pMemSchema;
- }
-
- taosMemoryFreeClear(pReader->pMemSchema);
- int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema);
- if (code != TSDB_CODE_SUCCESS || pReader->pMemSchema == NULL) {
+ STSchema* ptr = NULL;
+ int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr);
+ if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return NULL;
} else {
- return pReader->pMemSchema;
+ code = tSimpleHashPut(pReader->pSchemaMap, &sversion, sizeof(sversion), &ptr, POINTER_BYTES);
+ if (code != TSDB_CODE_SUCCESS) {
+ terrno = code;
+ return NULL;
+ }
+ return ptr;
}
}
@@ -2022,11 +2057,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
return code;
}
- doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
+ code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow);
tsdbRowMergerClear(&merge);
- return TSDB_CODE_SUCCESS;
+
+ return code;
}
static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader,
@@ -2034,15 +2070,21 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
bool mergeBlockData) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader);
-
- SRow* pTSRow = NULL;
- SRowMerger merge = {0};
- TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
+ bool copied = false;
+ int32_t code = TSDB_CODE_SUCCESS;
+ SRow* pTSRow = NULL;
+ SRowMerger merge = {0};
+ TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr);
// only last block exists
if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
- if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) {
+ code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied);
+ if (code) {
+ return code;
+ }
+
+ if (copied) {
pBlockScanInfo->lastKey = tsLastBlock;
return TSDB_CODE_SUCCESS;
} else {
@@ -2060,10 +2102,14 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
return code;
}
- doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
+ code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow);
tsdbRowMergerClear(&merge);
+
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
}
} else { // not merge block data
int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema);
@@ -2083,10 +2129,14 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
return code;
}
- doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
+ code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow);
tsdbRowMergerClear(&merge);
+
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
}
return TSDB_CODE_SUCCESS;
@@ -2131,7 +2181,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
return code;
}
- doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
+ code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow);
tsdbRowMergerClear(&merge);
@@ -2353,7 +2403,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
return code;
}
- doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
+ code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow);
tsdbRowMergerClear(&merge);
@@ -2508,7 +2558,13 @@ bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo*
int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key,
STsdbReader* pReader) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
- if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) {
+ bool copied = false;
+ int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied);
+ if (code) {
+ return code;
+ }
+
+ if (copied) {
pBlockScanInfo->lastKey = key;
return TSDB_CODE_SUCCESS;
} else {
@@ -2528,11 +2584,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
return code;
}
- doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
+ code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow);
tsdbRowMergerClear(&merge);
- return TSDB_CODE_SUCCESS;
+ return code;
}
}
@@ -2649,7 +2705,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) &&
pBlock->nRow <= pReader->capacity) {
if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) {
- copyBlockDataToSDataBlock(pReader);
+ code = copyBlockDataToSDataBlock(pReader);
+ if (code) {
+ goto _end;
+ }
// record the last key value
pBlockScanInfo->lastKey = asc ? pBlock->maxKey.ts : pBlock->minKey.ts;
@@ -2696,7 +2755,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
break;
}
- buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader);
+ code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader);
+ if (code) {
+ goto _end;
+ }
// currently loaded file data block is consumed
if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) {
@@ -2715,8 +2777,8 @@ _end:
updateComposedBlockInfo(pReader, el, pBlockScanInfo);
if (pResBlock->info.rows > 0) {
- tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
- " rows:%d, elapsed time:%.2f ms %s",
+ tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64
+ ", elapsed time:%.2f ms %s",
pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
pResBlock->info.rows, el, pReader->idStr);
}
@@ -2922,6 +2984,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status;
SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader;
STableUidList* pUidList = &pStatus->uidList;
+ int32_t code = TSDB_CODE_SUCCESS;
if (taosHashGetSize(pStatus->pTableMap) == 0) {
return TSDB_CODE_SUCCESS;
@@ -2952,7 +3015,11 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
break;
}
- buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
+ code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
+ if (code) {
+ return code;
+ }
+
if (pResBlock->info.rows >= pReader->capacity) {
break;
}
@@ -2962,8 +3029,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
updateComposedBlockInfo(pReader, el, pScanInfo);
if (pResBlock->info.rows > 0) {
- tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
- " rows:%d, elapsed time:%.2f ms %s",
+ tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64
+ ", elapsed time:%.2f ms %s",
pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
pResBlock->info.rows, el, pReader->idStr);
return TSDB_CODE_SUCCESS;
@@ -3032,7 +3099,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
break;
}
- buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
+ code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
+ if (code) {
+ return code;
+ }
+
if (pResBlock->info.rows >= pReader->capacity) {
break;
}
@@ -3042,8 +3113,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
updateComposedBlockInfo(pReader, el, pScanInfo);
if (pResBlock->info.rows > 0) {
- tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
- " rows:%d, elapsed time:%.2f ms %s",
+ tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64
+ ", elapsed time:%.2f ms %s",
pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
pResBlock->info.rows, el, pReader->idStr);
}
@@ -3069,6 +3140,148 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
return code;
}
+static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) {
+ int64_t st = taosGetTimestampUs();
+ LRUHandle* handle = NULL;
+ int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle);
+ if (code != TSDB_CODE_SUCCESS || handle == NULL) {
+ goto _end;
+ }
+
+ int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap);
+
+ SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle);
+ size_t num = taosArrayGetSize(aBlockIdx);
+ if (num == 0) {
+ tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SBlockIdx* pBlockIdx = NULL;
+ int32_t i = 0;
+ for (int32_t i = 0; i < num; ++i) {
+ pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i);
+ if (pBlockIdx->suid != pReader->suid) {
+ continue;
+ }
+
+ STableBlockScanInfo** p = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid));
+ if (p == NULL) {
+ continue;
+ }
+
+ STableBlockScanInfo* pScanInfo = *p;
+ tMapDataReset(&pScanInfo->mapData);
+ tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData);
+
+ SDataBlk block = {0};
+ for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) {
+ tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block);
+ pReader->rowsNum += block.nRow;
+ }
+ }
+
+_end:
+ tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle);
+ return code;
+}
+
+static int32_t doSumSttBlockRows(STsdbReader* pReader) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader;
+ SSttBlockLoadInfo* pBlockLoadInfo = NULL;
+
+ for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file
+ pBlockLoadInfo = &pLastBlockReader->pInfo[i];
+
+ code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk);
+ if (code) {
+ return code;
+ }
+
+ size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
+ if (size >= 1) {
+ SSttBlk* pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
+ SSttBlk* pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1);
+
+ // all identical
+ if (pStart->suid == pEnd->suid) {
+ if (pStart->suid != pReader->suid) {
+ // no qualified stt block existed
+ taosArrayClear(pBlockLoadInfo->aSttBlk);
+ continue;
+ }
+ for (int32_t i = 0; i < size; ++i) {
+ SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
+ pReader->rowsNum += p->nRow;
+ }
+ } else {
+ for (int32_t i = 0; i < size; ++i) {
+ SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
+ uint64_t s = p->suid;
+ if (s < pReader->suid) {
+ continue;
+ }
+
+ if (s == pReader->suid) {
+ pReader->rowsNum += p->nRow;
+ } else if (s > pReader->suid) {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ return code;
+}
+
+static int32_t readRowsCountFromFiles(STsdbReader* pReader) {
+ int32_t code = TSDB_CODE_SUCCESS;
+
+ while (1) {
+ bool hasNext = false;
+ int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext);
+ if (code) {
+ return code;
+ }
+
+ if (!hasNext) { // no data files on disk
+ break;
+ }
+
+ code = doSumFileBlockRows(pReader, pReader->pFileReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+
+ code = doSumSttBlockRows(pReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+ }
+
+ pReader->status.loadFromFile = false;
+
+ return code;
+}
+
+static int32_t readRowsCountFromMem(STsdbReader* pReader) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int64_t memNum = 0, imemNum = 0;
+ if (pReader->pReadSnap->pMem != NULL) {
+ tsdbMemTableCountRows(pReader->pReadSnap->pMem, pReader->status.pTableMap, &memNum);
+ }
+
+ if (pReader->pReadSnap->pIMem != NULL) {
+ tsdbMemTableCountRows(pReader->pReadSnap->pIMem, pReader->status.pTableMap, &imemNum);
+ }
+
+ pReader->rowsNum += memNum + imemNum;
+
+ return code;
+}
+
static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status;
STableUidList* pUidList = &pStatus->uidList;
@@ -3677,8 +3890,9 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p
if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem
STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
+ STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
- int32_t code = tsdbRowMergerInit(&merge, piRow, pSchema);
+ int32_t code = tsdbRowMergerInit2(&merge, pSchema, piRow, piSchema);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@@ -3689,7 +3903,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p
return code;
}
- tsdbRowMerge(&merge, pRow);
+ tsdbRowMergerAdd(&merge, pRow, pSchema);
code =
doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader);
if (code != TSDB_CODE_SUCCESS) {
@@ -3784,6 +3998,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) {
int32_t outputRowIndex = pBlock->info.rows;
int64_t uid = pScanInfo->uid;
+ int32_t code = TSDB_CODE_SUCCESS;
int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
@@ -3806,7 +4021,10 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
tRowGet(pTSRow, pSchema, j, &colVal);
- doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
+ code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
+ if (code) {
+ return code;
+ }
i += 1;
j += 1;
} else if (colId < pSchema->columns[j].colId) {
@@ -3836,6 +4054,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
int32_t rowIndex) {
int32_t i = 0, j = 0;
int32_t outputRowIndex = pResBlock->info.rows;
+ int32_t code = TSDB_CODE_SUCCESS;
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
if (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
@@ -3858,7 +4077,10 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]);
if (pData->cid == pSupInfo->colId[i]) {
tColDataGetValue(pData, rowIndex, &cv);
- doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
+ code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
+ if (code) {
+ return code;
+ }
j += 1;
} else if (pData->cid > pCol->info.colId) {
// the specified column does not exist in file block, fill with null data
@@ -3882,6 +4104,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,
STsdbReader* pReader) {
SSDataBlock* pBlock = pReader->pResBlock;
+ int32_t code = TSDB_CODE_SUCCESS;
do {
// SRow* pTSRow = NULL;
@@ -3893,13 +4116,20 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
}
if (row.type == TSDBROW_ROW_FMT) {
- doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);
+ code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);
if (freeTSRow) {
taosMemoryFree(row.pTSRow);
}
+
+ if (code) {
+ return code;
+ }
} else {
- doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow);
+ code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow);
+ if (code) {
+ break;
+ }
}
// no data in buffer, return immediately
@@ -3912,7 +4142,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
}
} while (1);
- return TSDB_CODE_SUCCESS;
+ return code;
}
// TODO refactor: with createDataBlockScanInfo
@@ -3929,8 +4159,13 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n
if (code) {
return code;
}
- pReader->status.uidList.tableUidList =
- (uint64_t*)taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num);
+
+ char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num);
+ if (p1 == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ pReader->status.uidList.tableUidList = (uint64_t*)p1;
}
taosHashClear(pReader->status.pTableMap);
@@ -3986,6 +4221,8 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
int32_t code = TSDB_CODE_SUCCESS;
if (pStatus->fileIter.numOfFiles == 0) {
pStatus->loadFromFile = false;
+ } else if (READ_MODE_COUNT_ONLY == pReader->readMode) {
+ // DO NOTHING
} else {
code = initForFirstBlockInFile(pReader, pBlockIter);
}
@@ -3997,14 +4234,15 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
return code;
}
+static void freeSchemaFunc(void* param) {
+ void* p = *(void**)param;
+ taosMemoryFree(p);
+}
+
// ====================================== EXPOSED APIs ======================================
int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
- SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) {
+ SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr, bool countOnly) {
STimeWindow window = pCond->twindows;
- if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) {
- pCond->twindows.skey += 1;
- pCond->twindows.ekey -= 1;
- }
int32_t capacity = pVnode->config.tsdbCfg.maxRows;
if (pResBlock != NULL) {
@@ -4027,11 +4265,11 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
// update the SQueryTableDataCond to create inner reader
int32_t order = pCond->order;
if (order == TSDB_ORDER_ASC) {
- pCond->twindows.ekey = window.skey;
+ pCond->twindows.ekey = window.skey - 1;
pCond->twindows.skey = INT64_MIN;
pCond->order = TSDB_ORDER_DESC;
} else {
- pCond->twindows.skey = window.ekey;
+ pCond->twindows.skey = window.ekey + 1;
pCond->twindows.ekey = INT64_MAX;
pCond->order = TSDB_ORDER_ASC;
}
@@ -4043,11 +4281,11 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
}
if (order == TSDB_ORDER_ASC) {
- pCond->twindows.skey = window.ekey;
+ pCond->twindows.skey = window.ekey + 1;
pCond->twindows.ekey = INT64_MAX;
} else {
pCond->twindows.skey = INT64_MIN;
- pCond->twindows.ekey = window.ekey;
+ pCond->twindows.ekey = window.ekey - 1;
}
pCond->order = order;
@@ -4073,6 +4311,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
}
}
+ pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash);
+ if (pReader->pSchemaMap == NULL) {
+ tsdbError("failed init schema hash for reader %s", pReader->idStr);
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc);
if (pReader->pSchema != NULL) {
code = updateBlockSMAInfo(pReader->pSchema, &pReader->suppInfo);
if (code != TSDB_CODE_SUCCESS) {
@@ -4091,6 +4337,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
pReader->suspended = true;
+ if (countOnly) {
+ pReader->readMode = READ_MODE_COUNT_ONLY;
+ }
+
tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
return code;
@@ -4115,7 +4365,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
p->status.uidList.tableUidList = NULL;
p->pReadSnap = NULL;
p->pSchema = NULL;
- p->pMemSchema = NULL;
+ p->pSchemaMap = NULL;
p = pReader->innerReader[1];
@@ -4123,7 +4373,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
p->status.uidList.tableUidList = NULL;
p->pReadSnap = NULL;
p->pSchema = NULL;
- p->pMemSchema = NULL;
+ p->pSchemaMap = NULL;
tsdbReaderClose(pReader->innerReader[0]);
tsdbReaderClose(pReader->innerReader[1]);
@@ -4203,10 +4453,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
taosMemoryFree(pReader->idStr);
taosMemoryFree(pReader->pSchema);
- if (pReader->pMemSchema != pReader->pSchema) {
- taosMemoryFree(pReader->pMemSchema);
- }
-
+ tSimpleHashCleanup(pReader->pSchemaMap);
taosMemoryFreeClear(pReader);
}
@@ -4366,14 +4613,14 @@ int32_t tsdbReaderResume(STsdbReader* pReader) {
pPrevReader->status.pTableMap = pReader->status.pTableMap;
pPrevReader->status.uidList = pReader->status.uidList;
pPrevReader->pSchema = pReader->pSchema;
- pPrevReader->pMemSchema = pReader->pMemSchema;
+ pPrevReader->pSchemaMap = pReader->pSchemaMap;
pPrevReader->pReadSnap = pReader->pReadSnap;
pNextReader->capacity = 1;
pNextReader->status.pTableMap = pReader->status.pTableMap;
pNextReader->status.uidList = pReader->status.uidList;
pNextReader->pSchema = pReader->pSchema;
- pNextReader->pMemSchema = pReader->pMemSchema;
+ pNextReader->pSchemaMap = pReader->pSchemaMap;
pNextReader->pReadSnap = pReader->pReadSnap;
code = doOpenReaderImpl(pPrevReader);
@@ -4394,43 +4641,82 @@ _err:
return code;
}
-static bool doTsdbNextDataBlock(STsdbReader* pReader) {
+static bool tsdbReadRowsCountOnly(STsdbReader* pReader) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ SSDataBlock* pBlock = pReader->pResBlock;
+
+ if (pReader->status.loadFromFile == false) {
+ return false;
+ }
+
+ code = readRowsCountFromFiles(pReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ return false;
+ }
+
+ code = readRowsCountFromMem(pReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ return false;
+ }
+
+ pBlock->info.rows = pReader->rowsNum;
+ pBlock->info.id.uid = 0;
+ pBlock->info.dataLoad = 0;
+
+ pReader->rowsNum = 0;
+
+ return pBlock->info.rows > 0;
+}
+
+static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) {
+ int32_t code = TSDB_CODE_SUCCESS;
+
// cleanup the data that belongs to the previous data block
SSDataBlock* pBlock = pReader->pResBlock;
blockDataCleanup(pBlock);
+ *hasNext = false;
+
SReaderStatus* pStatus = &pReader->status;
if (taosHashGetSize(pStatus->pTableMap) == 0) {
- return false;
+ return code;
+ }
+
+ if (READ_MODE_COUNT_ONLY == pReader->readMode) {
+ return tsdbReadRowsCountOnly(pReader);
}
if (pStatus->loadFromFile) {
- int32_t code = buildBlockFromFiles(pReader);
+ code = buildBlockFromFiles(pReader);
if (code != TSDB_CODE_SUCCESS) {
- return false;
+ return code;
}
- if (pBlock->info.rows > 0) {
- return true;
- } else {
+ if (pBlock->info.rows <= 0) {
resetTableListIndex(&pReader->status);
- buildBlockFromBufferSequentially(pReader);
- return pBlock->info.rows > 0;
+ code = buildBlockFromBufferSequentially(pReader);
}
} else { // no data in files, let's try the buffer
- buildBlockFromBufferSequentially(pReader);
- return pBlock->info.rows > 0;
+ code = buildBlockFromBufferSequentially(pReader);
}
+
+ *hasNext = pBlock->info.rows > 0;
+
+ return code;
}
-bool tsdbNextDataBlock(STsdbReader* pReader) {
+int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) {
+ int32_t code = TSDB_CODE_SUCCESS;
+
+ *hasNext = false;
+
if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT) {
- return false;
+ return code;
}
SReaderStatus* pStatus = &pReader->status;
- int32_t code = tsdbAcquireReader(pReader);
+ code = tsdbAcquireReader(pReader);
qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code);
if (pReader->suspended) {
@@ -4438,16 +4724,21 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
}
if (pReader->innerReader[0] != NULL && pReader->step == 0) {
- bool ret = doTsdbNextDataBlock(pReader->innerReader[0]);
+ code = doTsdbNextDataBlock(pReader->innerReader[0], hasNext);
+ if (code) {
+ tsdbReleaseReader(pReader);
+ return code;
+ }
+
pReader->step = EXTERNAL_ROWS_PREV;
- if (ret) {
+ if (*hasNext) {
pStatus = &pReader->innerReader[0]->status;
if (pStatus->composedDataBlock) {
qTrace("tsdb/read: %p, unlock read mutex", pReader);
tsdbReleaseReader(pReader);
}
- return ret;
+ return code;
}
}
@@ -4464,14 +4755,19 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
pReader->step = EXTERNAL_ROWS_MAIN;
}
- bool ret = doTsdbNextDataBlock(pReader);
- if (ret) {
+ code = doTsdbNextDataBlock(pReader, hasNext);
+ if (code != TSDB_CODE_SUCCESS) {
+ tsdbReleaseReader(pReader);
+ return code;
+ }
+
+ if (*hasNext) {
if (pStatus->composedDataBlock) {
qTrace("tsdb/read: %p, unlock read mutex", pReader);
tsdbReleaseReader(pReader);
}
- return ret;
+ return code;
}
if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) {
@@ -4483,23 +4779,28 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
return code;
}
- ret = doTsdbNextDataBlock(pReader->innerReader[1]);
+ code = doTsdbNextDataBlock(pReader->innerReader[1], hasNext);
+ if (code != TSDB_CODE_SUCCESS) {
+ tsdbReleaseReader(pReader);
+ return code;
+ }
+
pReader->step = EXTERNAL_ROWS_NEXT;
- if (ret) {
+ if (*hasNext) {
pStatus = &pReader->innerReader[1]->status;
if (pStatus->composedDataBlock) {
qTrace("tsdb/read: %p, unlock read mutex", pReader);
tsdbReleaseReader(pReader);
}
- return ret;
+ return code;
}
}
qTrace("tsdb/read: %p, unlock read mutex", pReader);
tsdbReleaseReader(pReader);
- return false;
+ return code;
}
static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) {
@@ -4507,6 +4808,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_
int32_t i = 0, j = 0;
int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg);
taosArrayInsert(pSup->pColAgg, 0, pTsAgg);
+ size++;
while (j < numOfCols && i < size) {
SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
@@ -4519,10 +4821,21 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_
if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) {
SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows};
taosArrayInsert(pSup->pColAgg, i, &nullColAgg);
+ i += 1;
+ size++;
}
j += 1;
}
}
+
+ while (j < numOfCols) {
+ if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) {
+ SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows};
+ taosArrayInsert(pSup->pColAgg, i, &nullColAgg);
+ i += 1;
+ }
+ j++;
+ }
}
int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) {
@@ -4602,8 +4915,8 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock,
} else if (pAgg->colId < pSup->colId[j]) {
i += 1;
} else if (pSup->colId[j] < pAgg->colId) {
- // ASSERT(pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID);
- pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg;
+ pResBlock->pBlockAgg[pSup->slotId[j]] = NULL;
+ *allHave = false;
j += 1;
}
}
@@ -4632,20 +4945,27 @@ STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, co
static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status;
+ int32_t code = TSDB_CODE_SUCCESS;
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter);
STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr);
if (pBlockScanInfo == NULL) {
return NULL;
}
- int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid);
+ code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid);
+ if (code != TSDB_CODE_SUCCESS) {
+ tBlockDataDestroy(&pStatus->fileBlockData);
+ terrno = code;
+ return NULL;
+ }
+
+ code = copyBlockDataToSDataBlock(pReader);
if (code != TSDB_CODE_SUCCESS) {
tBlockDataDestroy(&pStatus->fileBlockData);
terrno = code;
return NULL;
}
- copyBlockDataToSDataBlock(pReader);
return pReader->pResBlock;
}
@@ -4686,7 +5006,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
return TSDB_CODE_SUCCESS;
}
- SReaderStatus* pStatus = &pReader->status;
+ SReaderStatus* pStatus = &pReader->status;
SDataBlockIter* pBlockIter = &pStatus->blockIter;
pReader->order = pCond->order;
@@ -4707,9 +5027,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
resetDataBlockIterator(pBlockIter, pReader->order);
resetTableListIndex(&pReader->status);
- bool asc = ASCENDING_TRAVERSE(pReader->order);
- int32_t step = asc? 1:-1;
- int64_t ts = asc? pReader->window.skey - 1 : pReader->window.ekey + 1;
+ bool asc = ASCENDING_TRAVERSE(pReader->order);
+ int32_t step = asc ? 1 : -1;
+ int64_t ts = asc ? pReader->window.skey - 1 : pReader->window.ekey + 1;
resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step);
int32_t code = 0;
@@ -4768,7 +5088,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
pTableBlockInfo->defMinRows = pc->minRows;
pTableBlockInfo->defMaxRows = pc->maxRows;
- int32_t bucketRange = ceil(((pc->maxRows - pc->minRows)) / numOfBuckets);
+ int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBuckets);
pTableBlockInfo->numOfFiles += 1;
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 0779388ba9..09f1ca7877 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -156,13 +156,10 @@ int vnodeShouldCommit(SVnode *pVnode) {
bool needCommit = false;
taosThreadMutexLock(&pVnode->mutex);
- if (!pVnode->inUse || !diskAvail) {
- goto _out;
+ if (pVnode->inUse && diskAvail) {
+ needCommit =
+ ((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs));
}
- needCommit =
- (((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) ||
- (pVnode->inUse->size > 0 && pSched->commitMs + pSched->maxWaitMs < nowMs));
-_out:
taosThreadMutexUnlock(&pVnode->mutex);
return needCommit;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 71458acce2..052e4ab2c1 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -335,6 +335,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
// commit json
if (!rollback) {
+ pWriter->info.state.committed = pWriter->ever;
pVnode->config = pWriter->info.config;
pVnode->state = (SVState){.committed = pWriter->info.state.committed,
.applied = pWriter->info.state.committed,
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index b49ca70bfa..d681f5b65e 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -378,7 +378,7 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
return -1;
}
- int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg);
+ int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg);
if (code != 0) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c
index e6e375b555..fd6ee6c1b9 100644
--- a/source/libs/catalog/src/ctgUtil.c
+++ b/source/libs/catalog/src/ctgUtil.c
@@ -865,19 +865,19 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName
vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ);
-/*
- void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
- while (pIter) {
- vgInfo = pIter;
- if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
- taosHashCancelIterate(dbInfo->vgHash, pIter);
- break;
- }
+ /*
+ void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
+ while (pIter) {
+ vgInfo = pIter;
+ if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
+ taosHashCancelIterate(dbInfo->vgHash, pIter);
+ break;
+ }
- pIter = taosHashIterate(dbInfo->vgHash, pIter);
- vgInfo = NULL;
- }
-*/
+ pIter = taosHashIterate(dbInfo->vgHash, pIter);
+ vgInfo = NULL;
+ }
+ */
if (NULL == vgInfo) {
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db,
@@ -902,7 +902,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo*
CTG_ERR_RET(ctgMakeVgArray(dbInfo));
- int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
+ int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
if (vgNum <= 0) {
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
@@ -982,43 +982,43 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo*
CTG_RET(code);
}
-int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId) {
- int32_t code = 0;
- CTG_ERR_RET(ctgMakeVgArray(dbInfo));
+int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum,
+ int32_t* vgId) {
+ int32_t code = 0;
+ CTG_ERR_RET(ctgMakeVgArray(dbInfo));
- int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
+ int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
- if (vgNum <= 0) {
- ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
- CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
- }
+ if (vgNum <= 0) {
+ ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
+ CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
+ }
- SVgroupInfo* vgInfo = NULL;
- char tbFullName[TSDB_TABLE_FNAME_LEN];
- snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName);
- int32_t offset = strlen(tbFullName);
-
- for (int32_t i = 0; i < tbNum; ++i) {
- snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]);
- uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
- dbInfo->hashPrefix, dbInfo->hashSuffix);
+ SVgroupInfo* vgInfo = NULL;
+ char tbFullName[TSDB_TABLE_FNAME_LEN];
+ snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName);
+ int32_t offset = strlen(tbFullName);
- vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ);
- if (NULL == vgInfo) {
- ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName,
- (int32_t)taosArrayGetSize(dbInfo->vgArray));
- CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
- }
+ for (int32_t i = 0; i < tbNum; ++i) {
+ snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]);
+ uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
+ dbInfo->hashPrefix, dbInfo->hashSuffix);
- vgId[i] = vgInfo->vgId;
+ vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ);
+ if (NULL == vgInfo) {
+ ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName,
+ (int32_t)taosArrayGetSize(dbInfo->vgArray));
+ CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
+ }
- ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId);
- }
+ vgId[i] = vgInfo->vgId;
- CTG_RET(code);
+ ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId);
+ }
+
+ CTG_RET(code);
}
-
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) {
if (*(uint64_t*)key1 < ((SSTableVersion*)key2)->suid) {
return -1;
@@ -1063,26 +1063,25 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) {
if (NULL == dbInfo) {
return TSDB_CODE_SUCCESS;
}
-
+
if (dbInfo->vgHash && NULL == dbInfo->vgArray) {
dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo));
if (NULL == dbInfo->vgArray) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
-
- void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
+
+ void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
while (pIter) {
taosArrayPush(dbInfo->vgArray, pIter);
pIter = taosHashIterate(dbInfo->vgHash, pIter);
}
-
+
taosArraySort(dbInfo->vgArray, ctgVgInfoComp);
}
return TSDB_CODE_SUCCESS;
}
-
int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) {
CTG_ERR_RET(ctgMakeVgArray(src));
@@ -1425,15 +1424,15 @@ void ctgGetClusterCacheStat(SCatalog* pCtg) {
if (0 == (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_DB)) {
continue;
}
-
+
pCtg->cacheStat.cacheNum[i] = 0;
}
-
- SCtgDBCache *dbCache = NULL;
- void *pIter = taosHashIterate(pCtg->dbCache, NULL);
+
+ SCtgDBCache* dbCache = NULL;
+ void* pIter = taosHashIterate(pCtg->dbCache, NULL);
while (pIter) {
- dbCache = (SCtgDBCache *)pIter;
-
+ dbCache = (SCtgDBCache*)pIter;
+
for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) {
if (0 == (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_DB)) {
continue;
@@ -1446,33 +1445,33 @@ void ctgGetClusterCacheStat(SCatalog* pCtg) {
}
}
-void ctgSummaryClusterCacheStat(SCatalog *pCtg) {
+void ctgSummaryClusterCacheStat(SCatalog* pCtg) {
for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) {
if (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_GLOBAL) {
continue;
}
-
+
gCtgMgmt.statInfo.cache.cacheNum[i] += pCtg->cacheStat.cacheNum[i];
gCtgMgmt.statInfo.cache.cacheHit[i] += pCtg->cacheStat.cacheHit[i];
gCtgMgmt.statInfo.cache.cacheNHit[i] += pCtg->cacheStat.cacheNHit[i];
}
}
-void ctgGetGlobalCacheStat(SCtgCacheStat *pStat) {
+void ctgGetGlobalCacheStat(SCtgCacheStat* pStat) {
for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) {
if (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_GLOBAL) {
continue;
}
-
+
gCtgMgmt.statInfo.cache.cacheNum[i] = 0;
gCtgMgmt.statInfo.cache.cacheHit[i] = 0;
gCtgMgmt.statInfo.cache.cacheNHit[i] = 0;
}
-
- SCatalog *pCtg = NULL;
- void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
+
+ SCatalog* pCtg = NULL;
+ void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
while (pIter) {
- pCtg = *(SCatalog **)pIter;
+ pCtg = *(SCatalog**)pIter;
if (pCtg) {
ctgGetClusterCacheStat(pCtg);
@@ -1484,4 +1483,3 @@ void ctgGetGlobalCacheStat(SCtgCacheStat *pStat) {
memcpy(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache));
}
-
diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h
index f99c7de93d..9b8f034e44 100644
--- a/source/libs/executor/inc/executil.h
+++ b/source/libs/executor/inc/executil.h
@@ -107,6 +107,7 @@ int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t
uint64_t tableListGetSize(const STableListInfo* pTableList);
uint64_t tableListGetSuid(const STableListInfo* pTableList);
STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index);
+int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex);
size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
void initResultRowInfo(SResultRowInfo* pResultRowInfo);
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index fd610b5bcd..54a9ff0942 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -143,10 +143,7 @@ typedef struct {
SQueryTableDataCond tableCond;
int64_t fillHistoryVer1;
int64_t fillHistoryVer2;
-
- // int8_t triggerSaved;
- // int64_t deleteMarkSaved;
- SStreamState* pState;
+ SStreamState* pState;
} SStreamTaskInfo;
typedef struct {
@@ -168,26 +165,25 @@ typedef struct STaskStopInfo {
} STaskStopInfo;
struct SExecTaskInfo {
- STaskIdInfo id;
- uint32_t status;
- STimeWindow window;
- STaskCostInfo cost;
- int64_t owner; // if it is in execution
- int32_t code;
- int32_t qbufQuota; // total available buffer (in KB) during execution query
-
- int64_t version; // used for stream to record wal version, why not move to sschemainfo
+ STaskIdInfo id;
+ uint32_t status;
+ STimeWindow window;
+ STaskCostInfo cost;
+ int64_t owner; // if it is in execution
+ int32_t code;
+ int32_t qbufQuota; // total available buffer (in KB) during execution query
+ int64_t version; // used for stream to record wal version, why not move to sschemainfo
SStreamTaskInfo streamInfo;
SSchemaInfo schemaInfo;
- STableListInfo* pTableInfoList; // this is a table list
- const char* sql; // query sql string
- jmp_buf env; // jump to this position when error happens.
- EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model]
+ const char* sql; // query sql string
+ jmp_buf env; // jump to this position when error happens.
+ EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model]
SSubplan* pSubplan;
struct SOperatorInfo* pRoot;
SLocalFetch localFetch;
SArray* pResultBlockList; // result block list
STaskStopInfo stopInfo;
+ SRWLatch lock; // secure the access of STableListInfo
};
enum {
@@ -326,6 +322,8 @@ typedef struct STableScanBase {
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
int32_t dataBlockLoadFlag;
SLimitInfo limitInfo;
+ // there are more than one table list exists in one task, if only one vnode exists.
+ STableListInfo* pTableListInfo;
} STableScanBase;
typedef struct STableScanInfo {
@@ -339,6 +337,7 @@ typedef struct STableScanInfo {
int8_t scanMode;
int8_t assignBlockUid;
bool hasGroupByTag;
+ bool countOnly;
} STableScanInfo;
typedef struct STableMergeScanInfo {
@@ -365,11 +364,12 @@ typedef struct STableMergeScanInfo {
} STableMergeScanInfo;
typedef struct STagScanInfo {
- SColumnInfo* pCols;
- SSDataBlock* pRes;
- SColMatchInfo matchInfo;
- int32_t curPos;
- SReadHandle readHandle;
+ SColumnInfo* pCols;
+ SSDataBlock* pRes;
+ SColMatchInfo matchInfo;
+ int32_t curPos;
+ SReadHandle readHandle;
+ STableListInfo* pTableListInfo;
} STagScanInfo;
typedef enum EStreamScanMode {
@@ -485,16 +485,11 @@ typedef struct SStreamScanInfo {
} SStreamScanInfo;
typedef struct {
- // int8_t subType;
- // bool withMeta;
- // int64_t suid;
- // int64_t snapVersion;
- // void *metaInfo;
- // void *dataInfo;
- SVnode* vnode;
- SSDataBlock pRes; // result SSDataBlock
- STsdbReader* dataReader;
- SSnapContext* sContext;
+ SVnode* vnode;
+ SSDataBlock pRes; // result SSDataBlock
+ STsdbReader* dataReader;
+ SSnapContext* sContext;
+ STableListInfo* pTableListInfo;
} SStreamRawScanInfo;
typedef struct STableCountScanSupp {
@@ -523,6 +518,16 @@ typedef struct SOptrBasicInfo {
bool mergeResultBlock;
} SOptrBasicInfo;
+typedef struct SAggOperatorInfo {
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
+ STableQueryInfo* current;
+ uint64_t groupId;
+ SGroupResInfo groupResInfo;
+ SExprSupp scalarExprSup;
+ bool groupKeyOptimized;
+} SAggOperatorInfo;
+
typedef struct SIntervalAggOperatorInfo {
SOptrBasicInfo binfo; // basic info
SAggSupporter aggSup; // aggregate supporter
@@ -559,6 +564,7 @@ typedef struct SStreamIntervalOperatorInfo {
STimeWindowAggSupp twAggSup;
bool invertible;
bool ignoreExpiredData;
+ bool ignoreExpiredDataSaved;
SArray* pDelWins; // SWinRes
int32_t delIndex;
SSDataBlock* pDelRes;
@@ -620,6 +626,7 @@ typedef struct SStreamSessionAggOperatorInfo {
SPhysiNode* pPhyNode; // create new child
bool isFinal;
bool ignoreExpiredData;
+ bool ignoreExpiredDataSaved;
SArray* pUpdated;
SSHashObj* pStUpdated;
} SStreamSessionAggOperatorInfo;
@@ -637,6 +644,7 @@ typedef struct SStreamStateAggOperatorInfo {
void* pDelIterator;
SArray* pChildren; // cache for children's result;
bool ignoreExpiredData;
+ bool ignoreExpiredDataSaved;
SArray* pUpdated;
SSHashObj* pSeUpdated;
} SStreamStateAggOperatorInfo;
@@ -690,6 +698,9 @@ typedef struct SStreamFillOperatorInfo {
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED)
+SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model,
+ char* dbFName);
+
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain);
int32_t optrDummyOpenFn(SOperatorInfo* pOperator);
@@ -734,8 +745,9 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int3
STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order);
-int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder);
-int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
+SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id);
+int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder);
+int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
extern void doDestroyExchangeOperatorInfo(void* param);
@@ -756,11 +768,11 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
// clang-format off
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableList, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
@@ -776,7 +788,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
@@ -790,9 +802,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
@@ -837,9 +849,12 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
char* buildTaskId(uint64_t taskId, uint64_t queryId);
+SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo);
+
int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
- int32_t vgId, char* sql, EOPTR_EXEC_MODEL model);
-int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
+ int32_t vgId, char* sql, EOPTR_EXEC_MODEL model);
+int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* pTableListInfo,
+ SReadHandle* readHandle);
int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
@@ -853,7 +868,6 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap);
bool functionNeedToExecute(SqlFunctionCtx* pCtx);
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
-bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup);
void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
uint64_t* pGp, void* pTbName);
diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h
index fad4059515..726f0df1e8 100644
--- a/source/libs/executor/inc/tfill.h
+++ b/source/libs/executor/inc/tfill.h
@@ -92,8 +92,8 @@ typedef struct SResultRowData {
typedef struct SStreamFillLinearInfo {
TSKEY nextEnd;
- SArray* pDeltaVal; // double. value for Fill(linear).
- SArray* pNextDeltaVal; // double. value for Fill(linear).
+ SArray* pEndPoints;
+ SArray* pNextEndPoints;
int64_t winIndex;
bool hasNext;
} SStreamFillLinearInfo;
diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c
new file mode 100644
index 0000000000..a26e3ace7b
--- /dev/null
+++ b/source/libs/executor/src/aggregateoperator.c
@@ -0,0 +1,563 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "filter.h"
+#include "function.h"
+#include "functionMgt.h"
+#include "os.h"
+#include "querynodes.h"
+#include "tfill.h"
+#include "tname.h"
+
+#include "tdatablock.h"
+#include "tglobal.h"
+#include "tmsg.h"
+#include "ttime.h"
+
+#include "executorimpl.h"
+#include "index.h"
+#include "query.h"
+#include "tcompare.h"
+#include "thash.h"
+#include "ttypes.h"
+#include "vnode.h"
+
+typedef struct {
+ bool hasAgg;
+ int32_t numOfRows;
+ int32_t startOffset;
+} SFunctionCtxStatus;
+
+static void destroyAggOperatorInfo(void* param);
+static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId);
+
+static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock);
+static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock);
+
+static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator);
+static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx);
+static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator);
+
+static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize,
+ const char* pKey);
+
+static int32_t addNewResultRowBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size);
+
+static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId);
+
+static void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus);
+static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus);
+
+SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode,
+ SExecTaskInfo* pTaskInfo) {
+ SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo));
+ SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+ if (pInfo == NULL || pOperator == NULL) {
+ goto _error;
+ }
+
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
+ initBasicInfo(&pInfo->binfo, pResBlock);
+
+ size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
+ initResultSizeInfo(&pOperator->resultInfo, 4096);
+
+ int32_t num = 0;
+ SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
+ int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
+ pTaskInfo->streamInfo.pState);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
+ int32_t numOfScalarExpr = 0;
+ SExprInfo* pScalarExprInfo = NULL;
+ if (pAggNode->pExprs != NULL) {
+ pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr);
+ }
+
+ code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
+ code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
+ pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock;
+ pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized;
+ pInfo->groupId = UINT64_MAX;
+
+ setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo,
+ pTaskInfo);
+ pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo,
+ optrDefaultBufFn, NULL);
+
+ if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
+ STableScanInfo* pTableScanInfo = downstream->info;
+ pTableScanInfo->base.pdInfo.pExprSup = &pOperator->exprSupp;
+ pTableScanInfo->base.pdInfo.pAggSup = &pInfo->aggSup;
+ }
+
+ code = appendDownstream(pOperator, &downstream, 1);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
+ return pOperator;
+
+ _error:
+ if (pInfo != NULL) {
+ destroyAggOperatorInfo(pInfo);
+ }
+
+ if (pOperator != NULL) {
+ cleanupExprSupp(&pOperator->exprSupp);
+ }
+
+ taosMemoryFreeClear(pOperator);
+ pTaskInfo->code = code;
+ return NULL;
+}
+
+void destroyAggOperatorInfo(void* param) {
+ SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param;
+ cleanupBasicInfo(&pInfo->binfo);
+
+ cleanupAggSup(&pInfo->aggSup);
+ cleanupExprSupp(&pInfo->scalarExprSup);
+ cleanupGroupResInfo(&pInfo->groupResInfo);
+ taosMemoryFreeClear(param);
+}
+
+// this is a blocking operator
+int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
+ if (OPTR_IS_OPENED(pOperator)) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SAggOperatorInfo* pAggInfo = pOperator->info;
+
+ SExprSupp* pSup = &pOperator->exprSupp;
+ SOperatorInfo* downstream = pOperator->pDownstream[0];
+
+ int64_t st = taosGetTimestampUs();
+
+ int32_t order = TSDB_ORDER_ASC;
+ int32_t scanFlag = MAIN_SCAN;
+
+ bool hasValidBlock = false;
+ bool blockAllocated = false;
+
+ while (1) {
+ SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
+ if (pBlock == NULL) {
+ if (!hasValidBlock) {
+ createDataBlockForEmptyInput(pOperator, &pBlock);
+ if (pBlock == NULL) {
+ break;
+ }
+ blockAllocated = true;
+ } else {
+ break;
+ }
+ }
+ hasValidBlock = true;
+
+ int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false);
+ if (code != TSDB_CODE_SUCCESS) {
+ destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
+ T_LONG_JMP(pTaskInfo->env, code);
+ }
+
+ // there is an scalar expression that needs to be calculated before apply the group aggregation.
+ if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) {
+ SExprSupp* pSup1 = &pAggInfo->scalarExprSup;
+ code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL);
+ if (code != TSDB_CODE_SUCCESS) {
+ destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
+ T_LONG_JMP(pTaskInfo->env, code);
+ }
+ }
+
+ // the pDataBlock are always the same one, no need to call this again
+ setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId);
+ setInputDataBlock(pSup, pBlock, order, scanFlag, true);
+ code = doAggregateImpl(pOperator, pSup->pCtx);
+ if (code != 0) {
+ destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
+ T_LONG_JMP(pTaskInfo->env, code);
+ }
+
+ destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
+ }
+
+ // the downstream operator may return with error code, so let's check the code before generating results.
+ if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
+ T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
+ }
+
+ initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0);
+ OPTR_SET_OPENED(pOperator);
+
+ pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
+ return pTaskInfo->code;
+}
+
+SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
+ SAggOperatorInfo* pAggInfo = pOperator->info;
+ SOptrBasicInfo* pInfo = &pAggInfo->binfo;
+
+ if (pOperator->status == OP_EXEC_DONE) {
+ return NULL;
+ }
+
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
+ if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
+ setOperatorCompleted(pOperator);
+ return NULL;
+ }
+
+ blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+ while (1) {
+ doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf);
+ doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
+
+ if (!hasRemainResults(&pAggInfo->groupResInfo)) {
+ setOperatorCompleted(pOperator);
+ break;
+ }
+
+ if (pInfo->pRes->info.rows > 0) {
+ break;
+ }
+ }
+
+ size_t rows = blockDataGetNumOfRows(pInfo->pRes);
+ pOperator->resultInfo.totalRows += rows;
+
+ return (rows == 0) ? NULL : pInfo->pRes;
+}
+
+int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) {
+ for (int32_t k = 0; k < pOperator->exprSupp.numOfExprs; ++k) {
+ if (functionNeedToExecute(&pCtx[k])) {
+ // todo add a dummy funtion to avoid process check
+ if (pCtx[k].fpSet.process == NULL) {
+ continue;
+ }
+
+ int32_t code = pCtx[k].fpSet.process(&pCtx[k]);
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code));
+ return code;
+ }
+ }
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) {
+ if (!tsCountAlwaysReturnValue) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SAggOperatorInfo* pAggInfo = pOperator->info;
+ if (pAggInfo->groupKeyOptimized) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SOperatorInfo* downstream = pOperator->pDownstream[0];
+ if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION ||
+ (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN &&
+ ((STableScanInfo*)downstream->info)->hasGroupByTag == true)) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
+ bool hasCountFunc = false;
+
+ for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
+ const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
+ if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) ||
+ (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) {
+ hasCountFunc = true;
+ break;
+ }
+ }
+
+ if (!hasCountFunc) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SSDataBlock* pBlock = createDataBlock();
+ pBlock->info.rows = 1;
+ pBlock->info.capacity = 0;
+
+ for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
+ SColumnInfoData colInfo = {0};
+ colInfo.hasNull = true;
+ colInfo.info.type = TSDB_DATA_TYPE_NULL;
+ colInfo.info.bytes = 1;
+
+ SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i];
+ for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) {
+ SFunctParam* pFuncParam = &pOneExpr->base.pParam[j];
+ if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) {
+ int32_t slotId = pFuncParam->pCol->slotId;
+ int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+ if (slotId >= numOfCols) {
+ taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1);
+ for (int32_t k = numOfCols; k < slotId + 1; ++k) {
+ taosArrayPush(pBlock->pDataBlock, &colInfo);
+ }
+ }
+ } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
+ // do nothing
+ }
+ }
+ }
+
+ blockDataEnsureCapacity(pBlock, pBlock->info.rows);
+ *ppBlock = pBlock;
+
+ return TSDB_CODE_SUCCESS;
+}
+
+void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) {
+ if (!blockAllocated) {
+ return;
+ }
+
+ blockDataDestroy(*ppBlock);
+ *ppBlock = NULL;
+}
+
+void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) {
+ SAggOperatorInfo* pAggInfo = pOperator->info;
+ if (pAggInfo->groupId != UINT64_MAX && pAggInfo->groupId == groupId) {
+ return;
+ }
+
+ doSetTableGroupOutputBuf(pOperator, numOfOutput, groupId);
+
+ // record the current active group id
+ pAggInfo->groupId = groupId;
+}
+
+void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) {
+ // for simple group by query without interval, all the tables belong to one group result.
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SAggOperatorInfo* pAggInfo = pOperator->info;
+
+ SResultRowInfo* pResultRowInfo = &pAggInfo->binfo.resultRowInfo;
+ SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
+ int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset;
+
+ SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId,
+ sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup, true);
+ /*
+ * not assign result buffer yet, add new result buffer
+ * all group belong to one result set, and each group result has different group id so set the id to be one
+ */
+ if (pResultRow->pageId == -1) {
+ int32_t ret = addNewResultRowBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize);
+ if (ret != TSDB_CODE_SUCCESS) {
+ T_LONG_JMP(pTaskInfo->env, terrno);
+ }
+ }
+
+ setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
+}
+
+// a new buffer page for each table. Needs to opt this design
+int32_t addNewResultRowBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size) {
+ if (pWindowRes->pageId != -1) {
+ return 0;
+ }
+
+ SFilePage* pData = NULL;
+
+ // in the first scan, new space needed for results
+ int32_t pageId = -1;
+ SArray* list = getDataBufPagesIdList(pResultBuf);
+
+ if (taosArrayGetSize(list) == 0) {
+ pData = getNewBufPage(pResultBuf, &pageId);
+ pData->num = sizeof(SFilePage);
+ } else {
+ SPageInfo* pi = getLastPageInfo(list);
+ pData = getBufPage(pResultBuf, getPageId(pi));
+ if (pData == NULL) {
+ qError("failed to get buffer, code:%s", tstrerror(terrno));
+ return terrno;
+ }
+
+ pageId = getPageId(pi);
+
+ if (pData->num + size > getBufPageSize(pResultBuf)) {
+ // release current page first, and prepare the next one
+ releaseBufPageInfo(pResultBuf, pi);
+
+ pData = getNewBufPage(pResultBuf, &pageId);
+ if (pData != NULL) {
+ pData->num = sizeof(SFilePage);
+ }
+ }
+ }
+
+ if (pData == NULL) {
+ return -1;
+ }
+
+ // set the number of rows in current disk page
+ if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer
+ pWindowRes->pageId = pageId;
+ pWindowRes->offset = (int32_t)pData->num;
+
+ pData->num += size;
+ }
+
+ return 0;
+}
+
+int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize,
+ const char* pKey) {
+ int32_t code = 0;
+ // _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+
+ pAggSup->currentPageId = -1;
+ pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
+ pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t));
+ pAggSup->pResultRowHashTable = tSimpleHashInit(100, taosFastHash);
+
+ if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ uint32_t defaultPgsz = 0;
+ uint32_t defaultBufsz = 0;
+ getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz);
+
+ if (!osTempSpaceAvailable()) {
+ code = TSDB_CODE_NO_AVAIL_DISK;
+ qError("Init stream agg supporter failed since %s, %s", terrstr(code), pKey);
+ return code;
+ }
+
+ code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir);
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("Create agg result buf failed since %s, %s", tstrerror(code), pKey);
+ return code;
+ }
+
+ return code;
+}
+
+void cleanupAggSup(SAggSupporter* pAggSup) {
+ taosMemoryFreeClear(pAggSup->keyBuf);
+ tSimpleHashCleanup(pAggSup->pResultRowHashTable);
+ destroyDiskbasedBuf(pAggSup->pResultBuf);
+}
+
+int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
+ const char* pkey, void* pState) {
+ int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+
+ code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ if (pState) {
+ pSup->pCtx[i].saveHandle.pBuf = NULL;
+ pSup->pCtx[i].saveHandle.pState = pState;
+ pSup->pCtx[i].exprIdx = i;
+ } else {
+ pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf;
+ }
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData,
+ int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) {
+ for (int32_t k = 0; k < numOfOutput; ++k) {
+ // keep it temporarily
+ SFunctionCtxStatus status = {0};
+ functionCtxSave(&pCtx[k], &status);
+
+ pCtx[k].input.startRowIndex = offset;
+ pCtx[k].input.numOfRows = forwardStep;
+
+ // not a whole block involved in query processing, statistics data can not be used
+ // NOTE: the original value of isSet have been changed here
+ if (pCtx[k].input.colDataSMAIsSet && forwardStep < numOfTotal) {
+ pCtx[k].input.colDataSMAIsSet = false;
+ }
+
+ if (pCtx[k].isPseudoFunc) {
+ SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]);
+
+ char* p = GET_ROWCELL_INTERBUF(pEntryInfo);
+
+ SColumnInfoData idata = {0};
+ idata.info.type = TSDB_DATA_TYPE_BIGINT;
+ idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
+ idata.pData = p;
+
+ SScalarParam out = {.columnData = &idata};
+ SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
+ pCtx[k].sfp.process(&tw, 1, &out);
+ pEntryInfo->numOfRes = 1;
+ } else {
+ int32_t code = TSDB_CODE_SUCCESS;
+ if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) {
+ code = pCtx[k].fpSet.process(&pCtx[k]);
+
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code));
+ taskInfo->code = code;
+ T_LONG_JMP(taskInfo->env, code);
+ }
+ }
+
+ // restore it
+ functionCtxRestore(&pCtx[k], &status);
+ }
+ }
+}
+
+void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
+ pStatus->hasAgg = pCtx->input.colDataSMAIsSet;
+ pStatus->numOfRows = pCtx->input.numOfRows;
+ pStatus->startOffset = pCtx->input.startRowIndex;
+}
+
+void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
+ pCtx->input.colDataSMAIsSet = pStatus->hasAgg;
+ pCtx->input.numOfRows = pStatus->numOfRows;
+ pCtx->input.startRowIndex = pStatus->startOffset;
+}
\ No newline at end of file
diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c
index d42b348fd8..f6fc332b37 100644
--- a/source/libs/executor/src/cachescanoperator.c
+++ b/source/libs/executor/src/cachescanoperator.c
@@ -37,6 +37,7 @@ typedef struct SCacheRowsScanInfo {
SSDataBlock* pBufferredRes;
SArray* pUidList;
int32_t indexOfBufferedRes;
+ STableListInfo* pTableList;
} SCacheRowsScanInfo;
static SSDataBlock* doScanCache(SOperatorInfo* pOperator);
@@ -47,15 +48,17 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM
#define SCAN_ROW_TYPE(_t) ((_t)? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)
SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
- SExecTaskInfo* pTaskInfo) {
+ STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
+ tableListDestroy(pTableListInfo);
goto _error;
}
+ pInfo->pTableList = pTableListInfo;
pInfo->readHandle = *readHandle;
SDataBlockDescNode* pDescNode = pScanNode->scan.node.pOutputDataBlockDesc;
@@ -75,20 +78,18 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
goto _error;
}
- STableListInfo* pTableList = pTaskInfo->pTableInfoList;
-
- int32_t totalTables = tableListGetSize(pTableList);
+ int32_t totalTables = tableListGetSize(pTableListInfo);
int32_t capacity = 0;
pInfo->pUidList = taosArrayInit(4, sizeof(int64_t));
// partition by tbname
- if (oneTableForEachGroup(pTableList) || (totalTables == 1)) {
+ if (oneTableForEachGroup(pTableListInfo) || (totalTables == 1)) {
pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL | SCAN_ROW_TYPE(pScanNode->ignoreNull);
- STableKeyInfo* pList = tableListGetInfo(pTableList, 0);
+ STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
- uint64_t suid = tableListGetSuid(pTableList);
+ uint64_t suid = tableListGetSuid(pTableListInfo);
code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
@@ -136,7 +137,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
SCacheRowsScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- STableListInfo* pTableList = pTaskInfo->pTableInfoList;
+ STableListInfo* pTableList = pInfo->pTableList;
uint64_t suid = tableListGetSuid(pTableList);
int32_t size = tableListGetSize(pTableList);
@@ -281,6 +282,7 @@ void destroyCacheScanOperator(void* param) {
taosMemoryFree(pInfo->pSlotIds);
taosArrayDestroy(pInfo->pUidList);
taosArrayDestroy(pInfo->matchInfo.pList);
+ tableListDestroy(pInfo->pTableList);
if (pInfo->pLastrowReader != NULL) {
pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index 3bdecbe748..c855a104b2 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -70,7 +70,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
tsem_wait(&pExchangeInfo->ready);
if (isTaskKilled(pTaskInfo)) {
- longjmp(pTaskInfo->env, pTaskInfo->code);
+ T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
for (int32_t i = 0; i < totalSources; ++i) {
@@ -212,6 +212,11 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) {
return NULL;
}
+ doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL);
+ if (blockDataGetNumOfRows(pBlock) == 0) {
+ continue;
+ }
+
SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo;
if (hasLimitOffsetInfo(pLimitInfo)) {
int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false);
@@ -303,6 +308,11 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
pTaskInfo);
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock);
+ code = filterInitFromNode((SNode*)pExNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
pOperator->fpSet =
createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL);
return pOperator;
@@ -566,7 +576,7 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) {
pOperator->status = OP_RES_TO_RETURN;
pOperator->cost.openCost = taosGetTimestampUs() - startTs;
if (isTaskKilled(pTaskInfo)) {
- longjmp(pTaskInfo->env, pTaskInfo->code);
+ T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
return TSDB_CODE_SUCCESS;
@@ -619,7 +629,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current);
tsem_wait(&pExchangeInfo->ready);
if (isTaskKilled(pTaskInfo)) {
- longjmp(pTaskInfo->env, pTaskInfo->code);
+ T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index dce358ab6d..7d318786ba 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -34,7 +34,7 @@ struct STableListInfo {
int32_t numOfOuputGroups; // the data block will be generated one by one
int32_t* groupOffset; // keep the offset value for each group in the tableList
SArray* pTableList;
- SHashObj* map; // speedup acquire the tableQueryInfo by table uid
+ SHashObj* map; // speedup acquire the tableQueryInfo by table uid
uint64_t suid;
};
@@ -571,6 +571,10 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
memcpy(pStart, data, len);
pStart += len;
} else if (IS_VAR_DATA_TYPE(pValue->info.type)) {
+ if (varDataTLen(data) > pValue->info.bytes) {
+ code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
+ goto end;
+ }
memcpy(pStart, data, varDataTLen(data));
pStart += varDataTLen(data);
} else {
@@ -1800,6 +1804,21 @@ STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index)
return taosArrayGet(pTableList->pTableList, index);
}
+int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex) {
+ int32_t numOfTables = taosArrayGetSize(pTableList->pTableList);
+ if (startIndex >= numOfTables) {
+ return -1;
+ }
+
+ for (int32_t i = startIndex; i < numOfTables; ++i) {
+ STableKeyInfo* p = taosArrayGet(pTableList->pTableList, i);
+ if (p->uid == uid) {
+ return i;
+ }
+ }
+ return -1;
+}
+
uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) {
int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid));
ASSERT(pTableList->map != NULL && slot != NULL);
@@ -2042,7 +2061,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
if (!pBlock || pBlock->info.rows == 0) {
- qDebug("===stream===printDataBlock: Block is Null or Empty");
+ qDebug("===stream===%s: Block is Null or Empty", flag);
return;
}
char* pBuf = NULL;
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 86fcd319c5..6e05ea0d4f 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -29,8 +29,8 @@ static void cleanupRefPool() {
taosCloseRef(ref);
}
-static void initRefPool() {
- exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo);
+static void initRefPool() {
+ exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo);
atexit(cleanupRefPool);
}
@@ -242,30 +242,28 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
return code;
}
-qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, SSchemaWrapper** pSchema) {
- if (msg == NULL) {
- // create raw scan
- SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
+qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols,
+ uint64_t id) {
+ if (msg == NULL) { // create raw scan
+ SExecTaskInfo* pTaskInfo = doCreateExecTaskInfo(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, "");
if (NULL == pTaskInfo) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
-
- setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
-
- pTaskInfo->cost.created = taosGetTimestampUs();
- pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE;
pTaskInfo->pRoot = createRawScanOperatorInfo(pReaderHandle, pTaskInfo);
if (NULL == pTaskInfo->pRoot) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pTaskInfo);
return NULL;
}
+
+ qDebug("create raw scan task info completed, vgId:%d, %s", vgId, GET_TASKID(pTaskInfo));
return pTaskInfo;
}
struct SSubplan* pPlan = NULL;
- int32_t code = qStringToSubplan(msg, &pPlan);
+
+ int32_t code = qStringToSubplan(msg, &pPlan);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return NULL;
@@ -292,9 +290,6 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3
}
}
- if (pSchema) {
- *pSchema = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw);
- }
return pTaskInfo;
}
@@ -409,7 +404,8 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
}
}
- STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
+ STableListInfo* pTableListInfo = ((STableScanInfo*)pScanInfo->pTableScanOp->info)->base.pTableListInfo;
+ taosWLockLatch(&pTaskInfo->lock);
for (int32_t i = 0; i < numOfQualifiedTables; ++i) {
uint64_t* uid = taosArrayGet(qa, i);
@@ -424,6 +420,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(keyBuf);
taosArrayDestroy(qa);
+ taosWUnLockLatch(&pTaskInfo->lock);
return code;
}
}
@@ -445,6 +442,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
tableListAddTableInfo(pTableListInfo, keyInfo.uid, keyInfo.groupId);
}
+ taosWUnLockLatch(&pTaskInfo->lock);
if (keyBuf != NULL) {
taosMemoryFree(keyBuf);
}
@@ -452,7 +450,9 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
taosArrayDestroy(qa);
} else { // remove the table id in current list
qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList));
+ taosWLockLatch(&pTaskInfo->lock);
code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList);
+ taosWUnLockLatch(&pTaskInfo->lock);
}
return code;
@@ -485,9 +485,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan,
qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model) {
- assert(pSubplan != NULL);
SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
-
taosThreadOnce(&initPoolOnce, initRefPool);
qDebug("start to create task, TID:0x%" PRIx64 " QID:0x%" PRIx64 ", vgId:%d", taskId, pSubplan->id.queryId, vgId);
@@ -507,7 +505,12 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
if (handle) {
void* pSinkParam = NULL;
- code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle);
+
+ SArray* pInfoList = getTableListInfo(*pTask);
+ STableListInfo* pTableListInfo = taosArrayGetP(pInfoList, 0);
+ taosArrayDestroy(pInfoList);
+
+ code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTableListInfo, readHandle);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to createDataSinkParam, vgId:%d, code:%s, %s", vgId, tstrerror(code), (*pTask)->id.str);
goto _error;
@@ -749,6 +752,23 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) {
return TSDB_CODE_SUCCESS;
}
+int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) {
+ SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+ if (pTaskInfo == NULL) {
+ return TSDB_CODE_QRY_INVALID_QHANDLE;
+ }
+
+ qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo));
+ setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED);
+
+ while (qTaskIsExecuting(pTaskInfo)) {
+ taosMsleep(10);
+ }
+
+ pTaskInfo->code = rspCode;
+ return TSDB_CODE_SUCCESS;
+}
+
bool qTaskIsExecuting(qTaskInfo_t qinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
if (NULL == pTaskInfo) {
@@ -888,7 +908,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) {
pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark;
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
pInfo->twAggSup.deleteMark = INT64_MAX;
-
+ pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
+ pInfo->ignoreExpiredData = false;
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
@@ -904,6 +925,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) {
pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark;
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
pInfo->twAggSup.deleteMark = INT64_MAX;
+ pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
+ pInfo->ignoreExpiredData = false;
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) {
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||
@@ -917,6 +940,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) {
pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark;
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
pInfo->twAggSup.deleteMark = INT64_MAX;
+ pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
+ pInfo->ignoreExpiredData = false;
}
// iterate operator tree
@@ -944,35 +969,23 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) {
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
- /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);*/
- /*ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX);*/
-
pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved;
pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved;
- /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||*/
- /*pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);*/
+ pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved;
qInfo("restore stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark);
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
- /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);*/
- /*ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX);*/
-
pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved;
pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved;
- /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||*/
- /*pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);*/
+ pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved;
qInfo("restore stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark);
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) {
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
- /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);*/
- /*ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX);*/
-
pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved;
pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved;
- /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||*/
- /*pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);*/
+ pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved;
qInfo("restore stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark);
}
@@ -990,6 +1003,7 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) {
}
return 0;
}
+
bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
return pTaskInfo->streamInfo.recoverScanFinished;
@@ -1059,7 +1073,7 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
- if((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)){
+ if ((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)) {
qError("qStreamSetScanMemData err:%d,%p", pTaskInfo->execModel, pTaskInfo->streamInfo.submit.msgStr);
return -1;
}
@@ -1072,6 +1086,8 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) {
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SOperatorInfo* pOperator = pTaskInfo->pRoot;
+ const char* id = GET_TASKID(pTaskInfo);
+
pTaskInfo->streamInfo.prepareStatus = *pOffset;
pTaskInfo->streamInfo.returned = 0;
@@ -1081,24 +1097,20 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
if (subType == TOPIC_SUB_TYPE__COLUMN) {
pOperator->status = OP_OPENED;
-
- // TODO add more check
- if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
- if(pOperator->numOfDownstream != 1){
- qError("pOperator->numOfDownstream != 1:%d", pOperator->numOfDownstream);
- return -1;
- }
- pOperator = pOperator->pDownstream[0];
- }
+ pOperator = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id);
SStreamScanInfo* pInfo = pOperator->info;
+ STableScanInfo* pScanInfo = pInfo->pTableScanOp->info;
+ STableScanBase* pScanBaseInfo = &pScanInfo->base;
+ STableListInfo* pTableListInfo = pScanBaseInfo->pTableListInfo;
+
if (pOffset->type == TMQ_OFFSET__LOG) {
- STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
- tsdbReaderClose(pTSInfo->base.dataReader);
- pTSInfo->base.dataReader = NULL;
+ tsdbReaderClose(pScanBaseInfo->dataReader);
+ pScanBaseInfo->dataReader = NULL;
+
// let's seek to the next version in wal file
if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) {
- qError("tqSeekVer failed ver:%" PRId64, pOffset->version + 1);
+ qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
return -1;
}
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
@@ -1106,127 +1118,138 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
// those data are from the snapshot in tsdb, besides the data in the wal file.
int64_t uid = pOffset->uid;
int64_t ts = pOffset->ts;
+ int32_t index = 0;
+
+ // this value may be changed if new tables are created
+ taosRLockLatch(&pTaskInfo->lock);
+ int32_t numOfTables = tableListGetSize(pTableListInfo);
if (uid == 0) {
- if (tableListGetSize(pTaskInfo->pTableInfoList) != 0) {
- STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
+ if (numOfTables != 0) {
+ STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0);
uid = pTableInfo->uid;
ts = INT64_MIN;
+ pScanInfo->currentTable = 0;
} else {
- qError("uid == 0 and tablelist size is 0");
+ taosRUnLockLatch(&pTaskInfo->lock);
+ qError("no table in table list, %s", id);
return -1;
}
}
- /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/
- /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/
- STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
- int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList);
-
- qDebug("switch to table uid:%" PRId64 " ts:%" PRId64 "% "PRId64 " rows returned", uid, ts, pInfo->pTableScanOp->resultInfo.totalRows);
pInfo->pTableScanOp->resultInfo.totalRows = 0;
- bool found = false;
- for (int32_t i = 0; i < numOfTables; i++) {
- STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, i);
- if (pTableInfo->uid == uid) {
- found = true;
- pTableScanInfo->currentTable = i;
- break;
- }
- }
+ // start from current accessed position
+ // we cannot start from the pScanInfo->currentTable, since the commit offset may cause the rollback of the start
+ // position, let's find it from the beginning.
+ index = tableListFind(pTableListInfo, uid, 0);
+ taosRUnLockLatch(&pTaskInfo->lock);
- // TODO after dropping table, table may not found
- if(!found){
- qError("uid not found in tablelist %" PRId64, uid);
+ if (index >= 0) {
+ pScanInfo->currentTable = index;
+ } else {
+ qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid,
+ numOfTables, pScanInfo->currentTable, id);
return -1;
}
- if (pTableScanInfo->base.dataReader == NULL) {
- STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
- int32_t num = tableListGetSize(pTaskInfo->pTableInfoList);
+ STableKeyInfo keyInfo = {.uid = uid};
+ int64_t oldSkey = pScanBaseInfo->cond.twindows.skey;
- if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num,
- pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 ||
- pTableScanInfo->base.dataReader == NULL) {
- qError("tsdbReaderOpen failed. uid:%" PRIi64, pOffset->uid);
+ // let's start from the next ts that returned to consumer.
+ pScanBaseInfo->cond.twindows.skey = ts + 1;
+ pScanInfo->scanTimes = 0;
+
+ if (pScanBaseInfo->dataReader == NULL) {
+ int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1,
+ pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false);
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id);
+ terrno = code;
return -1;
}
+
+ qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s",
+ uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
+ } else {
+ tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
+ tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
+ qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s",
+ uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
}
- STableKeyInfo tki = {.uid = uid};
- tsdbSetTableList(pTableScanInfo->base.dataReader, &tki, 1);
- int64_t oldSkey = pTableScanInfo->base.cond.twindows.skey;
- pTableScanInfo->base.cond.twindows.skey = ts + 1;
- tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
- pTableScanInfo->base.cond.twindows.skey = oldSkey;
- pTableScanInfo->scanTimes = 0;
-
- qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid,
- ts, pTableScanInfo->currentTable, numOfTables);
+ // restore the key value
+ pScanBaseInfo->cond.twindows.skey = oldSkey;
} else {
- qError("invalid pOffset->type:%d", pOffset->type);
- return -1;
- }
- } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
- SStreamRawScanInfo* pInfo = pOperator->info;
- SSnapContext* sContext = pInfo->sContext;
- if (setForSnapShot(sContext, pOffset->uid) != 0) {
- qError("setDataForSnapShot error. uid:%" PRIi64, pOffset->uid);
+ qError("invalid pOffset->type:%d, %s", pOffset->type, id);
return -1;
}
- SMetaTableInfo mtInfo = getUidfromSnapShot(sContext);
- tsdbReaderClose(pInfo->dataReader);
- pInfo->dataReader = NULL;
+ } else { // subType == TOPIC_SUB_TYPE__TABLE/TOPIC_SUB_TYPE__DB
- cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
- tableListClear(pTaskInfo->pTableInfoList);
+ if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+ SStreamRawScanInfo* pInfo = pOperator->info;
+ SSnapContext* sContext = pInfo->sContext;
- if (mtInfo.uid == 0) {
- return 0; // no data
+ SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id);
+ STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo;
+
+ if (setForSnapShot(sContext, pOffset->uid) != 0) {
+ qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id);
+ return -1;
+ }
+
+ SMetaTableInfo mtInfo = getUidfromSnapShot(sContext);
+ tsdbReaderClose(pInfo->dataReader);
+ pInfo->dataReader = NULL;
+
+ cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
+ tableListClear(pTableListInfo);
+
+ if (mtInfo.uid == 0) {
+ return 0; // no data
+ }
+
+ initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
+ pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
+
+ tableListAddTableInfo(pTableListInfo, mtInfo.uid, 0);
+
+ STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
+ int32_t size = tableListGetSize(pTableListInfo);
+
+ tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL,
+ false);
+
+ cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
+ strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
+ tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
+ pTaskInfo->streamInfo.schema = mtInfo.schema;
+
+ qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64 " %s", mtInfo.uid, pOffset->ts, id);
+ } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
+ SStreamRawScanInfo* pInfo = pOperator->info;
+ SSnapContext* sContext = pInfo->sContext;
+ if (setForSnapShot(sContext, pOffset->uid) != 0) {
+ qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
+ return -1;
+ }
+ qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64 " %s", pOffset->uid, pOffset->ts,
+ id);
+ } else if (pOffset->type == TMQ_OFFSET__LOG) {
+ SStreamRawScanInfo* pInfo = pOperator->info;
+ tsdbReaderClose(pInfo->dataReader);
+ pInfo->dataReader = NULL;
+ qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id);
}
-
- initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
- pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
-
- if (pTaskInfo->pTableInfoList == NULL) {
- pTaskInfo->pTableInfoList = tableListCreate();
- }
-
- tableListAddTableInfo(pTaskInfo->pTableInfoList, mtInfo.uid, 0);
-
- STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
- int32_t size = tableListGetSize(pTaskInfo->pTableInfoList);
-
- tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL);
-
- cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
- strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
- tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
- pTaskInfo->streamInfo.schema = mtInfo.schema;
-
- qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64, mtInfo.uid, pOffset->ts);
- } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
- SStreamRawScanInfo* pInfo = pOperator->info;
- SSnapContext* sContext = pInfo->sContext;
- if (setForSnapShot(sContext, pOffset->uid) != 0) {
- qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
- return -1;
- }
- qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64, pOffset->uid, pOffset->ts);
- } else if (pOffset->type == TMQ_OFFSET__LOG) {
- SStreamRawScanInfo* pInfo = pOperator->info;
- tsdbReaderClose(pInfo->dataReader);
- pInfo->dataReader = NULL;
- qDebug("tmqsnap qStreamPrepareScan snapshot log");
}
+
return 0;
}
void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle;
- if(pMsg->info.ahandle == NULL){
+ if (pMsg->info.ahandle == NULL) {
qError("pMsg->info.ahandle is NULL");
return;
}
@@ -1247,4 +1270,3 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
rpcFreeCont(pMsg->pCont);
destroySendMsgInfo(pSendInfo);
}
-
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 74c5af31be..5105fbc181 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -34,9 +34,7 @@
#include "ttypes.h"
#include "vnode.h"
-#define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN)
-#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN)
-
+#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN)
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
#if 0
@@ -73,35 +71,22 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) {
#define realloc u_realloc
#endif
-#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st)))
-#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0)
-
-typedef struct SAggOperatorInfo {
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
- STableQueryInfo* current;
- uint64_t groupId;
- SGroupResInfo groupResInfo;
- SExprSupp scalarExprSup;
- bool groupKeyOptimized;
-} SAggOperatorInfo;
+#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st)))
static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock);
static void releaseQueryBuf(size_t numOfTables);
-static void destroyAggOperatorInfo(void* param);
-static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
-static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId);
-static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
-static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize,
- const char* pKey);
+static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
+static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
+
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep,
int32_t status);
static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol);
static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo);
+static SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode);
void setOperatorCompleted(SOperatorInfo* pOperator) {
pOperator->status = OP_EXEC_DONE;
@@ -200,7 +185,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
if (!keepGroup) {
*(uint64_t*)pSup->keyBuf = calcGroupId(pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
}
-
+
SResultRowPosition* p1 =
(SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
@@ -266,57 +251,6 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
return pResult;
}
-// a new buffer page for each table. Needs to opt this design
-static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size) {
- if (pWindowRes->pageId != -1) {
- return 0;
- }
-
- SFilePage* pData = NULL;
-
- // in the first scan, new space needed for results
- int32_t pageId = -1;
- SArray* list = getDataBufPagesIdList(pResultBuf);
-
- if (taosArrayGetSize(list) == 0) {
- pData = getNewBufPage(pResultBuf, &pageId);
- pData->num = sizeof(SFilePage);
- } else {
- SPageInfo* pi = getLastPageInfo(list);
- pData = getBufPage(pResultBuf, getPageId(pi));
- if (pData == NULL) {
- qError("failed to get buffer, code:%s", tstrerror(terrno));
- return terrno;
- }
-
- pageId = getPageId(pi);
-
- if (pData->num + size > getBufPageSize(pResultBuf)) {
- // release current page first, and prepare the next one
- releaseBufPageInfo(pResultBuf, pi);
-
- pData = getNewBufPage(pResultBuf, &pageId);
- if (pData != NULL) {
- pData->num = sizeof(SFilePage);
- }
- }
- }
-
- if (pData == NULL) {
- return -1;
- }
-
- // set the number of rows in current disk page
- if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer
- pWindowRes->pageId = pageId;
- pWindowRes->offset = (int32_t)pData->num;
-
- pData->num += size;
- }
-
- return 0;
-}
-
// query_range_start, query_range_end, window_duration, window_start, window_end
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) {
pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP;
@@ -332,72 +266,6 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
colDataSetInt64(pColData, 4, &pQueryWindow->ekey);
}
-typedef struct {
- bool hasAgg;
- int32_t numOfRows;
- int32_t startOffset;
-} SFunctionCtxStatus;
-
-static void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
- pStatus->hasAgg = pCtx->input.colDataSMAIsSet;
- pStatus->numOfRows = pCtx->input.numOfRows;
- pStatus->startOffset = pCtx->input.startRowIndex;
-}
-
-static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
- pCtx->input.colDataSMAIsSet = pStatus->hasAgg;
- pCtx->input.numOfRows = pStatus->numOfRows;
- pCtx->input.startRowIndex = pStatus->startOffset;
-}
-
-void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData,
- int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) {
- for (int32_t k = 0; k < numOfOutput; ++k) {
- // keep it temporarily
- SFunctionCtxStatus status = {0};
- functionCtxSave(&pCtx[k], &status);
-
- pCtx[k].input.startRowIndex = offset;
- pCtx[k].input.numOfRows = forwardStep;
-
- // not a whole block involved in query processing, statistics data can not be used
- // NOTE: the original value of isSet have been changed here
- if (pCtx[k].input.colDataSMAIsSet && forwardStep < numOfTotal) {
- pCtx[k].input.colDataSMAIsSet = false;
- }
-
- if (pCtx[k].isPseudoFunc) {
- SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]);
-
- char* p = GET_ROWCELL_INTERBUF(pEntryInfo);
-
- SColumnInfoData idata = {0};
- idata.info.type = TSDB_DATA_TYPE_BIGINT;
- idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
- idata.pData = p;
-
- SScalarParam out = {.columnData = &idata};
- SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
- pCtx[k].sfp.process(&tw, 1, &out);
- pEntryInfo->numOfRes = 1;
- } else {
- int32_t code = TSDB_CODE_SUCCESS;
- if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) {
- code = pCtx[k].fpSet.process(&pCtx[k]);
-
- if (code != TSDB_CODE_SUCCESS) {
- qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code));
- taskInfo->code = code;
- T_LONG_JMP(taskInfo->env, code);
- }
- }
-
- // restore it
- functionCtxRestore(&pCtx[k], &status);
- }
- }
-}
-
static void doSetInputDataBlockInfo(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) {
SqlFunctionCtx* pCtx = pExprSup->pCtx;
for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) {
@@ -512,25 +380,6 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
return code;
}
-static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) {
- for (int32_t k = 0; k < pOperator->exprSupp.numOfExprs; ++k) {
- if (functionNeedToExecute(&pCtx[k])) {
- // todo add a dummy funtion to avoid process check
- if (pCtx[k].fpSet.process == NULL) {
- continue;
- }
-
- int32_t code = pCtx[k].fpSet.process(&pCtx[k]);
- if (code != TSDB_CODE_SUCCESS) {
- qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code));
- return code;
- }
- }
- }
-
- return TSDB_CODE_SUCCESS;
-}
-
bool functionNeedToExecute(SqlFunctionCtx* pCtx) {
struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
@@ -633,7 +482,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
}
}
-bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code) ? true : false; }
+bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); }
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; }
@@ -654,149 +503,6 @@ STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int
return win;
}
-int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
- uint32_t* status) {
- *status = BLK_DATA_NOT_LOAD;
-
- pBlock->pDataBlock = NULL;
- pBlock->pBlockAgg = NULL;
-
- // int64_t groupId = pRuntimeEnv->current->groupIndex;
- // bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);
-
- STaskCostInfo* pCost = &pTaskInfo->cost;
-
-// pCost->totalBlocks += 1;
-// pCost->totalRows += pBlock->info.rows;
-#if 0
- // Calculate all time windows that are overlapping or contain current data block.
- // If current data block is contained by all possible time window, do not load current data block.
- if (/*pQueryAttr->pFilters || */pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 ||
- (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pTaskInfo, &pBlock->info))) {
- (*status) = BLK_DATA_DATA_LOAD;
- }
-
- // check if this data block is required to load
- if ((*status) != BLK_DATA_DATA_LOAD) {
- bool needFilter = true;
-
- // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet,
- // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer
- if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) {
- SResultRow* pResult = NULL;
-
- bool masterScan = IS_MAIN_SCAN(pRuntimeEnv);
- TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey;
-
- STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr);
- if (pQueryAttr->pointInterpQuery) {
- needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId,
- pTableScanInfo->pCtx, pTableScanInfo->numOfOutput,
- pTableScanInfo->rowEntryInfoOffset);
- } else {
- if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId,
- pTableScanInfo->pCtx, pTableScanInfo->numOfOutput,
- pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) {
- T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY);
- }
- }
- } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate
- doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx,
- pTableScanInfo->rowEntryInfoOffset, pTableScanInfo->numOfOutput,
- pRuntimeEnv->current->groupIndex);
- }
-
- if (needFilter) {
- (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock);
- } else {
- (*status) = BLK_DATA_DATA_LOAD;
- }
- }
-
- SDataBlockInfo* pBlockInfo = &pBlock->info;
-// *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status);
-
- if ((*status) == BLK_DATA_NOT_LOAD || (*status) == BLK_DATA_FILTEROUT) {
- //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey,
-// pBlockInfo->window.ekey, pBlockInfo->rows);
- pCost->skipBlocks += 1;
- } else if ((*status) == BLK_DATA_SMA_LOAD) {
- // this function never returns error?
- pCost->loadBlockStatis += 1;
-// tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg);
-
- if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block
-// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
- pCost->totalCheckedRows += pBlock->info.rows;
- }
- } else {
- assert((*status) == BLK_DATA_DATA_LOAD);
-
- // load the data block statistics to perform further filter
- pCost->loadBlockStatis += 1;
-// tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg);
-
- if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) {
- { // set previous window
- if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) {
- SResultRow* pResult = NULL;
-
- bool masterScan = IS_MAIN_SCAN(pRuntimeEnv);
- TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey;
-
- STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr);
- if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId,
- pTableScanInfo->pCtx, pTableScanInfo->numOfOutput,
- pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) {
- T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY);
- }
- }
- }
- bool load = false;
- for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
- int32_t functionId = pTableScanInfo->pCtx[i].functionId;
- if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) {
-// load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockAgg[i].min),
-// (char*)&(pBlock->pBlockAgg[i].max));
- if (!load) { // current block has been discard due to filter applied
- pCost->skipBlocks += 1;
- //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId,
-// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
- (*status) = BLK_DATA_FILTEROUT;
- return TSDB_CODE_SUCCESS;
- }
- }
- }
- }
-
- // current block has been discard due to filter applied
-// if (!doFilterByBlockSMA(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) {
-// pCost->skipBlocks += 1;
-// qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey,
-// pBlockInfo->window.ekey, pBlockInfo->rows);
-// (*status) = BLK_DATA_FILTEROUT;
-// return TSDB_CODE_SUCCESS;
-// }
-
- pCost->totalCheckedRows += pBlockInfo->rows;
- pCost->loadBlocks += 1;
-// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
-// if (pBlock->pDataBlock == NULL) {
-// return terrno;
-// }
-
-// if (pQueryAttr->pFilters != NULL) {
-// filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock);
-// }
-
-// if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) {
-// filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery);
-// }
- }
-#endif
- return TSDB_CODE_SUCCESS;
-}
-
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
if (status == TASK_NOT_COMPLETED) {
pTaskInfo->status = status;
@@ -1027,43 +733,6 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD
}
}
-void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) {
- // for simple group by query without interval, all the tables belong to one group result.
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- SAggOperatorInfo* pAggInfo = pOperator->info;
-
- SResultRowInfo* pResultRowInfo = &pAggInfo->binfo.resultRowInfo;
- SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
- int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset;
-
- SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId,
- sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup, true);
- /*
- * not assign result buffer yet, add new result buffer
- * all group belong to one result set, and each group result has different group id so set the id to be one
- */
- if (pResultRow->pageId == -1) {
- int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize);
- if (ret != TSDB_CODE_SUCCESS) {
- T_LONG_JMP(pTaskInfo->env, terrno);
- }
- }
-
- setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
-}
-
-static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) {
- SAggOperatorInfo* pAggInfo = pOperator->info;
- if (pAggInfo->groupId != UINT64_MAX && pAggInfo->groupId == groupId) {
- return;
- }
-
- doSetTableGroupOutputBuf(pOperator, numOfOutput, groupId);
-
- // record the current active group id
- pAggInfo->groupId = groupId;
-}
-
void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowEntryOffset) {
bool returnNotNull = false;
for (int32_t j = 0; j < numOfExprs; ++j) {
@@ -1201,10 +870,9 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
blockDataEnsureCapacity(pBlock, pBlock->info.rows + pRow->numOfRows);
- qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s",
- (pRow->numOfRows+pBlock->info.rows),
- pBlock->info.capacity, GET_TASKID(pTaskInfo));
- // todo set the pOperator->resultInfo size
+ qDebug("datablock capacity not sufficient, expand to required:%" PRId64 ", current capacity:%d, %s",
+ (pRow->numOfRows + pBlock->info.rows), pBlock->info.capacity, GET_TASKID(pTaskInfo));
+ // todo set the pOperator->resultInfo size
}
pGroupResInfo->index += 1;
@@ -1214,7 +882,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
pBlock->info.rows += pRow->numOfRows;
}
- qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
+ qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
pBlock->info.id.groupId);
pBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pBlock, 0);
@@ -1281,161 +949,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
}
}
-// static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo,
-// STableQueryInfo* pTableQueryInfo) {
-// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
-// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo;
-//
-// STimeWindow tw = *win;
-// getNextTimeWindow(pQueryAttr, &tw);
-//
-// if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) ||
-// (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) {
-//
-// // load the data block and check data remaining in current data block
-// // TODO optimize performance
-// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
-// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
-//
-// tw = *win;
-// int32_t startPos =
-// getNextQualifiedWindow(pQueryAttr, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1);
-//
-// // set the abort info
-// pQueryAttr->pos = startPos;
-//
-// // reset the query start timestamp
-// pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos];
-// pQueryAttr->window.skey = pTableQueryInfo->win.skey;
-// TSKEY key = pTableQueryInfo->win.skey;
-//
-// pWindowResInfo->prevSKey = tw.skey;
-// int32_t index = pRuntimeEnv->resultRowInfo.curIndex;
-//
-// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock);
-// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index
-//
-// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d,
-// lastKey:%" PRId64,
-// GET_TASKID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes,
-// pQueryAttr->current->lastKey);
-//
-// return key;
-// } else { // do nothing
-// pQueryAttr->window.skey = tw.skey;
-// pWindowResInfo->prevSKey = tw.skey;
-// pTableQueryInfo->lastKey = tw.skey;
-//
-// return tw.skey;
-// }
-//
-// return true;
-// }
-
-// static bool skipTimeInterval(STaskRuntimeEnv *pRuntimeEnv, TSKEY* start) {
-// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
-//
-// // if queried with value filter, do NOT forward query start position
-// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL ||
-// pRuntimeEnv->pFillInfo != NULL) {
-// return true;
-// }
-//
-// /*
-// * 1. for interval without interpolation query we forward pQueryAttr->interval.interval at a time for
-// * pQueryAttr->limit.offset times. Since hole exists, pQueryAttr->interval.interval*pQueryAttr->limit.offset
-// value is
-// * not valid. otherwise, we only forward pQueryAttr->limit.offset number of points
-// */
-// STimeWindow w = TSWINDOW_INITIALIZER;
-// bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);
-//
-// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo;
-// STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current;
-//
-// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
-// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) {
-// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo);
-//
-// if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
-// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
-// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.skey, blockInfo.window.skey, pQueryAttr->window.ekey,
-// &w); pWindowResInfo->prevSKey = w.skey;
-// }
-// } else {
-// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.ekey, pQueryAttr->window.ekey, blockInfo.window.ekey, &w);
-// pWindowResInfo->prevSKey = w.skey;
-// }
-//
-// // the first time window
-// STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQueryAttr);
-//
-// while (pQueryAttr->limit.offset > 0) {
-// STimeWindow tw = win;
-//
-// if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) {
-// pQueryAttr->limit.offset -= 1;
-// pWindowResInfo->prevSKey = win.skey;
-//
-// // current time window is aligned with blockInfo.window.ekey
-// // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL;
-// if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) {
-// pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
-// }
-// }
-//
-// if (pQueryAttr->limit.offset == 0) {
-// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo);
-// return true;
-// }
-//
-// // current window does not ended in current data block, try next data block
-// getNextTimeWindow(pQueryAttr, &tw);
-//
-// /*
-// * If the next time window still starts from current data block,
-// * load the primary timestamp column first, and then find the start position for the next queried time window.
-// * Note that only the primary timestamp column is required.
-// * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually
-// required
-// * time window resides in current data block.
-// */
-// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) {
-//
-// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
-// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
-//
-// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) {
-// pQueryAttr->limit.offset -= 1;
-// }
-//
-// if (pQueryAttr->limit.offset == 0) {
-// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo);
-// return true;
-// } else {
-// tw = win;
-// int32_t startPos =
-// getNextQualifiedWindow(pQueryAttr, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
-// // set the abort info
-// pQueryAttr->pos = startPos;
-// pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos];
-// pWindowResInfo->prevSKey = tw.skey;
-// win = tw;
-// }
-// } else {
-// break; // offset is not 0, and next time window begins or ends in the next block.
-// }
-// }
-// }
-//
-// // check for error
-// if (terrno != TSDB_CODE_SUCCESS) {
-// T_LONG_JMP(pRuntimeEnv->env, terrno);
-// }
-//
-// return true;
-// }
-
int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num) {
p->pDownstream = taosMemoryCalloc(1, num * POINTER_BYTES);
if (p->pDownstream == NULL) {
@@ -1481,191 +994,23 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
}
}
-static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) {
- if (!tsCountAlwaysReturnValue) {
- return TSDB_CODE_SUCCESS;
- }
-
- SAggOperatorInfo* pAggInfo = pOperator->info;
- if (pAggInfo->groupKeyOptimized) {
- return TSDB_CODE_SUCCESS;
- }
-
- SOperatorInfo* downstream = pOperator->pDownstream[0];
- if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION ||
- (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN &&
- ((STableScanInfo*)downstream->info)->hasGroupByTag == true)) {
- return TSDB_CODE_SUCCESS;
- }
-
- SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
- bool hasCountFunc = false;
-
- for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
- const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
- if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) ||
- (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) {
- hasCountFunc = true;
- break;
- }
- }
-
- if (!hasCountFunc) {
- return TSDB_CODE_SUCCESS;
- }
-
- SSDataBlock* pBlock = createDataBlock();
- pBlock->info.rows = 1;
- pBlock->info.capacity = 0;
-
- for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
- SColumnInfoData colInfo = {0};
- colInfo.hasNull = true;
- colInfo.info.type = TSDB_DATA_TYPE_NULL;
- colInfo.info.bytes = 1;
-
- SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i];
- for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) {
- SFunctParam* pFuncParam = &pOneExpr->base.pParam[j];
- if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) {
- int32_t slotId = pFuncParam->pCol->slotId;
- int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
- if (slotId >= numOfCols) {
- taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1);
- for (int32_t k = numOfCols; k < slotId + 1; ++k) {
- taosArrayPush(pBlock->pDataBlock, &colInfo);
- }
- }
- } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
- // do nothing
- }
- }
- }
-
- blockDataEnsureCapacity(pBlock, pBlock->info.rows);
- *ppBlock = pBlock;
-
- return TSDB_CODE_SUCCESS;
-}
-
-static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) {
- if (!blockAllocated) {
- return;
- }
-
- blockDataDestroy(*ppBlock);
- *ppBlock = NULL;
-}
-
-// this is a blocking operator
-static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
- if (OPTR_IS_OPENED(pOperator)) {
- return TSDB_CODE_SUCCESS;
- }
-
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- SAggOperatorInfo* pAggInfo = pOperator->info;
-
- SExprSupp* pSup = &pOperator->exprSupp;
- SOperatorInfo* downstream = pOperator->pDownstream[0];
-
- int64_t st = taosGetTimestampUs();
-
- int32_t order = TSDB_ORDER_ASC;
- int32_t scanFlag = MAIN_SCAN;
-
- bool hasValidBlock = false;
- bool blockAllocated = false;
-
- while (1) {
- SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
- if (pBlock == NULL) {
- if (!hasValidBlock) {
- createDataBlockForEmptyInput(pOperator, &pBlock);
- if (pBlock == NULL) {
- break;
- }
- blockAllocated = true;
- } else {
- break;
- }
- }
- hasValidBlock = true;
-
- int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false);
- if (code != TSDB_CODE_SUCCESS) {
- destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
- T_LONG_JMP(pTaskInfo->env, code);
- }
-
- // there is an scalar expression that needs to be calculated before apply the group aggregation.
- if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) {
- SExprSupp* pSup1 = &pAggInfo->scalarExprSup;
- code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL);
- if (code != TSDB_CODE_SUCCESS) {
- destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
- T_LONG_JMP(pTaskInfo->env, code);
- }
- }
-
- // the pDataBlock are always the same one, no need to call this again
- setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId);
- setInputDataBlock(pSup, pBlock, order, scanFlag, true);
- code = doAggregateImpl(pOperator, pSup->pCtx);
- if (code != 0) {
- destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
- T_LONG_JMP(pTaskInfo->env, code);
- }
-
- destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
- }
-
- // the downstream operator may return with error code, so let's check the code before generating results.
- if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
- T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
- }
-
- initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0);
- OPTR_SET_OPENED(pOperator);
-
- pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
- return pTaskInfo->code;
-}
-
-static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
- SAggOperatorInfo* pAggInfo = pOperator->info;
- SOptrBasicInfo* pInfo = &pAggInfo->binfo;
-
- if (pOperator->status == OP_EXEC_DONE) {
+// QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN
+SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id) {
+ if (pOperator == NULL) {
+ qError("invalid operator, failed to find tableScanOperator %s", id);
return NULL;
}
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
- if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
- setOperatorCompleted(pOperator);
- return NULL;
- }
-
- blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
- while (1) {
- doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf);
- doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
-
- if (!hasRemainResults(&pAggInfo->groupResInfo)) {
- setOperatorCompleted(pOperator);
- break;
+ if (pOperator->operatorType == type) {
+ return pOperator;
+ } else {
+ if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) {
+ qError("invalid operator, failed to find tableScanOperator %s", id);
+ return NULL;
}
- if (pInfo->pRes->info.rows > 0) {
- break;
- }
+ return extractOperatorInTree(pOperator->pDownstream[0], type, id);
}
-
- size_t rows = blockDataGetNumOfRows(pInfo->pRes);
- pOperator->resultInfo.totalRows += rows;
-
- return (rows == 0) ? NULL : pInfo->pRes;
}
void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) {
@@ -1732,70 +1077,6 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul
return 0;
}
-int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize,
- const char* pKey) {
- int32_t code = 0;
- // _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
-
- pAggSup->currentPageId = -1;
- pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
- pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t));
- pAggSup->pResultRowHashTable = tSimpleHashInit(100, taosFastHash);
-
- if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
-
- uint32_t defaultPgsz = 0;
- uint32_t defaultBufsz = 0;
- getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz);
-
- if (!osTempSpaceAvailable()) {
- code = TSDB_CODE_NO_AVAIL_DISK;
- qError("Init stream agg supporter failed since %s, %s", terrstr(code), pKey);
- return code;
- }
-
- code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir);
- if (code != TSDB_CODE_SUCCESS) {
- qError("Create agg result buf failed since %s, %s", tstrerror(code), pKey);
- return code;
- }
-
- return code;
-}
-
-void cleanupAggSup(SAggSupporter* pAggSup) {
- taosMemoryFreeClear(pAggSup->keyBuf);
- tSimpleHashCleanup(pAggSup->pResultRowHashTable);
- destroyDiskbasedBuf(pAggSup->pResultBuf);
-}
-
-int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
- const char* pkey, void* pState) {
- int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
-
- code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
-
- for (int32_t i = 0; i < numOfCols; ++i) {
- if (pState) {
- pSup->pCtx[i].saveHandle.pBuf = NULL;
- pSup->pCtx[i].saveHandle.pState = pState;
- pSup->pCtx[i].exprIdx = i;
- } else {
- pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf;
- }
- }
-
- return TSDB_CODE_SUCCESS;
-}
-
void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows) {
if (numOfRows == 0) {
numOfRows = 4096;
@@ -1814,7 +1095,7 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock) {
initResultRowInfo(&pInfo->resultRowInfo);
}
-void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
+static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
if (pCtx == NULL) {
return NULL;
}
@@ -1866,99 +1147,8 @@ void cleanupExprSupp(SExprSupp* pSupp) {
taosMemoryFree(pSupp->rowEntryInfoOffset);
}
-SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode,
- SExecTaskInfo* pTaskInfo) {
- SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo));
- SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
- if (pInfo == NULL || pOperator == NULL) {
- goto _error;
- }
-
- SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
- initBasicInfo(&pInfo->binfo, pResBlock);
-
- size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
- initResultSizeInfo(&pOperator->resultInfo, 4096);
-
- int32_t num = 0;
- SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
- int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
- if (code != TSDB_CODE_SUCCESS) {
- goto _error;
- }
-
- int32_t numOfScalarExpr = 0;
- SExprInfo* pScalarExprInfo = NULL;
- if (pAggNode->pExprs != NULL) {
- pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr);
- }
-
- code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr);
- if (code != TSDB_CODE_SUCCESS) {
- goto _error;
- }
-
- code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
- if (code != TSDB_CODE_SUCCESS) {
- goto _error;
- }
-
- pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock;
- pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized;
- pInfo->groupId = UINT64_MAX;
-
- setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo,
- pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo,
- optrDefaultBufFn, NULL);
-
- if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
- STableScanInfo* pTableScanInfo = downstream->info;
- pTableScanInfo->base.pdInfo.pExprSup = &pOperator->exprSupp;
- pTableScanInfo->base.pdInfo.pAggSup = &pInfo->aggSup;
- }
-
- code = appendDownstream(pOperator, &downstream, 1);
- if (code != TSDB_CODE_SUCCESS) {
- goto _error;
- }
-
- return pOperator;
-
-_error:
- if (pInfo != NULL) {
- destroyAggOperatorInfo(pInfo);
- }
-
- if (pOperator != NULL) {
- cleanupExprSupp(&pOperator->exprSupp);
- }
-
- taosMemoryFreeClear(pOperator);
- pTaskInfo->code = code;
- return NULL;
-}
-
void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); }
-static void freeItem(void* pItem) {
- void** p = pItem;
- if (*p != NULL) {
- taosMemoryFreeClear(*p);
- }
-}
-
-void destroyAggOperatorInfo(void* param) {
- SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param;
- cleanupBasicInfo(&pInfo->binfo);
-
- cleanupAggSup(&pInfo->aggSup);
- cleanupExprSupp(&pInfo->scalarExprSup);
- cleanupGroupResInfo(&pInfo->groupResInfo);
- taosMemoryFreeClear(param);
-}
-
char* buildTaskId(uint64_t taskId, uint64_t queryId) {
char* p = taosMemoryMalloc(64);
@@ -1974,7 +1164,8 @@ char* buildTaskId(uint64_t taskId, uint64_t queryId) {
return p;
}
-static SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, char* dbFName) {
+SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model,
+ char* dbFName) {
SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
if (pTaskInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -1982,21 +1173,20 @@ static SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, in
}
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
+ pTaskInfo->cost.created = taosGetTimestampUs();
pTaskInfo->schemaInfo.dbname = taosStrdup(dbFName);
pTaskInfo->execModel = model;
- pTaskInfo->pTableInfoList = tableListCreate();
pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo));
pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES);
+ taosInitRWLatch(&pTaskInfo->lock);
pTaskInfo->id.vgId = vgId;
pTaskInfo->id.queryId = queryId;
pTaskInfo->id.str = buildTaskId(taskId, queryId);
return pTaskInfo;
}
-SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode);
-
int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) {
SMetaReader mr = {0};
if (pHandle == NULL) {
@@ -2103,9 +1293,8 @@ bool groupbyTbname(SNodeList* pGroupList) {
SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SNode* pTagCond,
SNode* pTagIndexCond, const char* pUser) {
- int32_t type = nodeType(pPhyNode);
- STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
- const char* idstr = GET_TASKID(pTaskInfo);
+ int32_t type = nodeType(pPhyNode);
+ const char* idstr = GET_TASKID(pTaskInfo);
if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
SOperatorInfo* pOperator = NULL;
@@ -2118,11 +1307,13 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pTableScanNode->groupSort = true;
}
- int32_t code =
+ STableListInfo* pTableListInfo = tableListCreate();
+ int32_t code =
createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle,
pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
if (code) {
pTaskInfo->code = code;
+ tableListDestroy(pTableListInfo);
qError("failed to createScanTableListInfo, code:%s, %s", tstrerror(code), idstr);
return NULL;
}
@@ -2130,10 +1321,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
code = extractTableSchemaInfo(pHandle, &pTableScanNode->scan, pTaskInfo);
if (code) {
pTaskInfo->code = terrno;
+ tableListDestroy(pTableListInfo);
return NULL;
}
- pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
+ pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo);
if (NULL == pOperator) {
pTaskInfo->code = terrno;
return NULL;
@@ -2143,11 +1335,13 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder;
} else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
+ STableListInfo* pTableListInfo = tableListCreate();
int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle,
pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
if (code) {
pTaskInfo->code = code;
+ tableListDestroy(pTableListInfo);
qError("failed to createScanTableListInfo, code: %s", tstrerror(code));
return NULL;
}
@@ -2155,12 +1349,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
code = extractTableSchemaInfo(pHandle, &pTableScanNode->scan, pTaskInfo);
if (code) {
pTaskInfo->code = terrno;
+ tableListDestroy(pTableListInfo);
return NULL;
}
- pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
+ pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo);
if (NULL == pOperator) {
pTaskInfo->code = terrno;
+ tableListDestroy(pTableListInfo);
return NULL;
}
@@ -2171,29 +1367,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
+ STableListInfo* pTableListInfo = tableListCreate();
+
if (pHandle->vnode) {
int32_t code =
createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort,
pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
if (code) {
pTaskInfo->code = code;
+ tableListDestroy(pTableListInfo);
qError("failed to createScanTableListInfo, code: %s", tstrerror(code));
return NULL;
}
-
-#ifndef NDEBUG
- int32_t sz = tableListGetSize(pTableListInfo);
- qDebug("vgId:%d create stream task, total qualified tables:%d, %s", pTaskInfo->id.vgId, sz, idstr);
-
- for (int32_t i = 0; i < sz; i++) {
- STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i);
- qDebug("add table uid:%" PRIu64 ", gid:%" PRIu64, pKeyInfo->uid, pKeyInfo->groupId);
- }
-#endif
}
pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan);
- pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo);
+ pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode;
pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo);
@@ -2202,18 +1391,19 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
-
- int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond,
- pTagIndexCond, pTaskInfo);
+ STableListInfo* pTableListInfo = tableListCreate();
+ int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond,
+ pTagIndexCond, pTaskInfo);
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code;
qError("failed to getTableList, code: %s", tstrerror(code));
return NULL;
}
- pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTaskInfo);
+ pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
+ STableListInfo* pTableListInfo = tableListCreate();
if (pBlockNode->tableType == TSDB_SUPER_TABLE) {
SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo));
@@ -2234,9 +1424,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
tableListAddTableInfo(pTableListInfo, pBlockNode->uid, 0);
}
- pOperator = createDataBlockInfoScanOperator(pHandle, pBlockNode, pTaskInfo);
+ pOperator = createDataBlockInfoScanOperator(pHandle, pBlockNode, pTableListInfo, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) {
SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode;
+ STableListInfo* pTableListInfo = tableListCreate();
int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo,
pTagCond, pTagIndexCond, pTaskInfo);
@@ -2251,7 +1442,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return NULL;
}
- pOperator = createCacherowsScanOperator(pScanNode, pHandle, pTaskInfo);
+ pOperator = createCacherowsScanOperator(pScanNode, pHandle, pTableListInfo, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) {
pOperator = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo);
} else {
@@ -2259,7 +1450,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return NULL;
}
- if (pOperator != NULL) {
+ if (pOperator != NULL) { // todo moved away
pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId;
}
@@ -2400,9 +1591,8 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) {
return -1;
}
-int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle) {
- SExecTaskInfo* pTask = *(SExecTaskInfo**)pTaskInfo;
-
+int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* pTableListInfo,
+ SReadHandle* readHandle) {
switch (pNode->type) {
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: {
SInserterParam* pInserterParam = taosMemoryCalloc(1, sizeof(SInserterParam));
@@ -2419,8 +1609,9 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT
if (NULL == pDeleterParam) {
return TSDB_CODE_OUT_OF_MEMORY;
}
- int32_t tbNum = tableListGetSize(pTask->pTableInfoList);
- pDeleterParam->suid = tableListGetSuid(pTask->pTableInfoList);
+
+ int32_t tbNum = tableListGetSize(pTableListInfo);
+ pDeleterParam->suid = tableListGetSuid(pTableListInfo);
// TODO extract uid list
pDeleterParam->pUidList = taosArrayInit(tbNum, sizeof(uint64_t));
@@ -2430,11 +1621,12 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT
}
for (int32_t i = 0; i < tbNum; ++i) {
- STableKeyInfo* pTable = tableListGetInfo(pTask->pTableInfoList, i);
+ STableKeyInfo* pTable = tableListGetInfo(pTableListInfo, i);
taosArrayPush(pDeleterParam->pUidList, &pTable->uid);
}
*pParam = pDeleterParam;
+
break;
}
default:
@@ -2445,7 +1637,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT
}
int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
- int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) {
+ int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) {
*pTaskInfo = doCreateExecTaskInfo(pPlan->id.queryId, taskId, vgId, model, pPlan->dbFName);
if (*pTaskInfo == NULL) {
goto _complete;
@@ -2469,7 +1661,6 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand
goto _complete;
}
- (*pTaskInfo)->cost.created = taosGetTimestampUs();
return TSDB_CODE_SUCCESS;
_complete:
@@ -2485,8 +1676,6 @@ static void freeBlock(void* pParam) {
void doDestroyTask(SExecTaskInfo* pTaskInfo) {
qDebug("%s execTask is freed", GET_TASKID(pTaskInfo));
-
- pTaskInfo->pTableInfoList = tableListDestroy(pTaskInfo->pTableInfoList);
destroyOperatorInfo(pTaskInfo->pRoot);
cleanupTableSchemaInfo(&pTaskInfo->schemaInfo);
cleanupStreamInfo(&pTaskInfo->streamInfo);
@@ -2797,7 +1986,7 @@ void qStreamCloseTsdbReader(void* task) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
setOperatorCompleted(pInfo->pTableScanOp);
- while(pTaskInfo->owner != 0) {
+ while (pTaskInfo->owner != 0) {
taosMsleep(100);
qDebug("wait for the reader stopping");
}
@@ -2813,3 +2002,21 @@ void qStreamCloseTsdbReader(void* task) {
}
}
}
+
+static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) {
+ if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
+ STableScanInfo* pScanInfo = pOperator->info;
+ taosArrayPush(pList, &pScanInfo->base.pTableListInfo);
+ } else {
+ if (pOperator->pDownstream != NULL && pOperator->pDownstream[0] != NULL) {
+ extractTableList(pList, pOperator->pDownstream[0]);
+ }
+ }
+}
+
+SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) {
+ SArray* pArray = taosArrayInit(0, POINTER_BYTES);
+ SOperatorInfo* pOperator = pTaskInfo->pRoot;
+ extractTableList(pArray, pOperator);
+ return pArray;
+}
\ No newline at end of file
diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c
index 2a33e3527a..234f1a666c 100644
--- a/source/libs/executor/src/filloperator.c
+++ b/source/libs/executor/src/filloperator.c
@@ -447,9 +447,14 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
return NULL;
}
+void destroySPoint(void* ptr) {
+ SPoint* point = (SPoint*) ptr;
+ taosMemoryFreeClear(point->val);
+}
+
void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) {
- taosArrayDestroy(pFillLinear->pDeltaVal);
- taosArrayDestroy(pFillLinear->pNextDeltaVal);
+ taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint);
+ taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint);
taosMemoryFree(pFillLinear);
return NULL;
}
@@ -611,19 +616,15 @@ static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pR
}
}
-static void calcRowDeltaData(SResultRowData* pStartRow, SResultRowData* pEndRow, SArray* pDelta, SFillColInfo* pFillCol,
- int32_t numOfCol, int32_t winCount) {
+static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol,
+ int32_t numOfCol) {
for (int32_t i = 0; i < numOfCol; i++) {
if (!pFillCol[i].notFillCol) {
int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i);
- SResultCellData* pSCell = getResultCell(pStartRow, slotId);
- double start = 0.0;
- GET_TYPED_DATA(start, double, pSCell->type, pSCell->pData);
SResultCellData* pECell = getResultCell(pEndRow, slotId);
- double end = 0.0;
- GET_TYPED_DATA(end, double, pECell->type, pECell->pData);
- double delta = (end - start) / winCount;
- taosArraySet(pDelta, slotId, &delta);
+ SPoint* pPoint = taosArrayGet(pEndPoins, slotId);
+ pPoint->key = pEndRow->key;
+ memcpy(pPoint->val, pECell->pData, pECell->bytes);
}
}
}
@@ -674,10 +675,8 @@ void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillS
setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo);
pFillInfo->pLinearInfo->hasNext = false;
pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
- int32_t numOfWins = taosTimeCountInterval(pFillSup->prev.key, pFillSup->next.key, pFillSup->interval.sliding,
- pFillSup->interval.slidingUnit, pFillSup->interval.precision);
- calcRowDeltaData(&pFillSup->prev, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo,
- pFillSup->numOfAllCols, numOfWins);
+ calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
+ pFillSup->numOfAllCols);
pFillInfo->pResRow = &pFillSup->prev;
pFillInfo->pLinearInfo->winIndex = 0;
} break;
@@ -780,25 +779,19 @@ void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillS
setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
pFillInfo->pos = FILL_POS_MID;
pFillInfo->pLinearInfo->nextEnd = nextWKey;
- int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding,
- pFillSup->interval.slidingUnit, pFillSup->interval.precision);
- calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo,
- pFillSup->numOfAllCols, numOfWins);
+ calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
+ pFillSup->numOfAllCols);
pFillInfo->pResRow = &pFillSup->prev;
- numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
- pFillSup->interval.precision);
- calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pNextDeltaVal, pFillSup->pAllColInfo,
- pFillSup->numOfAllCols, numOfWins);
+ calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pNextEndPoints, pFillSup->pAllColInfo,
+ pFillSup->numOfAllCols);
pFillInfo->pLinearInfo->hasNext = true;
} else if (hasPrevWindow(pFillSup)) {
setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
pFillInfo->pos = FILL_POS_END;
pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
- int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding,
- pFillSup->interval.slidingUnit, pFillSup->interval.precision);
- calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo,
- pFillSup->numOfAllCols, numOfWins);
+ calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
+ pFillSup->numOfAllCols);
pFillInfo->pResRow = &pFillSup->prev;
pFillInfo->pLinearInfo->hasNext = false;
} else {
@@ -806,10 +799,8 @@ void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillS
setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo);
pFillInfo->pos = FILL_POS_START;
pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
- int32_t numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding,
- pFillSup->interval.slidingUnit, pFillSup->interval.precision);
- calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo,
- pFillSup->numOfAllCols, numOfWins);
+ calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
+ pFillSup->numOfAllCols);
pFillInfo->pResRow = &pFillSup->cur;
pFillInfo->pLinearInfo->hasNext = false;
}
@@ -906,13 +897,18 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
colDataSetNULL(pColData, index);
continue;
}
- double* pDelta = taosArrayGet(pFillInfo->pLinearInfo->pDeltaVal, slotId);
+ SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId);
double vCell = 0;
- GET_TYPED_DATA(vCell, double, pCell->type, pCell->pData);
- vCell += (*pDelta) * pFillInfo->pLinearInfo->winIndex;
- int64_t result = 0;
- SET_TYPED_DATA(&result, pCell->type, vCell);
- colDataSetVal(pColData, index, (const char*)&result, false);
+ SPoint start = {0};
+ start.key = pFillInfo->pResRow->key;
+ start.val = pCell->pData;
+
+ SPoint cur = {0};
+ cur.key = pFillInfo->current;
+ cur.val = taosMemoryCalloc(1, pCell->bytes);
+ taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type);
+ colDataSetVal(pColData, index, (const char*)cur.val, false);
+ destroySPoint(&cur);
}
}
pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
@@ -953,8 +949,7 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter*
if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) {
pFillInfo->pLinearInfo->hasNext = false;
pFillInfo->pLinearInfo->winIndex = 0;
- taosArrayClear(pFillInfo->pLinearInfo->pDeltaVal);
- taosArrayAddAll(pFillInfo->pLinearInfo->pDeltaVal, pFillInfo->pLinearInfo->pNextDeltaVal);
+ taosArraySwap(pFillInfo->pLinearInfo->pEndPoints, pFillInfo->pLinearInfo->pNextEndPoints);
pFillInfo->pResRow = &pFillSup->cur;
setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo);
doStreamFillLinear(pFillSup, pFillInfo, pRes);
@@ -1359,15 +1354,19 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo));
pFillInfo->pLinearInfo->hasNext = false;
pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
- pFillInfo->pLinearInfo->pDeltaVal = NULL;
- pFillInfo->pLinearInfo->pNextDeltaVal = NULL;
+ pFillInfo->pLinearInfo->pEndPoints = NULL;
+ pFillInfo->pLinearInfo->pNextEndPoints = NULL;
if (pFillSup->type == TSDB_FILL_LINEAR) {
- pFillInfo->pLinearInfo->pDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double));
- pFillInfo->pLinearInfo->pNextDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double));
+ pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
+ pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) {
- double value = 0.0;
- taosArrayPush(pFillInfo->pLinearInfo->pDeltaVal, &value);
- taosArrayPush(pFillInfo->pLinearInfo->pNextDeltaVal, &value);
+ SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i);
+ SPoint value = {0};
+ value.val = taosMemoryCalloc(1, pColData->info.bytes);
+ taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value);
+
+ value.val = taosMemoryCalloc(1, pColData->info.bytes);
+ taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value);
}
}
pFillInfo->pLinearInfo->winIndex = 0;
diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c
index 9fff7a4943..c943270df9 100644
--- a/source/libs/executor/src/projectoperator.c
+++ b/source/libs/executor/src/projectoperator.c
@@ -161,10 +161,9 @@ static int32_t discardGroupDataBlock(SSDataBlock* pBlock, SLimitInfo* pLimitInfo
if (pLimitInfo->remainGroupOffset > 0) {
return PROJECT_RETRIEVE_CONTINUE;
}
- }
- // set current group id of the project operator
- pLimitInfo->currentGroupId = pBlock->info.id.groupId;
+ pLimitInfo->currentGroupId = 0;
+ }
}
return PROJECT_RETRIEVE_DONE;
@@ -175,19 +174,29 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S
// here check for a new group data, we need to handle the data of the previous group.
ASSERT(pLimitInfo->remainGroupOffset == 0 || pLimitInfo->remainGroupOffset == -1);
- if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.id.groupId) {
+ bool newGroup = false;
+ if (0 == pBlock->info.id.groupId) {
+ pLimitInfo->numOfOutputGroups = 1;
+ } else if (pLimitInfo->currentGroupId != pBlock->info.id.groupId) {
+ pLimitInfo->currentGroupId = pBlock->info.id.groupId;
pLimitInfo->numOfOutputGroups += 1;
- if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
- setOperatorCompleted(pOperator);
- return PROJECT_RETRIEVE_DONE;
- }
-
- // reset the value for a new group data
- // existing rows that belongs to previous group.
- resetLimitInfoForNextGroup(pLimitInfo);
+ newGroup = true;
+ } else {
+ return PROJECT_RETRIEVE_CONTINUE;
}
- return PROJECT_RETRIEVE_DONE;
+ if ((pLimitInfo->slimit.limit >= 0) && (pLimitInfo->slimit.limit < pLimitInfo->numOfOutputGroups)) {
+ setOperatorCompleted(pOperator);
+ return PROJECT_RETRIEVE_DONE;
+ }
+
+ // reset the value for a new group data
+ // existing rows that belongs to previous group.
+ if (newGroup) {
+ resetLimitInfoForNextGroup(pLimitInfo);
+ }
+
+ return PROJECT_RETRIEVE_CONTINUE;
}
// todo refactor
@@ -199,7 +208,7 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS
if (pBlock->info.rows == 0) {
return PROJECT_RETRIEVE_CONTINUE;
} else {
- if (limitReached && (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
+ if (limitReached && (pLimitInfo->slimit.limit >= 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
setOperatorCompleted(pOperator);
}
}
@@ -262,7 +271,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
return NULL;
}
}
- qDebug("set op close, exec %d, status %d rows %d", pTaskInfo->execModel, pOperator->status,
+ qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status,
pFinalRes->info.rows);
setOperatorCompleted(pOperator);
break;
@@ -328,7 +337,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
// when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint.
if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) {
- qDebug("project return %d rows, status %d", pFinalRes->info.rows, pOperator->status);
+ qDebug("project return %" PRId64 " rows, status %d", pFinalRes->info.rows, pOperator->status);
break;
}
} else {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index be87dcd6ff..b9fd8512d0 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -31,7 +31,9 @@
#include "thash.h"
#include "ttypes.h"
-#define MULTI_READER_MAX_TABLE_NUM 5000
+int32_t scanDebug = 0;
+
+#define MULTI_READER_MAX_TABLE_NUM 5000
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
@@ -44,7 +46,7 @@ typedef struct STableMergeScanSortSourceParam {
SOperatorInfo* pOperator;
int32_t readerIdx;
uint64_t uid;
- SSDataBlock* inputBlock;
+ SSDataBlock* inputBlock;
bool multiReader;
STsdbReader* dataReader;
} STableMergeScanSortSourceParam;
@@ -308,15 +310,16 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
taosMemoryFreeClear(pBlock->pBlockAgg);
if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
- qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
+ qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->filterOutBlocks += 1;
pCost->totalRows += pBlock->info.rows;
tsdbReleaseDataBlock(pTableScanInfo->dataReader);
return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
- qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d, uid:%" PRIu64, GET_TASKID(pTaskInfo),
- pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, pBlockInfo->id.uid);
+ qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64,
+ GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows,
+ pBlockInfo->id.uid);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1);
pCost->skipBlocks += 1;
tsdbReleaseDataBlock(pTableScanInfo->dataReader);
@@ -326,7 +329,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
loadSMA = true; // mark the operation of load sma;
bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo);
if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead
- qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
+ qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1);
tsdbReleaseDataBlock(pTableScanInfo->dataReader);
@@ -346,8 +349,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
size_t size = taosArrayGetSize(pBlock->pDataBlock);
bool keep = doFilterByBlockSMA(pOperator->exprSupp.pFilterInfo, pBlock->pBlockAgg, size, pBlockInfo->rows);
if (!keep) {
- qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
- pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
+ qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64,
+ GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->filterOutBlocks += 1;
(*status) = FUNC_DATA_REQUIRED_FILTEROUT;
@@ -363,8 +366,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
// try to filter data block according to current results
doDynamicPruneDataBlock(pOperator, pBlockInfo, status);
if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
- qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
- pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
+ qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64,
+ GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->skipBlocks += 1;
tsdbReleaseDataBlock(pTableScanInfo->dataReader);
*status = FUNC_DATA_REQUIRED_FILTEROUT;
@@ -394,7 +397,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
if (pBlock->info.rows == 0) {
pCost->filterOutBlocks += 1;
- qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms",
+ qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", elapsed time:%.2f ms",
GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
} else {
qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
@@ -584,10 +587,16 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
if (isNullVal) {
colDataSetNNULL(pColInfoData, 0, pBlock->info.rows);
} else if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) {
- colDataSetNItems(pColInfoData, 0, data, pBlock->info.rows);
+ code = colDataSetNItems(pColInfoData, 0, data, pBlock->info.rows, false);
if (IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) {
taosMemoryFree(data);
}
+ if (code) {
+ if (freeReader) {
+ metaReaderClear(&mr);
+ }
+ return code;
+ }
} else { // todo opt for json tag
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
colDataSetVal(pColInfoData, i, data, false);
@@ -634,10 +643,22 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
STableScanInfo* pTableScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
+ bool hasNext = false;
+ int32_t code = TSDB_CODE_SUCCESS;
int64_t st = taosGetTimestampUs();
- while (tsdbNextDataBlock(pTableScanInfo->base.dataReader)) {
+ while (true) {
+ code = tsdbNextDataBlock(pTableScanInfo->base.dataReader, &hasNext);
+ if (code) {
+ tsdbReleaseDataBlock(pTableScanInfo->base.dataReader);
+ T_LONG_JMP(pTaskInfo->env, code);
+ }
+
+ if (!hasNext) {
+ break;
+ }
+
if (isTaskKilled(pTaskInfo)) {
tsdbReleaseDataBlock(pTableScanInfo->base.dataReader);
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
@@ -654,8 +675,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
continue;
}
- ASSERT(pBlock->info.id.uid != 0);
- pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
+ if (pBlock->info.id.uid) {
+ pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
+ }
uint32_t status = 0;
int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status);
@@ -680,7 +702,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid;
pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey;
- ASSERT(pBlock->info.id.uid != 0);
return pBlock;
}
return NULL;
@@ -750,42 +771,51 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
// scan table one by one sequentially
if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
- int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList);
+ int32_t numOfTables = 0; // tableListGetSize(pTaskInfo->pTableListInfo);
+ STableKeyInfo tInfo = {0};
while (1) {
SSDataBlock* result = doGroupedTableScan(pOperator);
- if (result || (pOperator->status == OP_EXEC_DONE)) {
+ if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) {
return result;
}
// if no data, switch to next table and continue scan
pInfo->currentTable++;
+
+ taosRLockLatch(&pTaskInfo->lock);
+ numOfTables = tableListGetSize(pInfo->base.pTableListInfo);
+
if (pInfo->currentTable >= numOfTables) {
+ qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo));
+ taosRUnLockLatch(&pTaskInfo->lock);
return NULL;
}
- STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable);
- tsdbSetTableList(pInfo->base.dataReader, pTableInfo, 1);
- qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables,
- pInfo->currentTable, pTaskInfo->id.str);
+ tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable);
+ taosRUnLockLatch(&pTaskInfo->lock);
+
+ tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1);
+ qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables,
+ pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo));
tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond);
pInfo->scanTimes = 0;
}
} else { // scan table group by group sequentially
if (pInfo->currentGroupId == -1) {
- if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) {
+ if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
setOperatorCompleted(pOperator);
return NULL;
}
int32_t num = 0;
STableKeyInfo* pList = NULL;
- tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num);
+ tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
ASSERT(pInfo->base.dataReader == NULL);
int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock,
- (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo));
+ (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
@@ -797,11 +827,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
SSDataBlock* result = doGroupedTableScan(pOperator);
if (result != NULL) {
- ASSERT(result->info.id.uid != 0);
return result;
}
- if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) {
+ if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
setOperatorCompleted(pOperator);
return NULL;
}
@@ -812,7 +841,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
int32_t num = 0;
STableKeyInfo* pList = NULL;
- tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num);
+ tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
tsdbSetTableList(pInfo->base.dataReader, pList, num);
tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond);
@@ -837,25 +866,30 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr
return 0;
}
+static void destroyTableScanBase(STableScanBase* pBase) {
+ cleanupQueryTableDataCond(&pBase->cond);
+
+ tsdbReaderClose(pBase->dataReader);
+ pBase->dataReader = NULL;
+
+ if (pBase->matchInfo.pList != NULL) {
+ taosArrayDestroy(pBase->matchInfo.pList);
+ }
+
+ tableListDestroy(pBase->pTableListInfo);
+ taosLRUCacheCleanup(pBase->metaCache.pTableMetaEntryCache);
+ cleanupExprSupp(&pBase->pseudoSup);
+}
+
static void destroyTableScanOperatorInfo(void* param) {
STableScanInfo* pTableScanInfo = (STableScanInfo*)param;
blockDataDestroy(pTableScanInfo->pResBlock);
- cleanupQueryTableDataCond(&pTableScanInfo->base.cond);
-
- tsdbReaderClose(pTableScanInfo->base.dataReader);
- pTableScanInfo->base.dataReader = NULL;
-
- if (pTableScanInfo->base.matchInfo.pList != NULL) {
- taosArrayDestroy(pTableScanInfo->base.matchInfo.pList);
- }
-
- taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache);
- cleanupExprSupp(&pTableScanInfo->base.pseudoSup);
+ destroyTableScanBase(&pTableScanInfo->base);
taosMemoryFreeClear(param);
}
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
- SExecTaskInfo* pTaskInfo) {
+ STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@@ -911,12 +945,17 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pTaskInfo);
pOperator->exprSupp.numOfExprs = numOfCols;
+ pInfo->base.pTableListInfo = pTableListInfo;
pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5);
if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) {
code = terrno;
goto _error;
}
+ if (scanDebug) {
+ pInfo->countOnly = true;
+ }
+
taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo,
optrDefaultBufFn, getTableScannerExecInfo);
@@ -1008,21 +1047,29 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
STsdbReader* pReader = NULL;
int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock,
- (STsdbReader**)&pReader, GET_TASKID(pTaskInfo));
+ (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
T_LONG_JMP(pTaskInfo->env, code);
return NULL;
}
- if (tsdbNextDataBlock(pReader)) {
+ bool hasNext = false;
+ code = tsdbNextDataBlock(pReader, &hasNext);
+ if (code != TSDB_CODE_SUCCESS) {
+ terrno = code;
+ T_LONG_JMP(pTaskInfo->env, code);
+ return NULL;
+ }
+
+ if (hasNext) {
/*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL);
doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
- pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
+ pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
}
tsdbReaderClose(pReader);
- qDebug("retrieve prev rows:%d, skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64
+ qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64
", suid:%" PRIu64,
pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid);
@@ -1039,7 +1086,8 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts,
}
static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) {
- return getTableGroupId(pInfo->pTableScanOp->pTaskInfo->pTableInfoList, uid);
+ STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
+ return getTableGroupId(pTableScanInfo->base.pTableListInfo, uid);
}
static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
@@ -1513,7 +1561,8 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
pInfo->pRes->info.type = STREAM_NORMAL;
pInfo->pRes->info.version = pBlock->info.version;
- pInfo->pRes->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
+ STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
+ pInfo->pRes->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
// todo extract method
for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
@@ -1569,19 +1618,15 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamScanInfo* pInfo = pOperator->info;
+ const char* id = GET_TASKID(pTaskInfo);
- qDebug("start to exec queue scan");
+ qDebug("start to exec queue scan, %s", id);
if (pTaskInfo->streamInfo.submit.msgStr != NULL) {
if (pInfo->tqReader->msg2.msgStr == NULL) {
- /*pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq;*/
-
- /*const SSubmitReq* pSubmit = pInfo->tqReader->pMsg;*/
- /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/
- /*void* msgStr = pTaskInfo->streamInfo.*/
SPackedData submit = pTaskInfo->streamInfo.submit;
if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) {
- qError("submit msg messed up when initing stream submit block %p", submit.msgStr);
+ qError("submit msg messed up when initing stream submit block %p, %s", submit.msgStr, id);
pInfo->tqReader->msg2 = (SPackedData){0};
pInfo->tqReader->setMsg = 0;
ASSERT(0);
@@ -1615,18 +1660,20 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
if (pResult && pResult->info.rows > 0) {
- qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows,
- pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion);
+ qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64 " %s",
+ pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey,
+ pInfo->tqReader->pWalReader->curVersion, id);
pTaskInfo->streamInfo.returned = 1;
return pResult;
} else {
+ // no data has return already, try to extract data in the WAL
if (!pTaskInfo->streamInfo.returned) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->base.dataReader);
- qDebug("3");
pTSInfo->base.dataReader = NULL;
tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
- qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1);
+
+ qDebug("queue scan tsdb over, switch to wal ver:%" PRId64 " %s", pTaskInfo->streamInfo.snapshotVer + 1, id);
if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) {
tqOffsetResetToLog(&pTaskInfo->streamInfo.lastStatus, pTaskInfo->streamInfo.snapshotVer);
return NULL;
@@ -1640,10 +1687,12 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__LOG) {
while (1) {
SFetchRet ret = {0};
+ terrno = 0;
+
if (tqNextBlock(pInfo->tqReader, &ret) < 0) {
// if the end is reached, terrno is 0
if (terrno != 0) {
- qError("failed to get next log block since %s", terrstr());
+ qError("failed to get next log block since %s, %s", terrstr(), id);
}
}
@@ -1652,15 +1701,12 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
setBlockIntoRes(pInfo, &ret.data, true);
if (pInfo->pRes->info.rows > 0) {
pOperator->status = OP_EXEC_RECV;
- qDebug("queue scan log return %d rows", pInfo->pRes->info.rows);
+ qDebug("queue scan log return %" PRId64 " rows", pInfo->pRes->info.rows);
return pInfo->pRes;
}
} else if (ret.fetchType == FETCH_TYPE__META) {
qError("unexpected ret.fetchType:%d", ret.fetchType);
continue;
- // pTaskInfo->streamInfo.lastStatus = ret.offset;
- // pTaskInfo->streamInfo.metaBlk = ret.meta;
- // return NULL;
} else if (ret.fetchType == FETCH_TYPE__NONE ||
(ret.fetchType == FETCH_TYPE__SEP && pOperator->status == OP_EXEC_RECV)) {
pTaskInfo->streamInfo.lastStatus = ret.offset;
@@ -1672,7 +1718,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
}
}
} else {
- qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.prepareStatus.type);
+ qError("unexpected streamInfo prepare type: %d %s", pTaskInfo->streamInfo.prepareStatus.type, id);
return NULL;
}
}
@@ -1803,8 +1849,18 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
+ printDataBlock(pInfo->pUpdateRes, "recover update");
return pInfo->pUpdateRes;
} break;
+ case STREAM_SCAN_FROM_DELETE_DATA: {
+ generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
+ prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
+ pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
+ copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
+ pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA;
+ printDataBlock(pInfo->pDeleteDataRes, "recover delete");
+ return pInfo->pDeleteDataRes;
+ } break;
case STREAM_SCAN_FROM_DATAREADER_RANGE: {
SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
if (pSDB) {
@@ -1813,7 +1869,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->base.cond.twindows, pInfo->groupId, version);
pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA;
checkUpdateData(pInfo, true, pSDB, false);
- // printDataBlock(pSDB, "stream scan update");
+ printDataBlock(pSDB, "scan recover update");
calBlockTbName(pInfo, pSDB);
return pSDB;
}
@@ -1838,9 +1894,10 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
}
if (pInfo->pCreateTbRes->info.rows > 0) {
pInfo->scanMode = STREAM_SCAN_FROM_RES;
+ printDataBlock(pInfo->pCreateTbRes, "recover createTbl");
return pInfo->pCreateTbRes;
}
- qDebug("stream recover scan get block, rows %d", pInfo->pRecoverRes->info.rows);
+ qDebug("stream recover scan get block, rows %" PRId64, pInfo->pRecoverRes->info.rows);
printDataBlock(pInfo->pRecoverRes, "scan recover");
return pInfo->pRecoverRes;
}
@@ -1996,6 +2053,7 @@ FETCH_NEXT_BLOCK:
copyDataBlock(pInfo->pUpdateRes, pSup->pScanBlock);
blockDataCleanup(pSup->pScanBlock);
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
+ pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA;
return pInfo->pUpdateRes;
}
@@ -2069,7 +2127,7 @@ FETCH_NEXT_BLOCK:
pOperator->resultInfo.totalRows += pBlockInfo->rows;
// printDataBlock(pInfo->pRes, "stream scan");
- qDebug("scan rows: %d", pBlockInfo->rows);
+ qDebug("scan rows: %" PRId64, pBlockInfo->rows);
if (pBlockInfo->rows > 0) {
return pInfo->pRes;
}
@@ -2102,20 +2160,30 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamRawScanInfo* pInfo = pOperator->info;
+ int32_t code = TSDB_CODE_SUCCESS;
pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta
pTaskInfo->streamInfo.metaRsp.metaRsp = NULL;
qDebug("tmqsnap doRawScan called");
if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
- if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) {
+ bool hasNext = false;
+ if (pInfo->dataReader) {
+ code = tsdbNextDataBlock(pInfo->dataReader, &hasNext);
+ if (code) {
+ tsdbReleaseDataBlock(pInfo->dataReader);
+ T_LONG_JMP(pTaskInfo->env, code);
+ }
+ }
+
+ if (pInfo->dataReader && hasNext) {
if (isTaskKilled(pTaskInfo)) {
tsdbReleaseDataBlock(pInfo->dataReader);
- longjmp(pTaskInfo->env, pTaskInfo->code);
+ T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL);
if (pBlock == NULL) {
- longjmp(pTaskInfo->env, terrno);
+ T_LONG_JMP(pTaskInfo->env, terrno);
}
qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.id.uid);
@@ -2214,6 +2282,7 @@ static void destroyRawScanOperatorInfo(void* param) {
SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param;
tsdbReaderClose(pRawScan->dataReader);
destroySnapContext(pRawScan->sContext);
+ tableListDestroy(pRawScan->pTableListInfo);
taosMemoryFree(pRawScan);
}
@@ -2235,6 +2304,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT
goto _end;
}
+ pInfo->pTableListInfo = tableListCreate();
pInfo->vnode = pHandle->vnode;
pInfo->sContext = pHandle->sContext;
@@ -2253,9 +2323,11 @@ _end:
static void destroyStreamScanOperatorInfo(void* param) {
SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param;
+
if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) {
destroyOperatorInfo(pStreamScan->pTableScanOp);
}
+
if (pStreamScan->tqReader) {
tqCloseReader(pStreamScan->tqReader);
}
@@ -2282,13 +2354,14 @@ static void destroyStreamScanOperatorInfo(void* param) {
}
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
- SExecTaskInfo* pTaskInfo) {
+ STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
SArray* pColIds = NULL;
SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ tableListDestroy(pTableListInfo);
goto _error;
}
@@ -2302,6 +2375,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
int32_t code =
extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
if (code != TSDB_CODE_SUCCESS) {
+ tableListDestroy(pTableListInfo);
goto _error;
}
@@ -2321,11 +2395,14 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
SExprInfo* pSubTableExpr = taosMemoryCalloc(1, sizeof(SExprInfo));
if (pSubTableExpr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ tableListDestroy(pTableListInfo);
goto _error;
}
+
pInfo->tbnameCalSup.pExprInfo = pSubTableExpr;
createExprFromOneNode(pSubTableExpr, pTableScanNode->pSubtable, 0);
if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1) != 0) {
+ tableListDestroy(pTableListInfo);
goto _error;
}
}
@@ -2335,10 +2412,12 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
SExprInfo* pTagExpr = createExpr(pTableScanNode->pTags, &numOfTags);
if (pTagExpr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ tableListDestroy(pTableListInfo);
goto _error;
}
if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ tableListDestroy(pTableListInfo);
goto _error;
}
}
@@ -2346,11 +2425,12 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData));
if (pInfo->pBlockLists == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ tableListDestroy(pTableListInfo);
goto _error;
}
if (pHandle->vnode) {
- SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
+ SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo);
STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info;
if (pHandle->version > 0) {
pTSInfo->base.cond.endVersion = pHandle->version;
@@ -2358,7 +2438,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
STableKeyInfo* pList = NULL;
int32_t num = 0;
- tableListGetGroupList(pTaskInfo->pTableInfoList, 0, &pList, &num);
+ tableListGetGroupList(pTableListInfo, 0, &pList, &num);
if (pHandle->initTableReader) {
pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
@@ -2389,16 +2469,18 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
// set the extract column id to streamHandle
tqReaderSetColIdList(pInfo->tqReader, pColIds);
- SArray* tableIdList = extractTableIdList(pTaskInfo->pTableInfoList);
+ SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo);
code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList);
if (code != 0) {
taosArrayDestroy(tableIdList);
goto _error;
}
+
taosArrayDestroy(tableIdList);
memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->base.cond, sizeof(SQueryTableDataCond));
} else {
taosArrayDestroy(pColIds);
+ tableListDestroy(pTableListInfo);
pColIds = NULL;
}
@@ -2433,7 +2515,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pTaskInfo);
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
- __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan;
+ __optr_fn_t nextFn = (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) ? doStreamScan : doQueueScan;
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL);
@@ -2464,7 +2546,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
SSDataBlock* pRes = pInfo->pRes;
blockDataCleanup(pRes);
- int32_t size = tableListGetSize(pTaskInfo->pTableInfoList);
+ int32_t size = tableListGetSize(pInfo->pTableListInfo);
if (size == 0) {
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL;
@@ -2476,7 +2558,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
metaReaderInit(&mr, pInfo->readHandle.meta, 0);
while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) {
- STableKeyInfo* item = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->curPos);
+ STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos);
int32_t code = metaGetTableEntryByUid(&mr, item->uid);
tDecoderClear(&mr.coder);
if (code != TSDB_CODE_SUCCESS) {
@@ -2537,11 +2619,12 @@ static void destroyTagScanOperatorInfo(void* param) {
STagScanInfo* pInfo = (STagScanInfo*)param;
pInfo->pRes = blockDataDestroy(pInfo->pRes);
taosArrayDestroy(pInfo->matchInfo.pList);
+ pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo);
taosMemoryFreeClear(param);
}
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
- SExecTaskInfo* pTaskInfo) {
+ STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@@ -2563,6 +2646,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
goto _error;
}
+ pInfo->pTableListInfo = pTableListInfo;
pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->readHandle = *pReadHandle;
pInfo->curPos = 0;
@@ -2596,20 +2680,33 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx);
int64_t st = taosGetTimestampUs();
- void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
+ void* p = tableListGetInfo(pInfo->base.pTableListInfo, readIdx + pInfo->tableStartIndex);
SReadHandle* pHandle = &pInfo->base.readHandle;
if (NULL == source->dataReader || !source->multiReader) {
- code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo));
+ code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false);
if (code != 0) {
T_LONG_JMP(pTaskInfo->env, code);
}
}
-
+
pInfo->base.dataReader = source->dataReader;
STsdbReader* reader = pInfo->base.dataReader;
+ bool hasNext = false;
qTrace("tsdb/read-table-data: %p, enter next reader", reader);
- while (tsdbNextDataBlock(reader)) {
+
+ while (true) {
+ code = tsdbNextDataBlock(reader, &hasNext);
+ if (code != 0) {
+ tsdbReleaseDataBlock(reader);
+ pInfo->base.dataReader = NULL;
+ T_LONG_JMP(pTaskInfo->env, code);
+ }
+
+ if (!hasNext) {
+ break;
+ }
+
if (isTaskKilled(pTaskInfo)) {
tsdbReleaseDataBlock(reader);
pInfo->base.dataReader = NULL;
@@ -2640,7 +2737,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
continue;
}
- pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
+ pBlock->info.id.groupId = getTableGroupId(pInfo->base.pTableListInfo, pBlock->info.id.uid);
pOperator->resultInfo.totalRows += pBlock->info.rows;
pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
@@ -2696,10 +2793,10 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
{
- size_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList);
+ size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo);
int32_t i = pInfo->tableStartIndex + 1;
for (; i < numOfTables; ++i) {
- STableKeyInfo* tableKeyInfo = tableListGetInfo(pTaskInfo->pTableInfoList, i);
+ STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i);
if (tableKeyInfo->groupId != pInfo->groupId) {
break;
}
@@ -2814,7 +2911,7 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock*
}
bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo);
- qDebug("%s get sorted row block, rows:%d, limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows,
+ qDebug("%s get sorted row block, rows:%" PRId64 ", limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows,
pInfo->limitInfo.numOfOutputRows);
return (pResBlock->info.rows > 0) ? pResBlock : NULL;
@@ -2833,7 +2930,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, code);
}
- size_t tableListSize = tableListGetSize(pTaskInfo->pTableInfoList);
+ size_t tableListSize = tableListGetSize(pInfo->base.pTableListInfo);
if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true;
@@ -2842,7 +2939,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
return NULL;
}
pInfo->tableStartIndex = 0;
- pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex))->groupId;
+ pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex))->groupId;
startGroupTableMergeScan(pOperator);
}
@@ -2867,9 +2964,9 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
}
pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
- pInfo->groupId = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex)->groupId;
+ pInfo->groupId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex)->groupId;
startGroupTableMergeScan(pOperator);
- resetLimitInfoForNextGroup(&pInfo->limitInfo);
+ resetLimitInfoForNextGroup(&pInfo->limitInfo);
}
}
@@ -2900,20 +2997,14 @@ void destroyTableMergeScanOperatorInfo(void* param) {
SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i);
taosMemoryFree(pCond->colList);
}
- taosArrayDestroy(pTableScanInfo->queryConds);
- if (pTableScanInfo->base.matchInfo.pList != NULL) {
- taosArrayDestroy(pTableScanInfo->base.matchInfo.pList);
- }
+ taosArrayDestroy(pTableScanInfo->queryConds);
+ destroyTableScanBase(&pTableScanInfo->base);
pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock);
pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock);
taosArrayDestroy(pTableScanInfo->pSortInfo);
- cleanupExprSupp(&pTableScanInfo->base.pseudoSup);
-
- taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache);
-
taosMemoryFreeClear(param);
}
@@ -2932,7 +3023,7 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
}
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
- SExecTaskInfo* pTaskInfo) {
+ STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@@ -2974,6 +3065,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pInfo->base.limitInfo.limit.limit = -1;
pInfo->base.limitInfo.slimit.limit = -1;
+ pInfo->base.pTableListInfo = pTableListInfo;
pInfo->sample.sampleRatio = pTableScanNode->ratio;
pInfo->sample.seed = taosGetTimestampSec();
@@ -3234,7 +3326,7 @@ static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountSca
} else {
groupId = calcGroupId("", 0);
}
-
+
pRes->info.id.groupId = groupId;
fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
} else if (pInfo->currGrpIdx == 1) {
@@ -3346,7 +3438,7 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S
if (pSupp->groupByDbName) {
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
}
-
+
uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
pRes->info.id.groupId = groupId;
int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta);
@@ -3366,7 +3458,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S
} else {
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", stbName);
}
-
+
uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
pRes->info.id.groupId = groupId;
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 90c7fa10ca..cb0f1aa068 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -698,7 +698,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
pDataBlock->info.dataLoad = 1;
}
- qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId,
+ qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%" PRId64 , GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId,
pDataBlock->info.rows);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c
index f24d3523c8..1abe678ac6 100644
--- a/source/libs/executor/src/sysscanoperator.c
+++ b/source/libs/executor/src/sysscanoperator.c
@@ -83,10 +83,11 @@ typedef struct MergeIndex {
} MergeIndex;
typedef struct SBlockDistInfo {
- SSDataBlock* pResBlock;
- STsdbReader* pHandle;
- SReadHandle readHandle;
- uint64_t uid; // table uid
+ SSDataBlock* pResBlock;
+ STsdbReader* pHandle;
+ SReadHandle readHandle;
+ STableListInfo* pTableListInfo;
+ uint64_t uid; // table uid
} SBlockDistInfo;
static int32_t sysChkFilter__Comm(SNode* pNode);
@@ -1627,7 +1628,7 @@ static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScan
char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(varTbName, name);
- colDataSetNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows);
+ colDataSetNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows, true);
}
doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL);
@@ -2214,6 +2215,7 @@ static void destroyBlockDistScanOperatorInfo(void* param) {
SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param;
blockDataDestroy(pDistInfo->pResBlock);
tsdbReaderClose(pDistInfo->pHandle);
+ tableListDestroy(pDistInfo->pTableListInfo);
taosMemoryFreeClear(param);
}
@@ -2245,7 +2247,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC
}
SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode,
- SExecTaskInfo* pTaskInfo) {
+ STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@@ -2263,11 +2265,11 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
goto _error;
}
- STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
- size_t num = tableListGetSize(pTableListInfo);
- void* pList = tableListGetInfo(pTableListInfo, 0);
+ pInfo->pTableListInfo = pTableListInfo;
+ size_t num = tableListGetSize(pTableListInfo);
+ void* pList = tableListGetInfo(pTableListInfo, 0);
- code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str);
+ code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false);
cleanupQueryTableDataCond(&cond);
if (code != 0) {
goto _error;
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 0f2eb4e0d7..987a82436f 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -128,8 +128,9 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn
if (end >= 0) {
forwardRows = end;
- if (pData[end + pos] == ekey) {
+ while (pData[end + pos] == ekey) {
forwardRows += 1;
+ ++pos;
}
}
} else {
@@ -137,8 +138,9 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn
if (end >= 0) {
forwardRows = end;
- if (pData[end + pos] == ekey) {
+ while (pData[end + pos] == ekey) {
forwardRows += 1;
+ ++pos;
}
}
// int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
@@ -936,6 +938,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
}
+
TSKEY ekey = ascScan ? win.ekey : win.skey;
int32_t forwardRows =
getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
@@ -2110,10 +2113,12 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3
} else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
if (code != TSDB_CODE_SUCCESS) {
- qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
- pTaskInfo->code = code;
- T_LONG_JMP(pTaskInfo->env, code);
+ qError("%s apply combine functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
}
+ } else if (pDestCtx[k].fpSet.combine == NULL) {
+ char* funName = fmGetFuncName(pDestCtx[k].functionId);
+ qError("%s error, combine funcion for %s is not implemented", GET_TASKID(pTaskInfo), funName);
+ taosMemoryFreeClear(funName);
}
}
}
@@ -2171,13 +2176,6 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, S
}
}
-bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
- SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
- SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
- GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
- return p1 == NULL;
-}
-
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) {
if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {
SWinKey key = {.ts = pWin->skey, .groupId = groupId};
@@ -2769,6 +2767,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
+ pInfo->ignoreExpiredDataSaved = false;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
pInfo->delIndex = 0;
pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
@@ -3587,6 +3586,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
pInfo->isFinal = false;
pInfo->pPhyNode = pPhyNode;
pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
+ pInfo->ignoreExpiredDataSaved = false;
pInfo->pUpdated = NULL;
pInfo->pStUpdated = NULL;
@@ -4112,6 +4112,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
pInfo->pChildren = NULL;
pInfo->ignoreExpiredData = pStateNode->window.igExpired;
+ pInfo->ignoreExpiredDataSaved = false;
pInfo->pUpdated = NULL;
pInfo->pSeUpdated = NULL;
@@ -4885,6 +4886,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->interval = interval;
pInfo->twAggSup = twAggSupp;
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
+ pInfo->ignoreExpiredDataSaved = false;
pInfo->isFinal = false;
SExprSupp* pSup = &pOperator->exprSupp;
diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h
index dc884a0581..c3afc30a7b 100644
--- a/source/libs/function/inc/builtinsimpl.h
+++ b/source/libs/function/inc/builtinsimpl.h
@@ -235,6 +235,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t groupKeyFunction(SqlFunctionCtx* pCtx);
int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
#ifdef __cplusplus
}
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index e2b68a8b05..a36204bab7 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -481,14 +481,16 @@ static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return code;
}
- pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
+ pFunc->node.resType =
+ (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
return TSDB_CODE_SUCCESS;
}
static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
- pFunc->node.resType = (SDataType){.bytes =tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
+ pFunc->node.resType =
+ (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
return TSDB_CODE_SUCCESS;
}
@@ -510,13 +512,11 @@ static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
-
uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_NUMERIC_TYPE(para1Type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
-
for (int32_t i = 1; i < numOfParams; ++i) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, i);
pValue->notReserved = true;
@@ -2376,7 +2376,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "spread",
.type = FUNCTION_TYPE_SPREAD,
- .classification = FUNC_MGT_AGG_FUNC,
+ .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.translateFunc = translateSpread,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getSpreadFuncEnv,
@@ -2418,7 +2418,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "elapsed",
.type = FUNCTION_TYPE_ELAPSED,
- .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
+ .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.dataRequiredFunc = statisDataRequired,
.translateFunc = translateElapsed,
.getEnvFunc = getElapsedFuncEnv,
@@ -2458,7 +2458,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "interp",
.type = FUNCTION_TYPE_INTERP,
.classification = FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
- FUNC_MGT_FORBID_STREAM_FUNC,
+ FUNC_MGT_FORBID_STREAM_FUNC|FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateInterp,
.getEnvFunc = getSelectivityFuncEnv,
.initFunc = functionSetup,
@@ -2481,7 +2481,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "irate",
.type = FUNCTION_TYPE_IRATE,
- .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+ .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
.translateFunc = translateIrate,
.getEnvFunc = getIrateFuncEnv,
.initFunc = irateFuncSetup,
@@ -2492,7 +2492,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "last_row",
.type = FUNCTION_TYPE_LAST_ROW,
- .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+ .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateFirstLast,
.dynDataRequiredFunc = lastDynDataReq,
.getEnvFunc = getFirstLastFuncEnv,
@@ -2501,7 +2501,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = firstLastScalarFunction,
.pPartialFunc = "_last_row_partial",
.pMergeFunc = "_last_row_merge",
- .finalizeFunc = firstLastFinalize
+ .finalizeFunc = firstLastFinalize,
+ .combineFunc = lastCombine
},
{
.name = "_cache_last_row",
@@ -2810,7 +2811,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "mode",
.type = FUNCTION_TYPE_MODE,
- .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC,
+ .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
.translateFunc = translateMode,
.getEnvFunc = getModeFuncEnv,
.initFunc = modeFunctionSetup,
@@ -3213,7 +3214,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_block_dist",
.type = FUNCTION_TYPE_BLOCK_DIST,
- .classification = FUNC_MGT_AGG_FUNC,
+ .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
.translateFunc = translateBlockDistFunc,
.getEnvFunc = getBlockDistFuncEnv,
.initFunc = blockDistSetup,
@@ -3235,6 +3236,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = functionSetup,
.processFunc = groupKeyFunction,
.finalizeFunc = groupKeyFinalize,
+ .combineFunc = groupKeyCombine,
.pPartialFunc = "_group_key",
.pMergeFunc = "_group_key"
},
@@ -3277,7 +3279,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_irowts",
.type = FUNCTION_TYPE_IROWTS,
- .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC,
+ .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC|FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateTimePseudoColumn,
.getEnvFunc = getTimePseudoFuncEnv,
.initFunc = NULL,
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 4760358e0d..01f81c9a99 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -490,8 +490,8 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true;
}
-static int32_t getNumOfElems(SqlFunctionCtx* pCtx) {
- int32_t numOfElem = 0;
+static int64_t getNumOfElems(SqlFunctionCtx* pCtx) {
+ int64_t numOfElem = 0;
/*
* 1. column data missing (schema modified) causes pInputCol->hasNull == true. pInput->colDataSMAIsSet == true;
@@ -524,7 +524,7 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) {
* count function does not use the pCtx->interResBuf to keep the intermediate buffer
*/
int32_t countFunction(SqlFunctionCtx* pCtx) {
- int32_t numOfElem = 0;
+ int64_t numOfElem = 0;
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SInputColumnInfoData* pInput = &pCtx->input;
@@ -551,7 +551,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) {
}
int32_t countInvertFunction(SqlFunctionCtx* pCtx) {
- int32_t numOfElem = getNumOfElems(pCtx);
+ int64_t numOfElem = getNumOfElems(pCtx);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
char* buf = GET_ROWCELL_INTERBUF(pResInfo);
@@ -867,6 +867,12 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
int32_t dstSlotId = pc->pExpr->base.resSchema.slotId;
+ // group_key function has its own process function
+ // do not process there
+ if (fmIsGroupKeyFunc(pc->functionId)) {
+ continue;
+ }
+
SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId);
if (nullList[j]) {
colDataSetNULL(pDstCol, rowIndex);
@@ -1925,7 +1931,7 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) {
SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
- qDebug("%s total %d rows will merge, %p", __FUNCTION__, pInput->numOfRows, pInfo->pHisto);
+ qDebug("%s total %" PRId64 " rows will merge, %p", __FUNCTION__, pInput->numOfRows, pInfo->pHisto);
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
@@ -3087,6 +3093,12 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid
for (int32_t i = 0; i < pSubsidiaryies->num; ++i) {
SqlFunctionCtx* pc = pSubsidiaryies->pCtx[i];
+ // group_key function has its own process function
+ // do not process there
+ if (fmIsGroupKeyFunc(pc->functionId)) {
+ continue;
+ }
+
SFunctParam* pFuncParam = &pc->pExpr->base.pParam[0];
int32_t srcSlotId = pFuncParam->pCol->slotId;
@@ -5925,6 +5937,39 @@ int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return pResInfo->numOfRes;
}
+int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
+ SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
+ SGroupKeyInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
+
+ SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
+ SGroupKeyInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
+
+ // escape rest of data blocks to avoid first entry to be overwritten.
+ if (pDBuf->hasResult) {
+ goto _group_key_over;
+ }
+
+ if (pSBuf->isNull) {
+ pDBuf->isNull = true;
+ pDBuf->hasResult = true;
+ goto _group_key_over;
+ }
+
+ if (IS_VAR_DATA_TYPE(pSourceCtx->resDataInfo.type)) {
+ memcpy(pDBuf->data, pSBuf->data,
+ (pSourceCtx->resDataInfo.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(pSBuf->data) : varDataTLen(pSBuf->data));
+ } else {
+ memcpy(pDBuf->data, pSBuf->data, pSourceCtx->resDataInfo.bytes);
+ }
+
+ pDBuf->hasResult = true;
+
+_group_key_over:
+
+ SET_VAL(pDResInfo, 1, 1);
+ return TSDB_CODE_SUCCESS;
+}
+
int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElems = 0;
diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c
index e3127fcd7b..94ab616dda 100644
--- a/source/libs/function/src/functionMgt.c
+++ b/source/libs/function/src/functionMgt.c
@@ -447,3 +447,10 @@ int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc
return code;
}
+
+char* fmGetFuncName(int32_t funcId) {
+ if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) {
+ return taosStrdup("invalid function");
+ }
+ return taosStrdup(funcMgtBuiltins[funcId].name);
+}
diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c
index 40c50ed9cb..9e7ed52104 100644
--- a/source/libs/index/src/indexFstFile.c
+++ b/source/libs/index/src/indexFstFile.c
@@ -127,8 +127,6 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
blk->blockId = blkId;
blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize);
ASSERTS(blk->nread <= kBlockSize, "index read incomplete data");
- if (blk->nread > kBlockSize) break;
-
if (blk->nread < kBlockSize && blk->nread < len) {
taosMemoryFree(blk);
break;
diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c
index b2fc88add1..c25d0e7036 100644
--- a/source/libs/parser/src/parCalcConst.c
+++ b/source/libs/parser/src/parCalcConst.c
@@ -183,16 +183,18 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) {
} else {
code = scalarCalculateConstants(pProject, pNew);
}
- if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) {
+ if (TSDB_CODE_SUCCESS == code) {
strcpy(((SExprNode*)*pNew)->aliasName, aliasName);
- int32_t size = taosArrayGetSize(pAssociation);
- for (int32_t i = 0; i < size; ++i) {
- SNode** pCol = taosArrayGetP(pAssociation, i);
- nodesDestroyNode(*pCol);
- *pCol = nodesCloneNode(*pNew);
- if (NULL == *pCol) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- break;
+ if (QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) {
+ int32_t size = taosArrayGetSize(pAssociation);
+ for (int32_t i = 0; i < size; ++i) {
+ SNode** pCol = taosArrayGetP(pAssociation, i);
+ nodesDestroyNode(*pCol);
+ *pCol = nodesCloneNode(*pNew);
+ if (NULL == *pCol) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
}
}
}
@@ -335,8 +337,14 @@ static SNodeList* getChildProjection(SNode* pStmt) {
static void eraseSetOpChildProjection(SSetOperator* pSetOp, int32_t index) {
SNodeList* pLeftProjs = getChildProjection(pSetOp->pLeft);
nodesListErase(pLeftProjs, nodesListGetCell(pLeftProjs, index));
+ if (QUERY_NODE_SET_OPERATOR == nodeType(pSetOp->pLeft)) {
+ eraseSetOpChildProjection((SSetOperator*)pSetOp->pLeft, index);
+ }
SNodeList* pRightProjs = getChildProjection(pSetOp->pRight);
nodesListErase(pRightProjs, nodesListGetCell(pRightProjs, index));
+ if (QUERY_NODE_SET_OPERATOR == nodeType(pSetOp->pRight)) {
+ eraseSetOpChildProjection((SSetOperator*)pSetOp->pRight, index);
+ }
}
typedef struct SNotRefByOrderByCxt {
@@ -361,11 +369,30 @@ static bool notRefByOrderBy(SColumnNode* pCol, SNodeList* pOrderByList) {
return !cxt.hasThisCol;
}
+static bool isSetUselessCol(SSetOperator* pSetOp, int32_t index, SExprNode* pProj) {
+ if (!isUselessCol(pProj)) {
+ return false;
+ }
+
+ SNodeList* pLeftProjs = getChildProjection(pSetOp->pLeft);
+ if (!isUselessCol((SExprNode*)nodesListGetNode(pLeftProjs, index))) {
+ return false;
+ }
+
+ SNodeList* pRightProjs = getChildProjection(pSetOp->pRight);
+ if (!isUselessCol((SExprNode*)nodesListGetNode(pRightProjs, index))) {
+ return false;
+ }
+
+ return true;
+}
+
static int32_t calcConstSetOpProjections(SCalcConstContext* pCxt, SSetOperator* pSetOp, bool subquery) {
int32_t index = 0;
SNode* pProj = NULL;
WHERE_EACH(pProj, pSetOp->pProjectionList) {
- if (subquery && notRefByOrderBy((SColumnNode*)pProj, pSetOp->pOrderByList) && isUselessCol((SExprNode*)pProj)) {
+ if (subquery && notRefByOrderBy((SColumnNode*)pProj, pSetOp->pOrderByList) &&
+ isSetUselessCol(pSetOp, index, (SExprNode*)pProj)) {
ERASE_NODE(pSetOp->pProjectionList);
eraseSetOpChildProjection(pSetOp, index);
continue;
diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c
index 106ee641af..0bcd777d2a 100644
--- a/source/libs/parser/src/parInsertSml.c
+++ b/source/libs/parser/src/parInsertSml.c
@@ -25,6 +25,8 @@ static void clearColValArray(SArray* pCols) {
if (TSDB_DATA_TYPE_NCHAR == pCol->type) {
taosMemoryFreeClear(pCol->value.pData);
}
+ pCol->flag = CV_FLAG_NONE;
+ pCol->value.val = 0;
}
}
diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c
index 704f381afa..132a3b2618 100644
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@@ -566,53 +566,19 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList,
return code;
}
-static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD* fields, int numFields) {
- bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool));
- if (NULL == pUseCols) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
-
- pBoundInfo->numOfBound = 0;
-
- int16_t lastColIdx = -1; // last column found
- int32_t code = TSDB_CODE_SUCCESS;
+static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) {
for (int i = 0; i < numFields; i++) {
- SToken token;
- token.z = fields[i].name;
- token.n = strlen(fields[i].name);
-
- int16_t t = lastColIdx + 1;
- int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema);
- if (index < 0 && t > 0) {
- index = insFindCol(&token, 0, t, pSchema);
- }
- if (index < 0) {
- uError("can not find column name:%s", token.z);
- code = TSDB_CODE_PAR_INVALID_COLUMN;
- break;
- } else if (pUseCols[index]) {
- code = TSDB_CODE_PAR_INVALID_COLUMN;
- uError("duplicated column name:%s", token.z);
- break;
- } else {
- lastColIdx = index;
- pUseCols[index] = true;
- pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index;
- ++pBoundInfo->numOfBound;
+ if(strcmp(pSchema->name, fields[i].name) == 0){
+ return true;
}
}
- if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) {
- uError("primary timestamp column can not be null:");
- code = TSDB_CODE_PAR_INVALID_COLUMN;
- }
-
- taosMemoryFree(pUseCols);
- return code;
+ return false;
}
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields,
int numFields, bool needChangeLength) {
+ void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid));
STableDataCxt* pTableCxt = NULL;
int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true);
@@ -620,19 +586,14 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
uError("insGetTableDataCxt error");
goto end;
}
- if (tFields != NULL) {
- ret = bindFileds(&pTableCxt->boundColsInfo, getTableColumnSchema(pTableMeta), tFields, numFields);
+
+ if(tmp == NULL){
+ ret = initTableColSubmitData(pTableCxt);
if (ret != TSDB_CODE_SUCCESS) {
- uError("bindFileds error");
+ uError("initTableColSubmitData error");
goto end;
}
}
- // no need to bind, because select * get all fields
- ret = initTableColSubmitData(pTableCxt);
- if (ret != TSDB_CODE_SUCCESS) {
- uError("initTableColSubmitData error");
- goto end;
- }
char* p = (char*)data;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column
@@ -660,35 +621,83 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta);
SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo;
- if (boundInfo->numOfBound != numOfCols) {
- uError("boundInfo->numOfBound:%d != numOfCols:%d", boundInfo->numOfBound, numOfCols);
+ if (tFields != NULL && numFields != numOfCols) {
+ uError("numFields:%d != numOfCols:%d", numFields, numOfCols);
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
- for (int c = 0; c < boundInfo->numOfBound; ++c) {
- SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]];
- SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c);
+ if (tFields != NULL && numFields > boundInfo->numOfBound) {
+ uError("numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound);
+ ret = TSDB_CODE_INVALID_PARA;
+ goto end;
+ }
+ if(tFields == NULL){
+ for (int j = 0; j < boundInfo->numOfBound; j++){
+ SSchema* pColSchema = &pSchema[j];
+ SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j);
+ if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
+ uError("type or bytes not equal");
+ ret = TSDB_CODE_INVALID_PARA;
+ goto end;
+ }
+
+ int8_t* offset = pStart;
+ if (IS_VAR_DATA_TYPE(pColSchema->type)) {
+ pStart += numOfRows * sizeof(int32_t);
+ } else {
+ pStart += BitmapLen(numOfRows);
+ }
+ char* pData = pStart;
+
+ tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData);
+ fields += sizeof(int8_t) + sizeof(int32_t);
+ if (needChangeLength) {
+ pStart += htonl(colLength[j]);
+ } else {
+ pStart += colLength[j];
+ }
+ }
+ }else{
+ for (int i = 0; i < numFields; i++) {
+ for (int j = 0; j < boundInfo->numOfBound; j++){
+ SSchema* pColSchema = &pSchema[j];
+ if(strcmp(pColSchema->name, tFields[i].name) == 0){
+ if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
+ uError("type or bytes not equal");
+ ret = TSDB_CODE_INVALID_PARA;
+ goto end;
+ }
+
+ int8_t* offset = pStart;
+ if (IS_VAR_DATA_TYPE(pColSchema->type)) {
+ pStart += numOfRows * sizeof(int32_t);
+ } else {
+ pStart += BitmapLen(numOfRows);
+ }
+ char* pData = pStart;
+
+ SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j);
+ tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData);
+ fields += sizeof(int8_t) + sizeof(int32_t);
+ if (needChangeLength) {
+ pStart += htonl(colLength[i]);
+ } else {
+ pStart += colLength[i];
+ }
+ boundInfo->pColIndex[j] = -1;
+ break;
+ }
+ }
- if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
- uError("type or bytes not equal");
- ret = TSDB_CODE_INVALID_PARA;
- goto end;
}
- int8_t* offset = pStart;
- if (IS_VAR_DATA_TYPE(pColSchema->type)) {
- pStart += numOfRows * sizeof(int32_t);
- } else {
- pStart += BitmapLen(numOfRows);
- }
- char* pData = pStart;
-
- tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData);
- fields += sizeof(int8_t) + sizeof(int32_t);
- if (needChangeLength) {
- pStart += htonl(colLength[c]);
- } else {
- pStart += colLength[c];
+ for (int c = 0; c < boundInfo->numOfBound; ++c) {
+ if( boundInfo->pColIndex[c] != -1){
+ SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c);
+ tColDataAddValueByDataBlock(pCol, 0, 0, numOfRows, NULL, NULL);
+ }else{
+ boundInfo->pColIndex[c] = c; // restore for next block
+ }
}
}
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index f0a5c275ea..78fc097f6f 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -755,9 +755,11 @@ static bool isPrimaryKeyImpl(SNode* pExpr) {
} else if (QUERY_NODE_FUNCTION == nodeType(pExpr)) {
SFunctionNode* pFunc = (SFunctionNode*)pExpr;
if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pFunc->funcType ||
- FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType) {
+ FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType ||
+ FUNCTION_TYPE_LAST_ROW == pFunc->funcType) {
return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0));
- } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType) {
+ } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType ||
+ FUNCTION_TYPE_IROWTS == pFunc->funcType) {
return true;
}
}
@@ -1422,6 +1424,9 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
if (isCountStar(pFunc)) {
return rewriteCountStar(pCxt, pFunc);
}
+ if (isCountNotNullValue(pFunc)) {
+ return rewriteCountNotNullValue(pCxt, pFunc);
+ }
if (isCountTbname(pFunc)) {
return rewriteCountTbname(pCxt, pFunc);
}
@@ -1631,13 +1636,15 @@ static bool isTableStar(SNode* pNode) {
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}
+static bool isStarParam(SNode* pNode) { return isStar(pNode) || isTableStar(pNode); }
+
static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
if (!fmIsMultiResFunc(pFunc->funcId)) {
return TSDB_CODE_SUCCESS;
}
if (SQL_CLAUSE_SELECT != pCxt->currClause) {
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
- if (isStar(pPara) || isTableStar(pPara)) {
+ if (isStarParam(pPara)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
"%s(*) is only supported in SELECTed list", pFunc->functionName);
}
@@ -1651,7 +1658,7 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu
static int32_t getMultiResFuncNum(SNodeList* pParameterList) {
if (1 == LIST_LENGTH(pParameterList)) {
- return isStar(nodesListGetNode(pParameterList, 0)) ? 2 : 1;
+ return isStarParam(nodesListGetNode(pParameterList, 0)) ? 2 : 1;
}
return LIST_LENGTH(pParameterList);
}
@@ -3114,6 +3121,19 @@ static const char* getPrecisionStr(uint8_t precision) {
return "unknown";
}
+static void convertVarDuration(SValueNode* pOffset, uint8_t precision) {
+ const int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
+ const int8_t units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND};
+
+ if (pOffset->unit == 'n') {
+ pOffset->datum.i = pOffset->datum.i * 31 * (NANOSECOND_PER_DAY / factors[precision]);
+ } else {
+ pOffset->datum.i = pOffset->datum.i * 365 * (NANOSECOND_PER_DAY / factors[precision]);
+ }
+
+ pOffset->unit = units[precision];
+}
+
static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) {
uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision;
@@ -3138,6 +3158,10 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG);
}
+
+ if (pOffset->unit == 'n' || pOffset->unit == 'y') {
+ convertVarDuration(pOffset, precision);
+ }
}
if (NULL != pInterval->pSliding) {
@@ -5922,11 +5946,15 @@ static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, STabl
return code;
}
+static bool isEventWindowQuery(SSelectStmt* pSelect) {
+ return NULL != pSelect->pWindow && QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow);
+}
+
static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type ||
!pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList ||
- crossTableWithUdaf(pSelect)) {
+ crossTableWithUdaf(pSelect) || isEventWindowQuery(pSelect)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
}
if (NULL != pSelect->pSubtable && TSDB_DATA_TYPE_VARCHAR != ((SExprNode*)pSelect->pSubtable)->resType.type) {
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 793d05721e..563bc5e780 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -444,7 +444,7 @@ static int32_t getInsTagsTableTargetNameFromOp(int32_t acctId, SOperatorNode* pO
} else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) {
pVal = (SValueNode*)pOper->pRight;
}
- if (NULL == pCol || NULL == pVal) {
+ if (NULL == pCol || NULL == pVal || NULL == pVal->literal || 0 == strcmp(pVal->literal, "")) {
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index 16a5ef7bae..8fee17d968 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -2279,7 +2279,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
if (NULL != cxt.pLastCols) {
cxt.doAgg = false;
lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols);
- nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt);
+ NODES_DESTORY_LIST(pScan->pScanPseudoCols);
lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols);
nodesClearList(cxt.pLastCols);
}
diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c
index e9a2efaac7..e2c2e4c655 100644
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@@ -1089,9 +1089,15 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic
pExchange->srcStartGroupId = pExchangeLogicNode->srcStartGroupId;
pExchange->srcEndGroupId = pExchangeLogicNode->srcEndGroupId;
pExchange->seqRecvData = pExchangeLogicNode->seqRecvData;
- *pPhyNode = (SPhysiNode*)pExchange;
- return TSDB_CODE_SUCCESS;
+ int32_t code = setConditionsSlotId(pCxt, (const SLogicNode*)pExchangeLogicNode, (SPhysiNode*)pExchange);
+ if (TSDB_CODE_SUCCESS == code) {
+ *pPhyNode = (SPhysiNode*)pExchange;
+ } else {
+ nodesDestroyNode((SNode*)pExchange);
+ }
+
+ return code;
}
static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode,
@@ -1119,6 +1125,9 @@ static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExc
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pScan->pScanCols, pScan->node.pOutputDataBlockDesc);
}
+ if (TSDB_CODE_SUCCESS == code) {
+ code = setConditionsSlotId(pCxt, (const SLogicNode*)pExchangeLogicNode, (SPhysiNode*)pScan);
+ }
if (TSDB_CODE_SUCCESS == code) {
*pPhyNode = (SPhysiNode*)pScan;
diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index 361cf33d58..fd77261818 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -1365,7 +1365,8 @@ static int32_t unAllSplCreateExchangeNode(SSplitContext* pCxt, int32_t startGrou
pExchange->srcEndGroupId = pCxt->groupId - 1;
pExchange->node.precision = pProject->node.precision;
pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets);
- if (NULL == pExchange->node.pTargets) {
+ pExchange->node.pConditions = nodesCloneNode(pProject->node.pConditions);
+ if (NULL == pExchange->node.pTargets || (NULL != pProject->node.pConditions && NULL == pExchange->node.pConditions)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TSWAP(pExchange->node.pLimit, pProject->node.pLimit);
diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c
index c68a08682c..9d8c170003 100644
--- a/source/libs/qcom/src/queryUtil.c
+++ b/source/libs/qcom/src/queryUtil.c
@@ -194,6 +194,8 @@ char* jobTaskStatusStr(int32_t status) {
return "EXECUTING";
case JOB_TASK_STATUS_PART_SUCC:
return "PARTIAL_SUCCEED";
+ case JOB_TASK_STATUS_FETCH:
+ return "FETCHING";
case JOB_TASK_STATUS_SUCC:
return "SUCCEED";
case JOB_TASK_STATUS_FAIL:
diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c
index b8d5d2e6ee..59e63e9eae 100644
--- a/source/libs/qworker/src/qwDbg.c
+++ b/source/libs/qworker/src/qwDbg.c
@@ -259,15 +259,26 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) {
static int32_t ignoreTime = 0;
if (++ignoreTime > 10 && 0 == taosRand() % 9) {
+ if (ctx->msgType == TDMT_SCH_FETCH) {
+ qwBuildAndSendErrorRsp(TDMT_SCH_LINK_BROKEN, &ctx->ctrlConnInfo, TSDB_CODE_RPC_BROKEN_LINK);
+ qwBuildAndSendErrorRsp(ctx->msgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST);
+ *rsped = true;
+
+ taosSsleep(3);
+ return;
+ }
+
+#if 0
SRpcHandleInfo *pConn =
((ctx->msgType == TDMT_SCH_FETCH || ctx->msgType == TDMT_SCH_MERGE_FETCH) ? &ctx->dataConnInfo
- : &ctx->ctrlConnInfo);
+ : &ctx->ctrlConnInfo);
qwBuildAndSendErrorRsp(ctx->msgType + 1, pConn, TSDB_CODE_RPC_BROKEN_LINK);
-
+
qwBuildAndSendDropMsg(QW_FPARAMS(), pConn);
*rsped = true;
-
+
return;
+#endif
}
}
diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c
index ec4049a3eb..92c7852dbc 100644
--- a/source/libs/qworker/src/qworker.c
+++ b/source/libs/qworker/src/qworker.c
@@ -199,7 +199,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
QW_ERR_JRET(code);
}
- QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue);
+ QW_TASK_DLOG("data put into sink, rows:%" PRId64 ", continueExecTask:%d", pRes->info.rows, qcontinue);
}
if (numOfResBlock == 0 || (hasMore == false)) {
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 88362201b4..24b25cec80 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1762,7 +1762,11 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
char* p = colDataGetVarData(pInput->columnData, 0);
- colDataSetNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows);
+ int32_t code = colDataSetNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows, true);
+ if (code) {
+ return code;
+ }
+
pOutput->numOfRows += pInput->numOfRows;
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h
index 85b952937f..d002b5dfa9 100644
--- a/source/libs/scheduler/inc/schInt.h
+++ b/source/libs/scheduler/inc/schInt.h
@@ -193,7 +193,7 @@ typedef struct SSchLevel {
int32_t taskSucceed;
int32_t taskNum;
int32_t taskLaunchedNum;
- int32_t taskDoneNum;
+ int32_t taskExecDoneNum;
SArray *subTasks; // Element is SSchTask
} SSchLevel;
@@ -299,6 +299,7 @@ typedef struct SSchJob {
SExecResult execRes;
void *fetchRes; // TODO free it or not
bool fetched;
+ bool noMoreRetry;
int64_t resNumOfRows; // from int32_t to int64_t
SSchResInfo userRes;
char *sql;
@@ -333,13 +334,16 @@ extern SSchedulerMgmt schMgmt;
((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \
(!SCH_IS_DATA_BIND_QRY_TASK(_task)))
-#define SCH_UPDATE_REDICT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code)
-#define SCH_GET_REDICT_CODE(job, _code) (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode)
+#define SCH_UPDATE_REDIRECT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code)
+#define SCH_GET_REDIRECT_CODE(job, _code) (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode)
#define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st)
#define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status)
#define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task))
+#define SCH_TASK_ALREADY_LAUNCHED(task) (SCH_GET_TASK_STATUS(task) >= JOB_TASK_STATUS_EXEC)
+#define SCH_TASK_EXEC_DONE(task) (SCH_GET_TASK_STATUS(task) >= JOB_TASK_STATUS_PART_SUCC)
+
#define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL)
#define SCH_SET_TASK_HANDLE(_task, _handle) ((_task)->handle = (_handle))
@@ -361,6 +365,7 @@ extern SSchedulerMgmt schMgmt;
(SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level))
#define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH)
#define SCH_TASK_NEED_FETCH(_task) ((_task)->plan->subplanType != SUBPLAN_TYPE_MODIFY)
+#define SCH_MULTI_LEVEL_LAUNCHED(_job) ((_job)->levelIdx != ((_job)->levelNum - 1))
#define SCH_SET_JOB_TYPE(_job, type) \
do { \
@@ -377,16 +382,24 @@ extern SSchedulerMgmt schMgmt;
#define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job))
#define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode)
#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED)
-#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) \
- (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)) || (_task)->redirectCtx.inRedirect))
#define SCH_REDIRECT_MSGTYPE(_msgType) \
((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || \
(_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH)
-#define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) \
- (SCH_REDIRECT_MSGTYPE(_msgType) && \
- (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen))))
-#define SCH_NEED_RETRY(_msgType, _code) \
- ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
+#define SCH_LOW_LEVEL_NETWORK_ERR(_job, _task, _code) \
+ (SCH_NETWORK_ERR(_code) && ((_task)->level->level == (_job)->levelIdx))
+#define SCH_TOP_LEVEL_NETWORK_ERR(_job, _task, _code) \
+ (SCH_NETWORK_ERR(_code) && ((_task)->level->level > (_job)->levelIdx))
+#define SCH_TASK_RETRY_NETWORK_ERR(_task, _code) \
+ (SCH_NETWORK_ERR(_code) && (_task)->redirectCtx.inRedirect)
+
+#define SCH_JOB_NEED_RETRY(_job, _task, _msgType, _code) \
+ (SCH_REDIRECT_MSGTYPE(_msgType) && SCH_TOP_LEVEL_NETWORK_ERR(_job, _task, _code))
+#define SCH_TASKSET_NEED_RETRY(_job, _task, _msgType, _code) \
+ (SCH_REDIRECT_MSGTYPE(_msgType) && \
+ (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_LOW_LEVEL_NETWORK_ERR((_job), (_task), (_code)) || SCH_TASK_RETRY_NETWORK_ERR((_task), (_code))))
+#define SCH_TASK_NEED_RETRY(_msgType, _code) \
+ ((SCH_REDIRECT_MSGTYPE(_msgType) && SCH_NETWORK_ERR(_code)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
+
#define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum)
#define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse])
@@ -510,6 +523,11 @@ extern SSchedulerMgmt schMgmt;
} \
} while (0)
+#define SCH_RESET_JOB_LEVEL_IDX(_job) do { \
+ (_job)->levelIdx = (_job)->levelNum - 1; \
+ SCH_JOB_DLOG("set job levelIdx to %d", (_job)->levelIdx); \
+} while (0)
+
void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask);
void schCleanClusterHb(void *pTrans);
int32_t schLaunchTask(SSchJob *job, SSchTask *task);
@@ -562,7 +580,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq);
int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq);
int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes);
int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet);
-int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode);
+int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode);
void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode);
int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq);
void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode);
@@ -591,6 +609,10 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode);
bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync);
int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode);
int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp);
+int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode);
+int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode);
+void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask);
+int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, int32_t rspCode);
extern SSchDebug gSCHDebug;
diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c
index 5e4fe4b8a1..9cb95a6bbe 100644
--- a/source/libs/scheduler/src/schFlowCtrl.c
+++ b/source/libs/scheduler/src/schFlowCtrl.c
@@ -282,7 +282,6 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) {
}
int32_t code = schLaunchTasksInFlowCtrlListImpl(pJob, ctrl);
- ;
SCH_ERR_RET(code);
return code; // to avoid compiler error
diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c
index 980a8ac6a1..e7bfe95795 100644
--- a/source/libs/scheduler/src/schJob.c
+++ b/source/libs/scheduler/src/schJob.c
@@ -83,6 +83,10 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) {
oriStatus = SCH_GET_JOB_STATUS(pJob);
if (oriStatus == newStatus) {
+ if (JOB_TASK_STATUS_FETCH == newStatus) {
+ return code;
+ }
+
SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR);
}
@@ -108,10 +112,19 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) {
break;
case JOB_TASK_STATUS_PART_SUCC:
if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC &&
- newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) {
+ newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC &&
+ newStatus != JOB_TASK_STATUS_FETCH) {
SCH_ERR_JRET(TSDB_CODE_APP_ERROR);
}
+ break;
+ case JOB_TASK_STATUS_FETCH:
+ if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC &&
+ newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC &&
+ newStatus != JOB_TASK_STATUS_FETCH) {
+ SCH_ERR_JRET(TSDB_CODE_APP_ERROR);
+ }
+
break;
case JOB_TASK_STATUS_SUCC:
case JOB_TASK_STATUS_FAIL:
@@ -288,7 +301,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
}
pJob->levelNum = levelNum;
- pJob->levelIdx = levelNum - 1;
+ SCH_RESET_JOB_LEVEL_IDX(pJob);
SSchLevel level = {0};
SNodeListNode *plans = NULL;
@@ -550,9 +563,9 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) {
}
SSchLevel *pLevel = pTask->level;
- int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1);
+ int32_t doneNum = atomic_add_fetch_32(&pLevel->taskExecDoneNum, 1);
if (doneNum == pLevel->taskNum) {
- pJob->levelIdx--;
+ atomic_sub_fetch_32(&pJob->levelIdx, 1);
pLevel = taosArrayGet(pJob->levels, pJob->levelIdx);
for (int32_t i = 0; i < pLevel->taskNum; ++i) {
@@ -562,6 +575,10 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) {
continue;
}
+ if (SCH_TASK_ALREADY_LAUNCHED(pTask)) {
+ continue;
+ }
+
SCH_ERR_RET(schLaunchTask(pJob, pTask));
}
}
@@ -811,6 +828,75 @@ void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode) {
}
}
+int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) {
+ if (pJob->status >= JOB_TASK_STATUS_PART_SUCC) {
+ SCH_LOCK(SCH_WRITE, &pJob->resLock);
+ if (pJob->fetched) {
+ SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
+ pJob->noMoreRetry = true;
+ SCH_JOB_ELOG("already fetched while got error %s", tstrerror(rspCode));
+ SCH_ERR_RET(rspCode);
+ }
+ SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
+
+ schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC);
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode) {
+ SCH_ERR_RET(schChkResetJobRetry(pJob, rspCode));
+
+ int32_t numOfLevels = taosArrayGetSize(pJob->levels);
+ for (int32_t i = 0; i < numOfLevels; ++i) {
+ SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
+
+ pLevel->taskExecDoneNum = 0;
+ pLevel->taskLaunchedNum = 0;
+
+ int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
+ for (int32_t j = 0; j < numOfTasks; ++j) {
+ SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
+ SCH_LOCK_TASK(pTask);
+ SCH_ERR_RET(schChkUpdateRedirectCtx(pJob, pTask, NULL, rspCode));
+ qClearSubplanExecutionNode(pTask->plan);
+ schResetTaskForRetry(pJob, pTask);
+ SCH_UNLOCK_TASK(pTask);
+ }
+ }
+
+ SCH_RESET_JOB_LEVEL_IDX(pJob);
+
+ return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode) {
+ int32_t code = 0;
+
+ taosMemoryFreeClear(pMsg->pData);
+ taosMemoryFreeClear(pMsg->pEpSet);
+
+ SCH_UNLOCK_TASK(pTask);
+
+ SCH_TASK_DLOG("start to redirect all job tasks cause of error: %s", tstrerror(rspCode));
+
+ SCH_ERR_JRET(schResetJobForRetry(pJob, rspCode));
+
+ SCH_ERR_JRET(schLaunchJob(pJob));
+
+ SCH_LOCK_TASK(pTask);
+
+ SCH_RET(code);
+
+_return:
+
+ SCH_LOCK_TASK(pTask);
+
+ SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
+}
+
bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync) {
bool r = false;
SCH_LOCK(SCH_READ, &pJob->opStatus.lock);
@@ -907,7 +993,7 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq
SCH_ERR_RET(TSDB_CODE_APP_ERROR);
}
- if (status != JOB_TASK_STATUS_PART_SUCC) {
+ if (status != JOB_TASK_STATUS_PART_SUCC && status != JOB_TASK_STATUS_FETCH) {
SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
}
diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index 6f4130fd9f..80fdc7594c 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -36,7 +36,7 @@ int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) {
TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_QW_MSG_ERROR);
}
- if (taskStatus != JOB_TASK_STATUS_PART_SUCC) {
+ if (taskStatus != JOB_TASK_STATUS_FETCH) {
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_QW_MSG_ERROR);
@@ -137,25 +137,12 @@ int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp) {
return TSDB_CODE_SUCCESS;
}
-// Note: no more task error processing, handled in function internal
-int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) {
+int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
char *msg = pMsg->pData;
int32_t msgSize = pMsg->len;
int32_t msgType = pMsg->msgType;
- bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode));
- if (SCH_IS_QUERY_JOB(pJob)) {
- SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId));
- }
-
- SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType));
-
- int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1);
- if (SCH_TASK_NEED_REDIRECT(pTask, reqType, rspCode, pMsg->len)) {
- SCH_RET(schHandleRedirect(pJob, pTask, (SDataBuf *)pMsg, rspCode));
- }
-
pTask->redirectCtx.inRedirect = false;
switch (msgType) {
@@ -423,6 +410,38 @@ _return:
SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
}
+
+
+// Note: no more task error processing, handled in function internal
+int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) {
+ int32_t code = 0;
+ int32_t msgType = pMsg->msgType;
+ char *msg = pMsg->pData;
+
+ bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode));
+ if (SCH_IS_QUERY_JOB(pJob)) {
+ SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId));
+ }
+
+ SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType));
+
+ int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1);
+ if (SCH_JOB_NEED_RETRY(pJob, pTask, reqType, rspCode)) {
+ SCH_RET(schHandleJobRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode));
+ } else if (SCH_TASKSET_NEED_RETRY(pJob, pTask, reqType, rspCode)) {
+ SCH_RET(schHandleTaskSetRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode));
+ }
+
+ pTask->redirectCtx.inRedirect = false;
+
+ SCH_RET(schProcessResponseMsg(pJob, pTask, execId, pMsg, rspCode));
+
+_return:
+
+ taosMemoryFreeClear(msg);
+
+ SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
+}
int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c
index 4c16a81a05..9d0ad30e2a 100644
--- a/source/libs/scheduler/src/schStatus.c
+++ b/source/libs/scheduler/src/schStatus.c
@@ -34,6 +34,9 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) {
case JOB_TASK_STATUS_PART_SUCC:
SCH_ERR_JRET(schProcessOnJobPartialSuccess(pJob));
break;
+ case JOB_TASK_STATUS_FETCH:
+ SCH_ERR_JRET(schJobFetchRows(pJob));
+ break;
case JOB_TASK_STATUS_SUCC:
break;
case JOB_TASK_STATUS_FAIL:
diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c
index bdab739327..207753ae25 100644
--- a/source/libs/scheduler/src/schTask.c
+++ b/source/libs/scheduler/src/schTask.c
@@ -378,7 +378,8 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet,
if (lastTime > tsMaxRetryWaitTime) {
SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d",
nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes);
- SCH_ERR_RET(SCH_GET_REDICT_CODE(pJob, rspCode));
+ pJob->noMoreRetry = true;
+ SCH_ERR_RET(SCH_GET_REDIRECT_CODE(pJob, rspCode));
}
pCtx->periodMs *= tsRedirectFactor;
@@ -404,32 +405,35 @@ _return:
return TSDB_CODE_SUCCESS;
}
+void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) {
+ pTask->waitRetry = true;
+
+ schDropTaskOnExecNode(pJob, pTask);
+ if (pTask->delayTimer) {
+ taosTmrStopA(&pTask->delayTimer);
+ }
+ taosHashClear(pTask->execNodes);
+ schRemoveTaskFromExecList(pJob, pTask);
+ schDeregisterTaskHb(pJob, pTask);
+ taosMemoryFreeClear(pTask->msg);
+ pTask->msgLen = 0;
+ pTask->lastMsgType = 0;
+ pTask->childReady = 0;
+ memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr));
+}
+
int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) {
int32_t code = 0;
SCH_TASK_DLOG("task will be redirected now, status:%s, code:%s", SCH_GET_TASK_STATUS_STR(pTask), tstrerror(rspCode));
- if (NULL == pData) {
- pTask->retryTimes = 0;
- }
-
if (!NO_RET_REDIRECT_ERROR(rspCode)) {
- SCH_UPDATE_REDICT_CODE(pJob, rspCode);
+ SCH_UPDATE_REDIRECT_CODE(pJob, rspCode);
}
SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL, rspCode));
- pTask->waitRetry = true;
-
- schDropTaskOnExecNode(pJob, pTask);
- taosHashClear(pTask->execNodes);
- schRemoveTaskFromExecList(pJob, pTask);
- schDeregisterTaskHb(pJob, pTask);
- atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
- taosMemoryFreeClear(pTask->msg);
- pTask->msgLen = 0;
- pTask->lastMsgType = 0;
- memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr));
+ schResetTaskForRetry(pJob, pTask);
if (SCH_IS_DATA_BIND_TASK(pTask)) {
if (pData && pData->pEpSet) {
@@ -445,12 +449,6 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps);
}
- if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) {
- if (JOB_TASK_STATUS_EXEC == SCH_GET_TASK_STATUS(pTask)) {
- SCH_ERR_JRET(schLaunchTasksInFlowCtrlList(pJob, pTask));
- }
- }
-
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
SCH_ERR_JRET(schDelayLaunchTask(pJob, pTask));
@@ -486,20 +484,10 @@ _return:
SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
}
-int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) {
+int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) {
int32_t code = 0;
- if (JOB_TASK_STATUS_PART_SUCC == pJob->status) {
- SCH_LOCK(SCH_WRITE, &pJob->resLock);
- if (pJob->fetched) {
- SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
- SCH_TASK_ELOG("already fetched while got error %s", tstrerror(rspCode));
- SCH_ERR_JRET(rspCode);
- }
- SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
-
- schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC);
- }
+ SCH_ERR_JRET(schChkResetJobRetry(pJob, rspCode));
if (SYNC_OTHER_LEADER_REDIRECT_ERROR(rspCode)) {
if (NULL == pData->pEpSet) {
@@ -509,7 +497,19 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
}
}
+ SCH_TASK_DLOG("start to redirect current task set cause of error: %s", tstrerror(rspCode));
+
+ for (int32_t i = 0; i < pJob->levelNum; ++i) {
+ SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
+
+ pLevel->taskExecDoneNum = 0;
+ pLevel->taskLaunchedNum = 0;
+ }
+
+ SCH_RESET_JOB_LEVEL_IDX(pJob);
+
code = schDoTaskRedirect(pJob, pTask, pData, rspCode);
+
taosMemoryFreeClear(pData->pData);
taosMemoryFreeClear(pData->pEpSet);
@@ -621,6 +621,13 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
*/
int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) {
+ if (pJob->noMoreRetry) {
+ *needRetry = false;
+ SCH_TASK_DLOG("task no more retry since job no more retry, retryTimes:%d/%d", pTask->retryTimes,
+ pTask->maxRetryTimes);
+ return TSDB_CODE_SUCCESS;
+ }
+
if (TSDB_CODE_SCH_TIMEOUT_ERROR == errCode) {
pTask->maxExecTimes++;
pTask->maxRetryTimes++;
@@ -645,7 +652,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
return TSDB_CODE_SUCCESS;
}
- if (!SCH_NEED_RETRY(pTask->lastMsgType, errCode)) {
+ if (!SCH_TASK_NEED_RETRY(pTask->lastMsgType, errCode)) {
*needRetry = false;
SCH_TASK_DLOG("task no more retry cause of errCode, errCode:%x - %s", errCode, tstrerror(errCode));
return TSDB_CODE_SUCCESS;
@@ -1067,7 +1074,6 @@ int32_t schLaunchTaskImpl(void *param) {
SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR);
}
- // NOTE: race condition: the task should be put into the hash table before send msg to server
if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXEC) {
SCH_ERR_JRET(schPushTaskToExecList(pJob, pTask));
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXEC);
@@ -1272,6 +1278,8 @@ int32_t schLaunchFetchTask(SSchJob *pJob) {
return TSDB_CODE_SUCCESS;
}
+ SCH_SET_TASK_STATUS(pJob->fetchTask, JOB_TASK_STATUS_FETCH);
+
if (SCH_IS_LOCAL_EXEC_TASK(pJob, pJob->fetchTask)) {
SCH_ERR_JRET(schExecLocalFetch(pJob, pJob->fetchTask));
} else {
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index 7cd5e957b6..2b46a4710e 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -91,7 +91,7 @@ int32_t schedulerFetchRows(int64_t jobId, SSchedulerReq *pReq) {
SCH_ERR_JRET(schHandleOpBeginEvent(jobId, &pJob, SCH_OP_FETCH, pReq));
- SCH_ERR_JRET(schJobFetchRows(pJob));
+ SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_FETCH, pReq));
_return:
diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c
index 60729c4d0e..a4a02b7d65 100644
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@@ -275,7 +275,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S
return 0;
}
-int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) {
- //
- return 0;
-}
+// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) {
+// //
+// return 0;
+// }
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index cb610ad6b5..25b2656365 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -152,8 +152,14 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
if (batchCnt >= batchSz) break;
}
if (taosArrayGetSize(pRes) == 0) {
- taosArrayDestroy(pRes);
- break;
+ if (finished) {
+ taosArrayDestroy(pRes);
+ qDebug("task %d finish recover exec task ", pTask->taskId);
+ break;
+ } else {
+ qDebug("task %d continue recover exec task ", pTask->taskId);
+ continue;
+ }
}
SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
if (qRes == NULL) {
diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c
index bd1dae54d9..9ab545075c 100644
--- a/source/libs/sync/src/syncAppendEntries.c
+++ b/source/libs/sync/src/syncAppendEntries.c
@@ -105,6 +105,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
SRpcMsg rpcRsp = {0};
bool accepted = false;
SSyncRaftEntry* pEntry = NULL;
+ bool resetElect = false;
// if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
@@ -137,7 +138,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
}
syncNodeStepDown(ths, pMsg->term);
- syncNodeResetElectTimer(ths);
+ resetElect = true;
if (pMsg->dataLen < sizeof(SSyncRaftEntry)) {
sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d",
@@ -184,10 +185,9 @@ _SEND_RESPONSE:
// commit index, i.e. leader notice me
if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) {
sError("vgId:%d, failed to commit raft fsm log since %s.", ths->vgId, terrstr());
- goto _out;
}
-_out:
+ if (resetElect) syncNodeResetElectTimer(ths);
return 0;
_IGNORE:
diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c
index e53b8ade1c..3699efbc59 100644
--- a/source/libs/sync/src/syncElection.c
+++ b/source/libs/sync/src/syncElection.c
@@ -115,6 +115,5 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) {
ASSERT(ret == 0);
syncNodeResetElectTimer(pSyncNode);
-
return ret;
}
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 452787d828..524daee7cf 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -183,6 +183,9 @@ int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) {
case TDMT_SYNC_TIMEOUT:
code = syncNodeOnTimeout(pSyncNode, pMsg);
break;
+ case TDMT_SYNC_TIMEOUT_ELECTION:
+ code = syncNodeOnTimeout(pSyncNode, pMsg);
+ break;
case TDMT_SYNC_CLIENT_REQUEST:
code = syncNodeOnClientRequest(pSyncNode, pMsg, NULL);
break;
@@ -1611,8 +1614,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) {
pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
syncNodeStopHeartbeatTimer(pSyncNode);
- // reset elect timer
- syncNodeResetElectTimer(pSyncNode);
+ // trace log
+ sNTrace(pSyncNode, "become follower %s", debugStr);
// send rsp to client
syncNodeLeaderChangeRsp(pSyncNode);
@@ -1628,8 +1631,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) {
// reset log buffer
syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode);
- // trace log
- sNTrace(pSyncNode, "become follower %s", debugStr);
+ // reset elect timer
+ syncNodeResetElectTimer(pSyncNode);
}
// TLA+ Spec
@@ -2295,6 +2298,7 @@ static int32_t syncNodeAppendNoopOld(SSyncNode* ths) {
int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
SyncHeartbeat* pMsg = pRpcMsg->pCont;
+ bool resetElect = false;
const STraceId* trace = &pRpcMsg->info.traceId;
char tbuf[40] = {0};
@@ -2326,12 +2330,11 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
if (pMsg->term == currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) {
syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs);
+ resetElect = true;
- syncNodeResetElectTimer(ths);
ths->minMatchIndex = pMsg->minMatchIndex;
if (ths->state == TAOS_SYNC_STATE_FOLLOWER) {
- // syncNodeFollowerCommit(ths, pMsg->commitIndex);
SRpcMsg rpcMsgLocalCmd = {0};
(void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId);
@@ -2354,7 +2357,6 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
}
if (pMsg->term >= currentTerm && ths->state != TAOS_SYNC_STATE_FOLLOWER) {
- // syncNodeStepDown(ths, pMsg->term);
SRpcMsg rpcMsgLocalCmd = {0};
(void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId);
@@ -2374,15 +2376,10 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
}
}
- /*
- // htonl
- SMsgHead* pHead = rpcMsg.pCont;
- pHead->contLen = htonl(pHead->contLen);
- pHead->vgId = htonl(pHead->vgId);
- */
-
// reply
syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg);
+
+ if (resetElect) syncNodeResetElectTimer(ths);
return 0;
}
diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c
index 2a44588eef..72c8887803 100644
--- a/source/libs/sync/src/syncMessage.c
+++ b/source/libs/sync/src/syncMessage.c
@@ -22,7 +22,7 @@ int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType timeoutType, uint64_t l
SSyncNode* pNode) {
int32_t bytes = sizeof(SyncTimeout);
pMsg->pCont = rpcMallocCont(bytes);
- pMsg->msgType = TDMT_SYNC_TIMEOUT;
+ pMsg->msgType = (timeoutType == SYNC_TIMEOUT_ELECTION) ? TDMT_SYNC_TIMEOUT_ELECTION : TDMT_SYNC_TIMEOUT;
pMsg->contLen = bytes;
if (pMsg->pCont == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -31,7 +31,7 @@ int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType timeoutType, uint64_t l
SyncTimeout* pTimeout = pMsg->pCont;
pTimeout->bytes = bytes;
- pTimeout->msgType = TDMT_SYNC_TIMEOUT;
+ pTimeout->msgType = pMsg->msgType;
pTimeout->vgId = pNode->vgId;
pTimeout->timeoutType = timeoutType;
pTimeout->logicClock = logicClock;
diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c
index 6a348d8290..77e5498127 100644
--- a/source/libs/sync/src/syncRequestVote.c
+++ b/source/libs/sync/src/syncRequestVote.c
@@ -90,6 +90,7 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* ths, SyncRequestVote* pMsg) {
int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
int32_t ret = 0;
SyncRequestVote* pMsg = pRpcMsg->pCont;
+ bool resetElect = false;
// if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &pMsg->srcId)) {
@@ -116,7 +117,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
syncNodeStepDown(ths, currentTerm);
// forbid elect for this round
- syncNodeResetElectTimer(ths);
+ resetElect = true;
}
// send msg
@@ -135,5 +136,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
syncLogRecvRequestVote(ths, pMsg, pReply->voteGranted, "");
syncLogSendRequestVoteReply(ths, pReply, "");
syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
+
+ if (resetElect) syncNodeResetElectTimer(ths);
return 0;
}
diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c
index 413d6adf05..a9d0ddad17 100644
--- a/source/libs/sync/src/syncSnapshot.c
+++ b/source/libs/sync/src/syncSnapshot.c
@@ -798,7 +798,6 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
if (pMsg->term > raftStoreGetTerm(pSyncNode)) {
syncNodeStepDown(pSyncNode, pMsg->term);
}
- syncNodeResetElectTimer(pSyncNode);
// state, term, seq/ack
int32_t code = 0;
@@ -840,6 +839,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
code = -1;
}
+ syncNodeResetElectTimer(pSyncNode);
return code;
}
diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c
index 859183db95..a27be2853e 100644
--- a/source/libs/sync/src/syncTimeout.c
+++ b/source/libs/sync/src/syncTimeout.c
@@ -120,9 +120,6 @@ int32_t syncNodeOnTimeout(SSyncNode* ths, const SRpcMsg* pRpc) {
if (atomic_load_64(&ths->pingTimerLogicClockUser) <= pMsg->logicClock) {
++(ths->pingTimerCounter);
- // syncNodePingAll(ths);
- // syncNodePingPeers(ths);
-
syncNodeTimerRoutine(ths);
}
@@ -138,8 +135,6 @@ int32_t syncNodeOnTimeout(SSyncNode* ths, const SRpcMsg* pRpc) {
++(ths->heartbeatTimerCounter);
sTrace("vgId:%d, sync timer, type:replicate count:%" PRIu64 ", lc-user:%" PRIu64, ths->vgId,
ths->heartbeatTimerCounter, ths->heartbeatTimerLogicClockUser);
-
- // syncNodeReplicate(ths, true);
}
} else {
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 29b7fa740c..5ea9be63db 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -947,6 +947,12 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
return 0;
}
+static int32_t txnIdCompareDesc(const void *pLeft, const void *pRight) {
+ int64_t lhs = *(int64_t *)pLeft;
+ int64_t rhs = *(int64_t *)pRight;
+ return lhs > rhs ? -1 : 1;
+}
+
int tdbPagerRestoreJournals(SPager *pPager) {
tdbDirEntryPtr pDirEntry;
tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName);
@@ -955,23 +961,33 @@ int tdbPagerRestoreJournals(SPager *pPager) {
return -1;
}
+ SArray *pTxnList = taosArrayInit(16, sizeof(int64_t));
+
while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry));
if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) {
- char jname[TD_PATH_MAX] = {0};
- int dirLen = strlen(pPager->pEnv->dbName);
- memcpy(jname, pPager->pEnv->dbName, dirLen);
- jname[dirLen] = '/';
- memcpy(jname + dirLen + 1, name, strlen(name));
- if (tdbPagerRestore(pPager, jname) < 0) {
- tdbCloseDir(&pDir);
+ int64_t txnId = -1;
+ sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId);
+ taosArrayPush(pTxnList, &txnId);
+ }
+ }
+ taosArraySort(pTxnList, txnIdCompareDesc);
+ for (int i = 0; i < TARRAY_SIZE(pTxnList); ++i) {
+ int64_t *pTxnId = taosArrayGet(pTxnList, i);
+ char jname[TD_PATH_MAX] = {0};
+ int dirLen = strlen(pPager->pEnv->dbName);
+ memcpy(jname, pPager->pEnv->dbName, dirLen);
+ jname[dirLen] = '/';
+ sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId);
+ if (tdbPagerRestore(pPager, jname) < 0) {
+ tdbCloseDir(&pDir);
- tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name);
- return -1;
- }
+ tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname);
+ return -1;
}
}
+ taosArrayDestroy(pTxnList);
tdbCloseDir(&pDir);
return 0;
diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index 7467fa2948..b15bb519e7 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -140,7 +140,7 @@ void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)
taosArraySet(pArray, pos + 1, p2);
memset(TARRAY_GET_ELEM(pArray, i), 0, pArray->elemSize);
- pos += 1;
+ pos += 1;
} else {
pos += 1;
}
@@ -157,45 +157,6 @@ void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)
pArray->size = pos + 1;
}
-void taosArrayRemoveDuplicateP(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)) {
- size_t size = pArray->size;
- if (size <= 1) {
- return;
- }
-
- int32_t pos = 0;
- for (int32_t i = 1; i < size; ++i) {
- char* p1 = taosArrayGet(pArray, pos);
- char* p2 = taosArrayGet(pArray, i);
-
- if (comparFn(p1, p2) == 0) {
- // do nothing
- } else {
- if (pos + 1 != i) {
- void* p = taosArrayGetP(pArray, pos + 1);
- if (fp != NULL) {
- fp(p);
- }
-
- taosArraySet(pArray, pos + 1, p2);
- memset(TARRAY_GET_ELEM(pArray, i), 0, pArray->elemSize);
- pos += 1;
- } else {
- pos += 1;
- }
- }
- }
-
- if (fp != NULL) {
- for (int32_t i = pos + 1; i < pArray->size; ++i) {
- void* p = taosArrayGetP(pArray, i);
- fp(p);
- }
- }
-
- pArray->size = pos + 1;
-}
-
void* taosArrayAddAll(SArray* pArray, const SArray* pInput) {
if (pInput) {
return taosArrayAddBatch(pArray, pInput->pData, (int32_t)taosArrayGetSize(pInput));
@@ -392,20 +353,6 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) {
pArray->size = 0;
}
-void taosArrayClearP(SArray* pArray, FDelete fp) {
- if (pArray == NULL) return;
- if (fp == NULL) {
- pArray->size = 0;
- return;
- }
-
- for (int32_t i = 0; i < pArray->size; ++i) {
- fp(*(void**)TARRAY_GET_ELEM(pArray, i));
- }
-
- pArray->size = 0;
-}
-
void* taosArrayDestroy(SArray* pArray) {
if (pArray) {
taosMemoryFree(pArray->pData);
@@ -495,6 +442,7 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void
if (pArray->size <= 1) {
return;
}
+
for (int32_t i = 1; i <= pArray->size - 1; ++i) {
for (int32_t j = i; j > 0; --j) {
if (fn(taosArrayGetP(pArray, j), taosArrayGetP(pArray, j - 1), param) == -1) {
@@ -507,7 +455,6 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void
}
}
}
- return;
}
int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode) {
@@ -556,4 +503,4 @@ void taosArraySwap(SArray* a, SArray* b) {
void* data = a->pData;
a->pData = b->pData;
b->pData = data;
-}
+}
\ No newline at end of file
diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index c15bd96903..f811d2f203 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -637,6 +637,8 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
int32_t code = 0;
char **pEnv = environ;
line[1023] = 0;
+
+ if (pEnv == NULL) return 0;
while (*pEnv != NULL) {
name = value = value2 = value3 = NULL;
olen = vlen = vlen2 = vlen3 = 0;
diff --git a/source/util/src/tsimplehash.c b/source/util/src/tsimplehash.c
index 70acffed5d..ec1991923f 100644
--- a/source/util/src/tsimplehash.c
+++ b/source/util/src/tsimplehash.c
@@ -28,19 +28,23 @@
#define HASH_INDEX(v, c) ((v) & ((c)-1))
-#define FREE_HASH_NODE(_n) \
- do { \
- taosMemoryFreeClear(_n); \
+#define FREE_HASH_NODE(_n, fp) \
+ do { \
+ if (fp) { \
+ fp((_n)->data); \
+ } \
+ taosMemoryFreeClear(_n); \
} while (0);
struct SSHashObj {
- SHNode **hashList;
- size_t capacity; // number of slots
- int64_t size; // number of elements in hash table
- _hash_fn_t hashFp; // hash function
- _equal_fn_t equalFp; // equal function
- SArray* pHashNodeBuf;// hash node allocation buffer, 1k size of each page by default
- int32_t offset; // allocation offset in current page
+ SHNode **hashList;
+ size_t capacity; // number of slots
+ int64_t size; // number of elements in hash table
+ _hash_fn_t hashFp; // hash function
+ _equal_fn_t equalFp; // equal function
+ _hash_free_fn_t freeFp; // free function
+ SArray *pHashNodeBuf; // hash node allocation buffer, 1k size of each page by default
+ int32_t offset; // allocation offset in current page
};
static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
@@ -71,7 +75,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) {
pHashObj->capacity = taosHashCapacity((int32_t)capacity);
pHashObj->equalFp = memcmp;
- pHashObj->pHashNodeBuf = taosArrayInit(10, sizeof(void*));
+ pHashObj->freeFp = NULL;
pHashObj->offset = 0;
pHashObj->size = 0;
@@ -92,6 +96,10 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) {
return (int32_t) pHashObj->size;
}
+void tSimpleHashSetFreeFp(SSHashObj* pHashObj, _hash_free_fn_t freeFp) {
+ pHashObj->freeFp = freeFp;
+}
+
static void* doInternalAlloc(SSHashObj* pHashObj, int32_t size) {
#if 0
void** p = taosArrayGetLast(pHashObj->pHashNodeBuf);
@@ -306,7 +314,8 @@ int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen) {
} else {
pPrev->next = pNode->next;
}
- FREE_HASH_NODE(pNode);
+
+ FREE_HASH_NODE(pNode, pHashObj->freeFp);
pHashObj->size -= 1;
code = TSDB_CODE_SUCCESS;
break;
@@ -341,7 +350,7 @@ int32_t tSimpleHashIterateRemove(SSHashObj *pHashObj, const void *key, size_t ke
*pIter = pPrev ? GET_SHASH_NODE_DATA(pPrev) : NULL;
}
- FREE_HASH_NODE(pNode);
+ FREE_HASH_NODE(pNode, pHashObj->freeFp);
pHashObj->size -= 1;
break;
}
@@ -370,14 +379,13 @@ void tSimpleHashClear(SSHashObj *pHashObj) {
while (pNode) {
pNext = pNode->next;
- FREE_HASH_NODE(pNode);
+ FREE_HASH_NODE(pNode, pHashObj->freeFp);
pNode = pNext;
}
pHashObj->hashList[i] = NULL;
}
- taosArrayClearEx(pHashObj->pHashNodeBuf, destroyItems);
pHashObj->offset = 0;
pHashObj->size = 0;
}
@@ -388,7 +396,6 @@ void tSimpleHashCleanup(SSHashObj *pHashObj) {
}
tSimpleHashClear(pHashObj);
- taosArrayDestroy(pHashObj->pHashNodeBuf);
taosMemoryFreeClear(pHashObj->hashList);
taosMemoryFree(pHashObj);
}
diff --git a/source/util/src/tversion.c b/source/util/src/tversion.c
index dc357c61a1..0f84f6bafa 100644
--- a/source/util/src/tversion.c
+++ b/source/util/src/tversion.c
@@ -26,8 +26,9 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) {
int32_t vnum[4] = {0};
int32_t len = strlen(vstr);
char tmp[16] = {0};
+ int32_t vpos = 0;
- for (int32_t spos = 0, tpos = 0, vpos = 0; spos < len && vpos < 4; ++spos) {
+ for (int32_t spos = 0, tpos = 0; spos < len && vpos < 4; ++spos) {
if (vstr[spos] != '.') {
tmp[spos - tpos] = vstr[spos];
} else {
@@ -38,6 +39,10 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) {
}
}
+ if ('\0' != tmp[0] && vpos < 4) {
+ vnum[vpos] = atoi(tmp);
+ }
+
if (vnum[0] <= 0) {
terrno = TSDB_CODE_INVALID_VERSION_STRING;
return -1;
@@ -66,16 +71,16 @@ int32_t taosCheckVersionCompatible(int32_t clientVer, int32_t serverVer, int32_t
case 4:
break;
case 3:
- clientVer %= 100;
- serverVer %= 100;
+ clientVer /= 100;
+ serverVer /= 100;
break;
case 2:
- clientVer %= 10000;
- serverVer %= 10000;
+ clientVer /= 10000;
+ serverVer /= 10000;
break;
case 1:
- clientVer %= 1000000;
- serverVer %= 1000000;
+ clientVer /= 1000000;
+ serverVer /= 1000000;
break;
default:
terrno = TSDB_CODE_INVALID_VERSION_NUMBER;
diff --git a/source/util/src/version.c.in b/source/util/src/version.c.in
index cb307b57fc..71998e3321 100644
--- a/source/util/src/version.c.in
+++ b/source/util/src/version.c.in
@@ -1,6 +1,7 @@
char version[64] = "${TD_VER_NUMBER}";
char compatible_version[12] = "${TD_VER_COMPATIBLE}";
char gitinfo[48] = "${TD_VER_GIT}";
+char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}";
char buildinfo[64] = "Built at ${TD_VER_DATE}";
void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {};
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index c0a899bbe4..0515a5cb69 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -10,18 +10,21 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py
@@ -43,6 +46,11 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3
@@ -93,6 +101,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/raw_block_interface_test.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3
@@ -866,6 +875,8 @@
,,y,script,./test.sh -f tsim/query/udf.sim
,,n,script,./test.sh -f tsim/query/udfpy.sim
,,y,script,./test.sh -f tsim/query/udf_with_const.sim
+,,y,script,./test.sh -f tsim/query/join_interval.sim
+,,y,script,./test.sh -f tsim/query/unionall_as_table.sim
,,y,script,./test.sh -f tsim/query/sys_tbname.sim
,,y,script,./test.sh -f tsim/query/groupby.sim
,,y,script,./test.sh -f tsim/query/event.sim
@@ -873,6 +884,7 @@
,,y,script,./test.sh -f tsim/query/emptyTsRange.sim
,,y,script,./test.sh -f tsim/query/partitionby.sim
,,y,script,./test.sh -f tsim/query/tableCount.sim
+,,y,script,./test.sh -f tsim/query/nullColSma.sim
,,y,script,./test.sh -f tsim/qnode/basic1.sim
,,y,script,./test.sh -f tsim/snode/basic1.sim
,,y,script,./test.sh -f tsim/mnode/basic1.sim
diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh
index f0880bdb04..de343752c6 100755
--- a/tests/parallel_test/run.sh
+++ b/tests/parallel_test/run.sh
@@ -167,8 +167,10 @@ function run_thread() {
local case_build_san=`echo "$line"|cut -d, -f3`
if [ "${case_build_san}" == "y" ]; then
case_build_san="y"
+ DEBUGPATH="debugSan"
elif [[ "${case_build_san}" == "n" ]] || [[ "${case_build_san}" == "" ]]; then
case_build_san="n"
+ DEBUGPATH="debugNoSan"
else
usage
exit 1
@@ -301,10 +303,10 @@ function run_thread() {
if [ ! -z "$corefile" ]; then
echo -e "\e[34m corefiles: $corefile \e[0m"
local build_dir=$log_dir/build_${hosts[index]}
- local remote_build_dir="${workdirs[index]}/TDengine/debug/build"
- if [ $ent -ne 0 ]; then
- remote_build_dir="${workdirs[index]}/TDinternal/debug/build"
- fi
+ local remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build"
+ # if [ $ent -ne 0 ]; then
+ # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build"
+ # fi
mkdir $build_dir 2>/dev/null
if [ $? -eq 0 ]; then
# scp build binary
diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh
index a74e6dae73..535b6802f2 100755
--- a/tests/script/coverage_test.sh
+++ b/tests/script/coverage_test.sh
@@ -12,6 +12,7 @@ fi
today=`date +"%Y%m%d"`
TDENGINE_DIR=/root/TDengine
JDBC_DIR=/root/taos-connector-jdbc
+TAOSKEEPER_DIR=/root/taoskeeper
TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log
# Color setting
@@ -184,6 +185,24 @@ function runJDBCCases() {
echo -e "### JDBC test result: $summary ###" | tee -a $TDENGINE_COVERAGE_REPORT
}
+function runTaosKeeperCases() {
+ echo "=== Run taoskeeper cases ==="
+
+ cd $TAOSKEEPER_DIR
+ git checkout -- .
+ git reset --hard HEAD
+ git checkout master
+ git pull
+
+ stopTaosd
+ stopTaosadapter
+
+ taosd -c /etc/taos >> /dev/null 2>&1 &
+ taosadapter >> /dev/null 2>&1 &
+
+ go mod tidy && go test -v ./...
+}
+
function runTest() {
echo "run Test"
@@ -195,6 +214,7 @@ function runTest() {
runSimCases
runPythonCases
runJDBCCases
+ runTaosKeeperCases
stopTaosd
cd $TDENGINE_DIR/tests/script
@@ -212,7 +232,7 @@ function lcovFunc {
lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
# remove exclude paths
- if [ "$branch" == "3.0" ]; then
+ if [ "$branch" == "main" ] ; then
lcov --remove coverage.info \
'*/contrib/*' '*/tests/*' '*/test/*' '*/tools/*' '*/libs/sync/*'\
'*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\
@@ -222,6 +242,8 @@ function lcovFunc {
'*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' '*/catalog.c'\
'*/tqSnapshot.c' '*/tsdbSnapshot.c''*/metaSnapshot.c' '*/smaSnapshot.c' '*/tqOffsetSnapshot.c'\
'*/vnodeSnapshot.c' '*/metaSnapshot.c' '*/tsdbSnapshot.c' '*/mndGrant.c' '*/mndSnode.c' '*/streamRecover.c'\
+ '*/osAtomic.c' '*/osDir.c' '*/osFile.c' '*/osMath.c' '*/osSignal.c' '*/osSleep.c' '*/osString.c' '*/osSystem.c'\
+ '*/osThread.c' '*/osTime.c' '*/osTimezone.c' \
--rc lcov_branch_coverage=1 -o coverage.info
else
lcov --remove coverage.info \
diff --git a/tests/script/tsim/parser/limit1_stb.sim b/tests/script/tsim/parser/limit1_stb.sim
index 1a5d57efbc..731a218de5 100644
--- a/tests/script/tsim/parser/limit1_stb.sim
+++ b/tests/script/tsim/parser/limit1_stb.sim
@@ -484,6 +484,7 @@ if $rows != 2 then
return -1
endi
+print === select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 5 and c1 > 0 and c2 < 9 and c3 > 1 and c4 < 7 and c5 > 4 partition by t1 interval(5m) limit 1 offset 0
sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 5 and c1 > 0 and c2 < 9 and c3 > 1 and c4 < 7 and c5 > 4 partition by t1 interval(5m) limit 1 offset 0
if $rows != 3 then
return -1
diff --git a/tests/script/tsim/parser/slimit1_query.sim b/tests/script/tsim/parser/slimit1_query.sim
index 9a27d5523b..1167fe0b3d 100644
--- a/tests/script/tsim/parser/slimit1_query.sim
+++ b/tests/script/tsim/parser/slimit1_query.sim
@@ -70,7 +70,7 @@ endi
### empty result set
sql select count(*) from stb partition by t2,t1 order by t2 asc slimit 0 soffset 0
-if $rows != 9 then
+if $rows != 0 then
return -1
endi
diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim
index f4d95df083..0d796af0a0 100644
--- a/tests/script/tsim/query/interval-offset.sim
+++ b/tests/script/tsim/query/interval-offset.sim
@@ -212,10 +212,10 @@ print ===> rows2: $data20 $data21 $data22 $data23 $data24
if $rows != 3 then
return -1
endi
-if $data01 != 2 then
+if $data01 != 4 then
return -1
endi
-if $data04 != 2 then
+if $data04 != 4 then
return -1
endi
diff --git a/tests/script/tsim/query/join_interval.sim b/tests/script/tsim/query/join_interval.sim
new file mode 100644
index 0000000000..14994a5cc1
--- /dev/null
+++ b/tests/script/tsim/query/join_interval.sim
@@ -0,0 +1,42 @@
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c udf -v 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print ======== step create databases
+sql create database d1
+sql create database d2
+sql create table d1.t1(ts timestamp, i int) tags(t int);
+sql create table d2.t1(ts timestamp, i int);
+sql insert into d1.t11 using d1.t1 tags(1) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4)
+sql insert into d1.t12 using d1.t1 tags(2) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4)
+sql insert into d1.t13 using d1.t1 tags(3) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4)
+
+sql insert into d2.t1 values(1500000000000,0)(1500000000001,1)(1500000000002,2)
+
+sql select _wstart,_wend,count((a.ts)),count(b.ts) from d1.t1 a, d2.t1 b where a.ts is not null and a.ts = b.ts interval(1a) ;
+if $data02 != 3 then
+ return -1
+endi
+
+if $data03 != 3 then
+ return -1
+endi
+
+if $data12 != 3 then
+ return -1
+endi
+
+if $data13 != 3 then
+ return -1
+endi
+if $data22 != 3 then
+ return -1
+endi
+
+if $data23 != 3 then
+ return -1
+endi
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/query/nullColSma.sim b/tests/script/tsim/query/nullColSma.sim
new file mode 100644
index 0000000000..886274e7e5
--- /dev/null
+++ b/tests/script/tsim/query/nullColSma.sim
@@ -0,0 +1,139 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+$dbPrefix = m_in_db
+$tbPrefix = m_in_tb
+$mtPrefix = m_in_mt
+$tbNum = 1
+$rowNum = 200
+$totalNum = 400
+
+print =============== step1
+$i = 0
+$db = $dbPrefix . $i
+$mt = $mtPrefix . $i
+
+sql drop database if exists $db
+sql create database $db vgroups 1 maxrows 200 minrows 10;
+sql use $db
+sql create table $mt (ts timestamp, f1 int, f2 float) TAGS(tgcol int)
+
+print ====== start create child tables and insert data
+$i = 0
+while $i < $tbNum
+ $tb = $tbPrefix . $i
+ sql create table $tb using $mt tags( $i )
+
+ $x = 0
+ while $x < $rowNum
+ $cc = $x * 1
+ $ms = 1601481600000 + $cc
+
+ sql insert into $tb values ($ms , NULL , $x )
+ $x = $x + 1
+ endw
+
+ $i = $i + 1
+endw
+
+$i = 1
+$tb = $tbPrefix . $i
+sql create table $tb using $mt tags( $i )
+
+$x = 0
+while $x < $rowNum
+ $cc = $x * 1
+ $ms = 1601481600000 + $cc
+
+ sql insert into $tb values ($ms , $x , NULL )
+ $x = $x + 1
+endw
+
+sql flush database $db
+
+print =============== step2
+$i = 0
+$tb = $tbPrefix . $i
+sql select max(f1) from $tb
+if $rows != 1 then
+ return -1
+endi
+if $data00 != NULL then
+ return -1
+endi
+
+$i = 1
+$tb = $tbPrefix . $i
+sql select max(f2) from $tb
+if $rows != 1 then
+ return -1
+endi
+if $data00 != NULL then
+ return -1
+endi
+
+$rowNum = 10
+
+print ====== insert more data
+$i = 0
+while $i < $tbNum
+ $tb = $tbPrefix . $i
+
+ $x = 0
+ while $x < $rowNum
+ $cc = $x * 1
+ $ms = 1601481700000 + $cc
+
+ sql insert into $tb values ($ms , $x , $x )
+ $x = $x + 1
+ endw
+
+ $i = $i + 1
+endw
+
+$i = 1
+$tb = $tbPrefix . $i
+$x = 0
+while $x < $rowNum
+ $cc = $x * 1
+ $ms = 1601481700000 + $cc
+
+ sql insert into $tb values ($ms , $x , $x )
+ $x = $x + 1
+endw
+
+sql flush database $db
+
+print =============== step3
+$i = 0
+$tb = $tbPrefix . $i
+sql select max(f1) from $tb
+if $rows != 1 then
+ return -1
+endi
+if $data00 != 9 then
+ return -1
+endi
+
+$i = 1
+$tb = $tbPrefix . $i
+sql select max(f2) from $tb
+if $rows != 1 then
+ return -1
+endi
+if $data00 != 9.00000 then
+ print $data00
+ return -1
+endi
+
+
+print =============== clear
+#sql drop database $db
+#sql select * from information_schema.ins_databases
+#if $rows != 0 then
+# return -1
+#endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim
index 7b3953129a..c676f2b1e0 100644
--- a/tests/script/tsim/query/sys_tbname.sim
+++ b/tests/script/tsim/query/sys_tbname.sim
@@ -51,6 +51,11 @@ if $data00 != @ins_stables@ then
return -1
endi
+sql select * from information_schema.ins_tables where table_name='';
+if $rows != 0 then
+ return -1
+endi
+
sql select tbname from information_schema.ins_tables;
print $rows $data00
if $rows != 33 then
diff --git a/tests/script/tsim/query/unionall_as_table.sim b/tests/script/tsim/query/unionall_as_table.sim
new file mode 100644
index 0000000000..dc3d2cbec4
--- /dev/null
+++ b/tests/script/tsim/query/unionall_as_table.sim
@@ -0,0 +1,28 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql create database test;
+sql use test;
+sql CREATE STABLE bw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50));
+sql CREATE STABLE aw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50));
+sql CREATE STABLE dw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50));
+sql insert into t1 using dw_yc_h_substation_mea tags('1234567890','testa','0021001','abc01') values(now,'2023-03-27','00:01:00',now,2.3,3.3,4.4,5.5);
+sql insert into t2 using dw_yc_h_substation_mea tags('2234567890','testb','0022001','abc02') values(now,'2023-03-27','00:01:00',now,2.3,2.3,2.4,2.5);
+sql insert into t3 using aw_yc_h_substation_mea tags('2234567890','testc','0023001','abc03') values(now,'2023-03-27','00:15:00',now,2.3,2.3,2.4,2.5);
+sql insert into t4 using bw_yc_h_substation_mea tags('4234567890','testd','0021001','abc03') values(now,'2023-03-27','00:45:00',now,2.3,2.3,2.4,2.5);
+sql insert into t5 using bw_yc_h_substation_mea tags('5234567890','testd','0021001','abc03') values(now,'2023-03-27','00:00:00',now,2.3,2.3,2.4,2.5);
+sql select t.ts,t.id,t.name,t.create_date,t.create_time,t.datasource,t.sum_p_value from (select ts,id,name,create_date,create_time,datasource,sum_p_value from bw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45') union all select ts,id,name,create_date,create_time,datasource,sum_p_value from aw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45') union all select ts,id,name,create_date,create_time,datasource,sum_p_value from dw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45')) t where t.datasource='0021001' and t.id='4234567890' order by t.create_time;
+
+if $rows != 1 then
+ return -1
+endi
+if $data01 != @4234567890@ then
+ return -1
+endi
+if $data05 != @0021001@ then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim
index 1d58922928..1559d3d32b 100644
--- a/tests/script/tsim/stream/distributeInterval0.sim
+++ b/tests/script/tsim/stream/distributeInterval0.sim
@@ -272,4 +272,122 @@ if $data12 != 2 then
goto loop3
endi
+print ===== step3
+
+sql drop database if exists test4;
+sql create database test4 vgroups 10;
+sql use test4;
+sql create stable st(ts timestamp,a int,b int,c varchar(250) ) tags(ta int,tb int,tc int);
+sql create table aaa using st tags(1,1,1);
+sql create table bbb using st tags(2,2,2);
+sql create table ccc using st tags(3,2,2);
+sql create table ddd using st tags(4,2,2);
+
+
+sql create stream streams1 ignore expired 0 fill_history 0 watermark 3s into streamst subtable(c) as select _wstart, c , count(*) c1, last_row(b) c2 from st partition by c interval(1s) ;
+
+sql insert into aaa values(1648791221001,2,2,"/a1/aa/aa");
+sql insert into bbb values(1648791221001,2,2,"/a1/aa/aa");
+sql insert into ccc values(1648791221001,2,2,"/a1/aa/aa");
+sql insert into ddd values(1648791221001,2,2,"/a1/aa/aa");
+
+sql insert into aaa values(1648791222002,2,2,"/a2/aa/aa");
+sql insert into bbb values(1648791222002,2,2,"/a2/aa/aa");
+sql insert into ccc values(1648791222002,2,2,"/a2/aa/aa");
+sql insert into ddd values(1648791222002,2,2,"/a2/aa/aa");
+
+sql insert into aaa values(1648791223003,2,2,"/a3/aa/aa");
+sql insert into bbb values(1648791223003,2,2,"/a3/aa/aa");
+sql insert into ccc values(1648791223003,2,2,"/a3/aa/aa");
+sql insert into ddd values(1648791223003,2,2,"/a3/aa/aa");
+
+sql insert into aaa values(1648791224003,2,2,"/a4/aa/aa");
+sql insert into bbb values(1648791224003,2,2,"/a4/aa/aa");
+sql insert into ccc values(1648791224003,2,2,"/a4/aa/aa");
+sql insert into ddd values(1648791224003,2,2,"/a4/aa/aa");
+
+
+sql insert into aaa values(1648791225003,2,2,"/a5/aa/aa");
+sql insert into bbb values(1648791225003,2,2,"/a5/aa/aa");
+sql insert into ccc values(1648791225003,2,2,"/a5/aa/aa");
+sql insert into ddd values(1648791225003,2,2,"/a5/aa/aa");
+
+sql insert into aaa values(1648791226003,2,2,"/a6/aa/aa");
+sql insert into bbb values(1648791226003,2,2,"/a6/aa/aa");
+sql insert into ccc values(1648791226003,2,2,"/a6/aa/aa");
+sql insert into ddd values(1648791226003,2,2,"/a6/aa/aa");
+
+$loop_count = 0
+
+loop4:
+sleep 200
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+sql select * from streamst;
+
+if $rows == 0 then
+ goto loop4
+endi
+
+sql delete from aaa where ts = 1648791223003 ;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+sql select * from streamst;
+
+if $rows == 0 then
+ goto loop5
+endi
+
+
+sql delete from ccc;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+sql select * from streamst;
+
+if $rows == 0 then
+ goto loop6
+endi
+
+sql delete from ddd;
+
+$loop_count = 0
+
+loop7:
+sleep 200
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+sql select * from streamst;
+
+if $rows == 0 then
+ goto loop7
+endi
+
+print ===== over
+
system sh/stop_dnodes.sh
diff --git a/tests/script/tsim/stream/distributeIntervalRetrive0.sim b/tests/script/tsim/stream/distributeIntervalRetrive0.sim
index 529a2a1b30..a2d9c4ab45 100644
--- a/tests/script/tsim/stream/distributeIntervalRetrive0.sim
+++ b/tests/script/tsim/stream/distributeIntervalRetrive0.sim
@@ -3,7 +3,6 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/exec.sh -n dnode1 -s start
-#==system sh/exec.sh -n dnode1 -s start -v
sleep 50
sql connect
@@ -11,7 +10,6 @@ sql connect
sql create dnode $hostname2 port 7200
system sh/exec.sh -n dnode2 -s start
-#==system sh/exec.sh -n dnode2 -s start -v
print ===== step1
$x = 0
@@ -37,14 +35,14 @@ endi
print ===== step2
-sql create database test vgroups 4;
+sql create database test vgroups 10;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2);
sql create table ts3 using st tags(3,2,2);
sql create table ts4 using st tags(4,2,2);
-sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 into streamtST1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s);
+sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 delete_mark 10s into streamtST1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s);
sleep 1000
@@ -235,6 +233,29 @@ endi
print loop3 over
+sql insert into ts1 values(1648791200001,1,12,3,1.0);
+sql insert into ts2 values(1648791200001,1,12,3,1.0);
+sql insert into ts3 values(1648791200001,1,12,3,1.0);
+sql insert into ts4 values(1648791200001,1,12,3,1.0);
+
+$loop_count = 0
+loop31:
+sleep 1000
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+sql select * from streamtST1;
+
+if $rows <= 4 then
+ print =====rows=$rows
+ goto loop31
+endi
+
+print loop31 over
+
sql drop stream if exists streams1;
sql drop database if exists test1;
@@ -243,7 +264,7 @@ sql use test1;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
-sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ;
+sql create stream streams1 trigger at_once IGNORE EXPIRED 0 delete_mark 20s into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ;
sql insert into t1 values(1648791211000,1,2,3);
diff --git a/tests/script/tsim/stream/state1.sim b/tests/script/tsim/stream/state1.sim
index 2ae5739642..67e02c0890 100644
--- a/tests/script/tsim/stream/state1.sim
+++ b/tests/script/tsim/stream/state1.sim
@@ -4,6 +4,7 @@ system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
+print step 1
print =============== create database
sql create database test vgroups 4;
sql select * from information_schema.ins_databases;
@@ -33,8 +34,8 @@ if $loop_count == 10 then
endi
sql select * from streamt1;
-print data00 data01
-print data10 data11
+print $data00 $data01
+print $data10 $data11
if $rows != 0 then
print =====rows=$rows
@@ -52,8 +53,8 @@ if $loop_count == 10 then
endi
sql select * from streamt1;
-print data00 data01
-print data10 data11
+print $data00 $data01
+print $data10 $data11
if $rows != 1 then
print =====rows=$rows
@@ -92,9 +93,64 @@ endi
sql select * from streamt1;
if $rows != 2 then
print =====rows=$rows
- goto loop2
+ goto loop3
endi
+print step 1 over
+print step 2
+
+sql create database test2 vgroups 1;
+sql use test2;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+print create stream streams2 trigger at_once watermark 1000s into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a)
+sql create stream streams2 trigger at_once watermark 1000s into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a);
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+sql insert into t1 values(1648791213010,1,2,3,1.1);
+
+$loop_count = 0
+loop4:
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+sql select * from streamt2;
+print $data00 $data01
+print $data10 $data11
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop4
+endi
+
+print insert into t1 values(1648791213005,2,2,3,1.1)
+sql insert into t1 values(1648791213005,2,2,3,1.1);
+
+$loop_count = 0
+loop5:
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+print select * from streamt2
+sql select * from streamt2;
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+print $data30 $data31
+
+if $rows != 3 then
+ print =====rows=$rows
+ goto loop5
+endi
+
+print step 2 over
print state1 end
diff --git a/tests/system-test/0-others/check_assert.py b/tests/system-test/0-others/check_assert.py
index 59fb223528..ff69b9eeec 100644
--- a/tests/system-test/0-others/check_assert.py
+++ b/tests/system-test/0-others/check_assert.py
@@ -28,10 +28,10 @@ import os
NO_FOUND = 0 # not found assert or ASSERT
FOUND_OK = 1 # found ASSERT and valid usage
FOUND_NOIF = 2 # found ASSERT but no if like ASSERT(...)
-FOUND_LOWER = 3 # found assert write with lower letters
+FOUND_LOWER = 3 # found assert write with system assert
FOUND_HAVENOT = 4 # found ASSERT have if but have not like if(!ASSERT)
-code_strs = ["not found", "valid", "found but no if", "lower assert","found but have not"]
+code_strs = ["not found", "valid", "found but no if", "system assert","found but have not"]
#
diff --git a/tests/system-test/0-others/tmqBasic.json b/tests/system-test/0-others/tmqBasic.json
index 24e815708a..d716bff3ac 100644
--- a/tests/system-test/0-others/tmqBasic.json
+++ b/tests/system-test/0-others/tmqBasic.json
@@ -14,7 +14,6 @@
"auto.offset.reset": "earliest",
"enable.auto.commit": "true",
"auto.commit.interval.ms": 1000,
- "enable.heartbeat.background": "true",
"experimental.snapshot.enable": "true",
"msg.with.table.name": "false",
"topic_list": [
diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py
new file mode 100644
index 0000000000..e43b32a716
--- /dev/null
+++ b/tests/system-test/2-query/columnLenUpdated.py
@@ -0,0 +1,215 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import platform
+if platform.system().lower() == 'windows':
+ import wexpect as taosExpect
+else:
+ import pexpect as taosExpect
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+def taos_command (buildPath, key, value, expectString, sqlString=''):
+ if len(key) == 0:
+ tdLog.exit("taos test key is null!")
+
+ if platform.system().lower() == 'windows':
+ taosCmd = buildPath + '\\build\\bin\\taos.exe '
+ taosCmd = taosCmd.replace('\\','\\\\')
+ else:
+ taosCmd = buildPath + '/build/bin/taos '
+
+ cfgPath = buildPath + "/../sim/psim/cfg"
+ taosCmd = taosCmd + ' -c' + cfgPath + ' -' + key
+ if len(value) != 0:
+ taosCmd = taosCmd + ' ' + value
+
+ tdLog.info ("taos cmd: %s" % taosCmd)
+
+ child = taosExpect.spawn(taosCmd, timeout=20)
+ #output = child.readline()
+ #print (output.decode())
+ if len(expectString) != 0:
+ i = child.expect([expectString, taosExpect.TIMEOUT, taosExpect.EOF], timeout=20)
+ else:
+ i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=20)
+
+ if platform.system().lower() == 'windows':
+ retResult = child.before
+ else:
+ retResult = child.before.decode()
+ print(retResult)
+ #print(child.after.decode())
+ if i == 0:
+ print ('taos login success! Here can run sql, taos> ')
+ return "TAOS_OK"
+ else:
+ return "TAOS_FAIL"
+
+class TDTestCase:
+ #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\
+ # 'serverPort': 7080, 'firstEp': 'trd02:7080'}
+ hostname = socket.gethostname()
+ if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+ try:
+ config = eval(tdDnodes.dnodes[0].remoteIP)
+ hostname = config["host"]
+ except Exception:
+ hostname = tdDnodes.dnodes[0].remoteIP
+ serverPort = '7080'
+ rpcDebugFlagVal = '143'
+ clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
+ clientCfgDict["serverPort"] = serverPort
+ clientCfgDict["firstEp"] = hostname + ':' + serverPort
+ clientCfgDict["secondEp"] = hostname + ':' + serverPort
+ clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal
+ clientCfgDict["fqdn"] = hostname
+
+ updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
+ updatecfgDict["clientCfg"] = clientCfgDict
+ updatecfgDict["serverPort"] = serverPort
+ updatecfgDict["firstEp"] = hostname + ':' + serverPort
+ updatecfgDict["secondEp"] = hostname + ':' + serverPort
+ updatecfgDict["fqdn"] = hostname
+
+ print ("===================: ", updatecfgDict)
+
+ def init(self, conn, logSql, replicaVar=1):
+ self.replicaVar = int(replicaVar)
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor(), True)
+
+ def getBuildPath(self):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files or "taosd.exe" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/build/bin")]
+ break
+ return buildPath
+
+ def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
+ tdSql.prepare()
+ # time.sleep(2)
+ tdSql.query("create user testpy pass 'testpy'")
+
+ buildPath = self.getBuildPath()
+ if (buildPath == ""):
+ tdLog.exit("taosd not found!")
+ else:
+ tdLog.info("taosd found in %s" % buildPath)
+ cfgPath = buildPath + "/../sim/psim/cfg"
+ tdLog.info("cfgPath: %s" % cfgPath)
+
+ checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting']
+ netrole = ['client', 'server']
+
+ keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \
+ 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''}
+
+ keyDict['h'] = self.hostname
+ keyDict['c'] = cfgPath
+ keyDict['P'] = self.serverPort
+
+ tdSql.query("drop database if exists db1")
+ tdSql.query("create database if not exists db1 vgroups 1")
+ tdSql.query("use db1")
+ tdSql.query("create table tba (ts timestamp, f1 binary(2))")
+ tdSql.query("insert into tba values (now, '22')")
+ tdSql.query("select * from tba")
+ tdSql.checkData(0, 1, '22')
+
+ keyDict['s'] = "\"alter table db1.tba modify column f1 binary(5) \""
+ retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '')
+ if retCode != "TAOS_OK":
+ tdLog.exit("taos -s fail")
+
+ keyDict['s'] = "\"insert into db1.tba values (now, '55555')\""
+ retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '')
+ if retCode != "TAOS_OK":
+ tdLog.exit("taos -s fail")
+
+ tdSql.query("select * from tba order by ts")
+ tdSql.checkData(0, 1, '22')
+ tdSql.checkData(1, 1, '55555')
+
+
+
+ keyDict['s'] = "\"alter table db1.tba add column f2 binary(5) \""
+ retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '')
+ if retCode != "TAOS_OK":
+ tdLog.exit("taos -s fail")
+
+ tdSql.query("select * from tba order by ts")
+ tdSql.query("select * from tba order by ts")
+ tdSql.checkData(0, 2, None)
+ tdSql.checkData(1, 2, None)
+
+
+
+
+ keyDict['s'] = "\"alter table db1.tba add column f3 binary(5) \""
+ retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '')
+ if retCode != "TAOS_OK":
+ tdLog.exit("taos -s fail")
+
+ tdSql.query("select f3 from tba order by ts")
+ tdSql.checkData(0, 0, None)
+ tdSql.checkData(1, 0, None)
+
+
+ tdSql.query("create table stb (ts timestamp, f1 int, f2 binary(2)) tags (tg1 binary(2))")
+ tdSql.query("create table tb1 using stb tags('bb')")
+ tdSql.query("insert into tb1 values (now, 2,'22')")
+ tdSql.query("select count(*) from stb group by tg1")
+ tdSql.checkData(0, 0, 1)
+
+ keyDict['s'] = "\"alter table db1.stb modify tag tg1 binary(5) \""
+ retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '')
+ if retCode != "TAOS_OK":
+ tdLog.exit("taos -s fail")
+
+ keyDict['s'] = "\"create table db1.tb2 using db1.stb tags('bbbbb')\""
+ retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", '')
+ if retCode != "TAOS_OK":
+ tdLog.exit("taos -s fail")
+
+ keyDict['s'] = "\"insert into db1.tb2 values (now, 2,'22')\""
+ retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '')
+ if retCode != "TAOS_OK":
+ tdLog.exit("taos -s fail")
+
+ keyDict['s'] = "\"alter table db1.stb modify column f2 binary(5) \""
+ retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '')
+ if retCode != "TAOS_OK":
+ tdLog.exit("taos -s fail")
+
+ keyDict['s'] = "\"insert into db1.tb2 values (now, 3,'55555')\""
+ retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '')
+ if retCode != "TAOS_OK":
+ tdLog.exit("taos -s fail")
+
+ tdSql.query("select count(*) from stb group by tg1")
+ tdSql.checkData(0, 0, 2)
+ tdSql.checkData(1, 0, 1)
+
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py
index 6557aad05f..1b843defce 100755
--- a/tests/system-test/2-query/nestedQuery.py
+++ b/tests/system-test/2-query/nestedQuery.py
@@ -6144,7 +6144,7 @@ class TDTestCase:
startTime = time.time()
- self.function_before_26()
+ #self.function_before_26()
self.math_nest(['UNIQUE'])
self.math_nest(['MODE'])
@@ -6157,9 +6157,9 @@ class TDTestCase:
# self.math_nest(['MAVG'])
# self.math_nest(['HYPERLOGLOG'])
# self.math_nest(['TAIL'])
- # self.math_nest(['CSUM'])
- # self.math_nest(['statecount','stateduration'])
- # self.math_nest(['HISTOGRAM'])
+ self.math_nest(['CSUM'])
+ self.math_nest(['statecount','stateduration'])
+ self.math_nest(['HISTOGRAM'])
# self.str_nest(['LTRIM','RTRIM','LOWER','UPPER'])
# self.str_nest(['LENGTH','CHAR_LENGTH'])
diff --git a/tests/system-test/2-query/nestedQuery_26.py b/tests/system-test/2-query/nestedQuery_26.py
new file mode 100755
index 0000000000..9d5f31d1e0
--- /dev/null
+++ b/tests/system-test/2-query/nestedQuery_26.py
@@ -0,0 +1,76 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from util.cases import tdCases
+from .nestedQuery import *
+
+class TDTestCase(TDTestCase):
+
+
+ def run(self):
+ tdSql.prepare()
+
+ startTime = time.time()
+
+ self.function_before_26()
+
+ # self.math_nest(['UNIQUE'])
+ # self.math_nest(['MODE'])
+ # self.math_nest(['SAMPLE'])
+
+ # self.math_nest(['ABS','SQRT'])
+ # self.math_nest(['SIN','COS','TAN','ASIN','ACOS','ATAN'])
+ # self.math_nest(['POW','LOG'])
+ # self.math_nest(['FLOOR','CEIL','ROUND'])
+ # self.math_nest(['MAVG'])
+ # self.math_nest(['HYPERLOGLOG'])
+ # self.math_nest(['TAIL'])
+ # self.math_nest(['CSUM'])
+ # self.math_nest(['statecount','stateduration'])
+ # self.math_nest(['HISTOGRAM'])
+
+ # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER'])
+ # self.str_nest(['LENGTH','CHAR_LENGTH'])
+ # self.str_nest(['SUBSTR'])
+ # self.str_nest(['CONCAT'])
+ # self.str_nest(['CONCAT_WS'])
+ # self.time_nest(['CAST']) #放到time里起来弄
+ # self.time_nest(['CAST_1'])
+ # self.time_nest(['CAST_2'])
+ # self.time_nest(['CAST_3'])
+ # self.time_nest(['CAST_4'])
+
+
+
+ # self.time_nest(['NOW','TODAY'])
+ # self.time_nest(['TIMEZONE'])
+ # self.time_nest(['TIMETRUNCATE'])
+ # self.time_nest(['TO_ISO8601'])
+ # self.time_nest(['TO_UNIXTIMESTAMP'])
+ # self.time_nest(['ELAPSED'])
+ #self.time_nest(['TIMEDIFF_1'])
+ #self.time_nest(['TIMEDIFF_2'])
+
+
+ endTime = time.time()
+ print("total time %ds" % (endTime - startTime))
+
+
+
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/system-test/2-query/nestedQuery_math.py b/tests/system-test/2-query/nestedQuery_math.py
index 2d0bbcb352..3e37a2c15e 100755
--- a/tests/system-test/2-query/nestedQuery_math.py
+++ b/tests/system-test/2-query/nestedQuery_math.py
@@ -34,9 +34,9 @@ class TDTestCase(TDTestCase):
self.math_nest(['MAVG'])
self.math_nest(['HYPERLOGLOG'])
self.math_nest(['TAIL'])
- self.math_nest(['CSUM'])
- self.math_nest(['statecount','stateduration'])
- self.math_nest(['HISTOGRAM'])
+ # self.math_nest(['CSUM'])
+ # self.math_nest(['statecount','stateduration'])
+ # self.math_nest(['HISTOGRAM'])
# self.str_nest(['LTRIM','RTRIM','LOWER','UPPER'])
# self.str_nest(['LENGTH','CHAR_LENGTH'])
diff --git a/tests/system-test/2-query/out_of_order.py b/tests/system-test/2-query/out_of_order.py
index e07eaed092..8685b11206 100644
--- a/tests/system-test/2-query/out_of_order.py
+++ b/tests/system-test/2-query/out_of_order.py
@@ -168,19 +168,18 @@ class TDTestCase:
sql = "select count(*) from (select distinct(tbname) from %s.meters)" %dbname
tdSql.query(sql)
- num = tdSql.getData(0,0)
+ # 目前不需要了
+ # num = tdSql.getData(0,0)
- for i in range(0,num):
- sql1 = "select count(*) from %s.d%d" %(dbname,i)
- tdSql.query(sql1)
- sql1_result = tdSql.getData(0,0)
- tdLog.info("sql:%s , result: %s" %(sql1,sql1_result))
+ # for i in range(0,num):
+ # sql1 = "select count(*) from %s.d%d" %(dbname,i)
+ # tdSql.query(sql1)
+ # sql1_result = tdSql.getData(0,0)
+ # tdLog.info("sql:%s , result: %s" %(sql1,sql1_result))
+
def check_out_of_order(self,dbname,tables,per_table_num,order,replica):
self.run_benchmark(dbname,tables,per_table_num,order,replica)
- print("sleep 10 seconds")
- #time.sleep(10)
- print("sleep 10 seconds finish")
self.run_sql(dbname)
@@ -188,7 +187,7 @@ class TDTestCase:
startTime = time.time()
#self.check_out_of_order('db1',10,random.randint(10000,50000),random.randint(1,10),1)
- self.check_out_of_order('db1',random.randint(50,200),random.randint(10000,20000),random.randint(1,5),1)
+ self.check_out_of_order('db1',random.randint(50,100),random.randint(10000,20000),random.randint(1,5),1)
# self.check_out_of_order('db2',random.randint(50,200),random.randint(10000,50000),random.randint(5,50),1)
diff --git a/tests/system-test/7-tmq/raw_block_interface_test.py b/tests/system-test/7-tmq/raw_block_interface_test.py
new file mode 100644
index 0000000000..1e89de1cce
--- /dev/null
+++ b/tests/system-test/7-tmq/raw_block_interface_test.py
@@ -0,0 +1,54 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+ def init(self, conn, logSql, replicaVar=1):
+ self.replicaVar = int(replicaVar)
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor())
+
+ def checkData(self):
+ tdSql.execute('use db_raw')
+ tdSql.query("select * from d1")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, 120)
+
+ tdSql.query("select * from d2")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, None)
+
+ return
+
+ def check(self):
+ buildPath = tdCom.getBuildPath()
+ cmdStr = '%s/build/bin/write_raw_block_test'%(buildPath)
+ tdLog.info(cmdStr)
+ os.system(cmdStr)
+
+ self.checkData()
+
+ return
+
+ def run(self):
+ tdSql.prepare()
+ self.check()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py
index 9f01f20470..631813e7ab 100644
--- a/tests/system-test/7-tmq/subscribeDb.py
+++ b/tests/system-test/7-tmq/subscribeDb.py
@@ -13,11 +13,11 @@ from util.dnodes import *
class TDTestCase:
hostname = socket.gethostname()
- #rpcDebugFlagVal = '143'
+ # rpcDebugFlagVal = '143'
#clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
#clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal
#updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
- #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal
+ # updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal
#print ("===================: ", updatecfgDict)
def init(self, conn, logSql, replicaVar=1):
diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py
index 67dee363b3..6d4b7d2380 100644
--- a/tests/system-test/7-tmq/subscribeStb1.py
+++ b/tests/system-test/7-tmq/subscribeStb1.py
@@ -245,8 +245,8 @@ class TDTestCase:
for i in range(expectRows):
totalConsumeRows += resultList[i]
- if totalConsumeRows != expectrowcnt/4:
- tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4))
+ tdLog.info("act consume rows: %d, expect consume rows greater than or equal to: %d"%(totalConsumeRows, expectrowcnt/4))
+ if totalConsumeRows < expectrowcnt/4:
tdLog.exit("tmq consume rows error!")
self.initConsumerInfoTable()
@@ -267,8 +267,8 @@ class TDTestCase:
for i in range(expectRows):
totalConsumeRows += resultList[i]
- if totalConsumeRows != expectrowcnt:
- tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+ tdLog.info("act consume rows: %d, expect consume rows greater than or equal to: %d"%(totalConsumeRows, expectrowcnt))
+ if totalConsumeRows < expectrowcnt:
tdLog.exit("tmq consume rows error!")
tdSql.query("drop topic %s"%topicFromStb1)
diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py
index 6f556382bb..22ef8cebdc 100644
--- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py
+++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py
@@ -212,7 +212,7 @@ class TDTestCase:
# init consume info, and start tmq_sim, then check consume result
tdLog.info("insert consume info to consume processor")
consumerId = 0
- expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+ expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 # because taosd switch, may be consume duplication data
topicList = topicNameList[0]
ifcheckdata = 1
ifManualCommit = 1
@@ -252,11 +252,12 @@ class TDTestCase:
expectRows = 1
resultList = tmqCom.selectConsumeResult(expectRows)
- if expectRowsList[0] != resultList[0]:
- tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+ tdLog.info("expect consume rows: %d should less/equal than act consume rows: %d"%(expectRowsList[0], resultList[0]))
+ if expectRowsList[0] > resultList[0]:
tdLog.exit("0 tmq consume rows error!")
- self.checkFileContent(consumerId, queryString)
+ if expectRowsList[0] == resultList[0]:
+ self.checkFileContent(consumerId, queryString)
time.sleep(10)
for i in range(len(topicNameList)):
diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py
index 2862bcb09b..26f7a9fb4d 100644
--- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py
+++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py
@@ -16,6 +16,8 @@ sys.path.append("./7-tmq")
from tmqCommon import *
class TDTestCase:
+ updatecfgDict = {"tsdbDebugFlag":135}
+
def __init__(self):
self.vgroups = 4
self.ctbNum = 10
diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py
index 6a62247541..6dc8de7efc 100644
--- a/tests/system-test/7-tmq/tmqDelete-1ctb.py
+++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py
@@ -324,7 +324,7 @@ class TDTestCase:
if self.snapshot == 0:
consumerId = 4
- expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4 + 3/4))
+ expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1/4 + 3/4))
elif self.snapshot == 1:
consumerId = 5
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4))
@@ -370,9 +370,14 @@ class TDTestCase:
tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt))
if self.snapshot == 0:
- if totalConsumeRows != expectrowcnt:
+ # If data writing is completed before consumer get snapshot, will consume 7500 from wal;
+ # If data writing has not started before consumer get snapshot, will consume 10000 from wal;
+ minRows = int(expectrowcnt * (1 - 1/4)) # 7500
+ tdLog.info("consume rows should be between %d and %d, "%(minRows, expectrowcnt))
+ if not ((totalConsumeRows >= minRows) and (totalConsumeRows <= expectrowcnt)):
tdLog.exit("tmq consume rows error with snapshot = 0!")
elif self.snapshot == 1:
+ tdLog.info("consume rows should be between %d and %d, "%(totalRowsFromQuery, expectrowcnt))
if not ((totalConsumeRows >= totalRowsFromQuery) and (totalConsumeRows <= expectrowcnt)):
tdLog.exit("tmq consume rows error with snapshot = 1!")
@@ -382,7 +387,113 @@ class TDTestCase:
tdLog.printNoPrefix("======== test case 3 end ...... ")
+ def tmqCase4(self):
+ tdLog.printNoPrefix("======== test case 4: ")
+ paraDict = {'dbName': 'dbt',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbStartIdx': 0,
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 5000,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 15,
+ 'showMsg': 1,
+ 'showRow': 1,
+ 'snapshot': 0}
+ paraDict['snapshot'] = self.snapshot
+ paraDict['vgroups'] = self.vgroups
+ paraDict['ctbNum'] = self.ctbNum
+ paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+ tdLog.info("restart taosd to ensure that the data falls into the disk")
+ tdSql.query("flush database %s"%(paraDict['dbName']))
+
+ tmqCom.initConsumerTable()
+ tdLog.info("create topics from stb1")
+ topicFromStb1 = 'topic_stb1'
+ queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as %s" %(topicFromStb1, queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ # paraDict['ctbNum'] = self.ctbNum
+ paraDict['rowsPerTbl'] = self.rowsPerTbl
+ consumerId = 1
+
+ if self.snapshot == 0:
+ consumerId = 4
+ expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
+ elif self.snapshot == 1:
+ consumerId = 5
+ expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4))
+
+ topicList = topicFromStb1
+ ifcheckdata = 1
+ ifManualCommit = 1
+ keyList = 'group.id:cgrp1,\
+ enable.auto.commit:true,\
+ auto.commit.interval.ms:1000,\
+ auto.offset.reset:earliest'
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ # del some data
+ rowsOfDelete = int(self.rowsPerTbl / 4 )
+ paraDict["endTs"] = paraDict["startTs"] + rowsOfDelete - 1
+ # pDeleteThread = self.asyncDeleteData(paraDict)
+ self.threadFunctionForDeletaData(paraDict)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+
+ tmqCom.getStartConsumeNotifyFromTmqsim()
+
+ # update to 1/4 rows and insert 3/4 new rows
+ paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl * 3 / 4)
+ # paraDict['rowsPerTbl'] = self.rowsPerTbl
+ # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],
+ # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+ # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+ pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+
+ pInsertThread.join()
+
+ tdLog.info("start to check consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+ totalConsumeRows = 0
+ for i in range(expectRows):
+ totalConsumeRows += resultList[i]
+
+ tdSql.query(queryString)
+ totalRowsFromQuery = tdSql.getRows()
+
+ tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt))
+
+ if self.snapshot == 0:
+ tdLog.info("consume rows should be %d"%(expectrowcnt))
+ if (totalConsumeRows != expectrowcnt):
+ tdLog.exit("tmq consume rows error with snapshot = 0!")
+ elif self.snapshot == 1:
+ # If data writing has not started before consumer get snapshot, will consume 10000 from wal, and consumer 7500 from tsdb;
+ tdLog.info("consume rows should be %d, "%(expectrowcnt))
+ if (totalConsumeRows != expectrowcnt):
+ tdLog.exit("tmq consume rows error with snapshot = 1!")
+
+ # tmqCom.checkFileContent(consumerId, queryString)
+
+ tdSql.query("drop topic %s"%topicFromStb1)
+
+ tdLog.printNoPrefix("======== test case 4 end ...... ")
+
def run(self):
# tdSql.prepare()
tdLog.printNoPrefix("=============================================")
@@ -410,6 +521,17 @@ class TDTestCase:
self.prepareTestEnv()
self.tmqCase3()
+ tdLog.printNoPrefix("=============================================")
+ tdLog.printNoPrefix("======== snapshot is 0: only consume from wal")
+ self.snapshot = 0
+ self.prepareTestEnv()
+ self.tmqCase4()
+ tdLog.printNoPrefix("====================================================================")
+ tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal")
+ self.snapshot = 1
+ self.prepareTestEnv()
+ self.tmqCase4()
+
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py
index 3ad7d7692d..648d629e5c 100644
--- a/tests/system-test/7-tmq/tmqDnodeRestart.py
+++ b/tests/system-test/7-tmq/tmqDnodeRestart.py
@@ -137,7 +137,7 @@ class TDTestCase:
tdLog.info("================= restart dnode ===========================")
tdDnodes.stoptaosd(1)
tdDnodes.starttaosd(1)
- # time.sleep(3)
+ time.sleep(5)
tdLog.info(" restart taosd end and wait to check consume result")
expectRows = 1
@@ -222,7 +222,7 @@ class TDTestCase:
tdLog.info("================= restart dnode ===========================")
tdDnodes.stoptaosd(1)
tdDnodes.starttaosd(1)
- # time.sleep(3)
+ time.sleep(5)
tdLog.info("create some new child table and insert data ")
paraDict["batchNum"] = 100
diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py
index 198a5ed6df..7a9c1bbb8c 100644
--- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py
+++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py
@@ -101,7 +101,7 @@ class TDTestCase:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
- if not ((totalConsumeRows >= expectrowcnt * 3/4) and (totalConsumeRows < expectrowcnt)):
+ if not ((totalConsumeRows >= expectrowcnt * 3/4) and (totalConsumeRows <= expectrowcnt)):
tdLog.exit("tmq consume rows error with snapshot = 0!")
tdLog.info("wait subscriptions exit ....")
@@ -132,7 +132,7 @@ class TDTestCase:
'batchNum': 100,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'endTs': 0,
- 'pollDelay': 10,
+ 'pollDelay': 20,
'showMsg': 1,
'showRow': 1,
'snapshot': 0}
@@ -195,7 +195,7 @@ class TDTestCase:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
- if not ((totalConsumeRows >= expectrowcnt / 2 * (1 + 3/4)) and (totalConsumeRows < expectrowcnt)):
+ if not ((totalConsumeRows >= expectrowcnt / 2 * (1 + 3/4)) and (totalConsumeRows <= expectrowcnt)):
tdLog.exit("tmq consume rows error with snapshot = 0!")
tdLog.info("wait subscriptions exit ....")
diff --git a/tests/system-test/7-tmq/tmqMultiConsumer.py b/tests/system-test/7-tmq/tmqMultiConsumer.py
new file mode 100644
index 0000000000..cc217e0c4c
--- /dev/null
+++ b/tests/system-test/7-tmq/tmqMultiConsumer.py
@@ -0,0 +1,174 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+# from tests.pytest.util.common import TDCom
+# from tests.pytest.util.log import TDLog
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+ def __init__(self):
+ self.vgroups = 32
+ self.ctbNum = 100
+ self.rowsPerTbl = 1000
+ self.snapshot = 1
+ self.replicaVar = 3
+
+ def init(self, conn, logSql, replicaVar=1):
+ self.replicaVar = int(replicaVar)
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor(), False)
+
+ def prepareTestEnv(self):
+ tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+ paraDict = {'dbName': 'dbt',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbStartIdx': 0,
+ 'ctbNum': 10,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 1000,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1,
+ 'snapshot': 0}
+
+ paraDict['vgroups'] = self.vgroups
+ paraDict['ctbNum'] = self.ctbNum
+ paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+ tmqCom.initConsumerTable()
+ tmqCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar)
+ tdLog.info("create stb")
+ tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+ tdLog.info("create ctb")
+ tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+ ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+ tdLog.info("insert data")
+ tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+ ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+ startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+
+ # tdLog.info("restart taosd to ensure that the data falls into the disk")
+ # tdDnodes.stop(1)
+ # tdDnodes.start(1)
+ # tdSql.query("flush database %s"%(paraDict['dbName']))
+ return
+
+ def tmqCase1(self):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'dbt',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 1,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbStartIdx': 0,
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 1000,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 20,
+ 'showMsg': 1,
+ 'showRow': 1,
+ 'snapshot': 1}
+
+ paraDict['vgroups'] = self.vgroups
+ paraDict['ctbNum'] = self.ctbNum
+ paraDict['rowsPerTbl'] = self.rowsPerTbl
+ paraDict['snapshot'] = self.snapshot
+
+ topicNameList = ['topic1', 'topic2']
+ tmqCom.initConsumerTable()
+
+ tdLog.info("create topics from stb with filter")
+ # queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+9379)
+ queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s "%(paraDict['dbName'], paraDict['stbName'])
+ # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+ sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ sqlString = "create topic %s as %s" %(topicNameList[1], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ # tdSql.query(queryString)
+ # expectRowsList.append(tdSql.getRows())
+
+ # init consume info, and start tmq_sim, then check consume result
+ tdLog.info("insert consume info to consume processor")
+ consumerId = 0
+ expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 4
+ topicList = topicNameList[0] + ',' + topicNameList[0] + ',' + topicNameList[1]
+ ifcheckdata = 0
+ ifManualCommit = 1
+ keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ consumerId = 1
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+ tdLog.info("wait the consume result")
+
+ # continue to insert new rows
+ paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl)
+ pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+ pInsertThread.join()
+
+ expectRows = 2
+ resultList = tmqCom.selectConsumeResult(expectRows)
+ actConsumeTotalRows = resultList[0] + resultList[1]
+
+ tdLog.info("act consume rows: %d, expect consume rows: %d"%(actConsumeTotalRows, expectrowcnt))
+
+ if not ((expectrowcnt <= actConsumeTotalRows) or ((resultList[0] == 0) and (resultList[1] >= expectrowcnt)) or ((resultList[1] == 0) and (resultList[0] >= expectrowcnt))):
+ tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt, actConsumeTotalRows))
+ tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+ # tmqCom.checkFileContent(consumerId, queryString)
+
+ time.sleep(10)
+ for i in range(len(topicNameList)):
+ tdSql.query("drop topic %s"%topicNameList[i])
+
+ tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+
+ def run(self):
+ tdSql.prepare()
+ self.prepareTestEnv()
+ self.tmqCase1()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c
index 1eb4edb28a..0f5585991e 100644
--- a/tools/shell/src/shellArguments.c
+++ b/tools/shell/src/shellArguments.c
@@ -18,6 +18,19 @@
#endif
#include "shellInt.h"
+#include "version.h"
+
+#ifndef CUS_NAME
+char cusName[] = "TDengine";
+#endif
+
+#ifndef CUS_PROMPT
+char cusPrompt[] = "taos";
+#endif
+
+#ifndef CUS_EMAIL
+char cusEmail[] = "";
+#endif
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@@ -46,9 +59,9 @@
#define SHELL_VERSION "Print program version."
#ifdef WEBSOCKET
-#define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection."
-#define SHELL_REST "Use RESTful mode when connecting."
-#define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 30."
+#define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection."
+#define SHELL_REST "Use RESTful mode when connecting."
+#define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 30."
#endif
static int32_t shellParseSingleOpt(int32_t key, char *arg);
@@ -141,7 +154,7 @@ static void shellParseArgsUseArgp(int argc, char *argv[]) {
#endif
#ifndef ARGP_ERR_UNKNOWN
- #define ARGP_ERR_UNKNOWN E2BIG
+#define ARGP_ERR_UNKNOWN E2BIG
#endif
static int32_t shellParseSingleOpt(int32_t key, char *arg) {
@@ -242,8 +255,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
SShellArgs *pArgs = &shell.args;
for (int i = 1; i < argc; i++) {
- if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0
- || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) {
+ if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 ||
+ strcmp(argv[i], "/?") == 0) {
shellParseSingleOpt('?', NULL);
return 0;
}
@@ -259,10 +272,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
return -1;
}
- if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u'
- || key[1] == 'a' || key[1] == 'c' || key[1] == 's'
- || key[1] == 'f' || key[1] == 'd' || key[1] == 'w'
- || key[1] == 'n' || key[1] == 'l' || key[1] == 'N'
+ if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' ||
+ key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N'
#ifdef WEBSOCKET
|| key[1] == 'E' || key[1] == 'T'
#endif
@@ -278,12 +289,10 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
}
shellParseSingleOpt(key[1], val);
i++;
- } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C'
- || key[1] == 'r' || key[1] == 'k'
- || key[1] == 't' || key[1] == 'V'
- || key[1] == '?' || key[1] == 1
+ } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' ||
+ key[1] == 'V' || key[1] == '?' || key[1] == 1
#ifdef WEBSOCKET
- ||key[1] == 'R'
+ || key[1] == 'R'
#endif
) {
shellParseSingleOpt(key[1], NULL);
@@ -425,9 +434,15 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) {
#endif
sprintf(shell.info.promptContinue, promptContinueFormat, " ");
shell.info.promptSize = strlen(shell.info.promptHeader);
+#ifdef TD_ENTERPRISE
+ snprintf(shell.info.programVersion, sizeof(shell.info.programVersion),
+ "version: %s compatible_version: %s\ngitinfo: %s\ngitinfoOfInternal: %s\nbuildInfo: %s", version,
+ compatible_version, gitinfo, gitinfoOfInternal, buildinfo);
+#else
snprintf(shell.info.programVersion, sizeof(shell.info.programVersion),
"version: %s compatible_version: %s\ngitinfo: %s\nbuildInfo: %s", version, compatible_version, gitinfo,
buildinfo);
+#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
shell.info.osname = "Windows";
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 616540a54a..70130397ba 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -543,7 +543,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
printf("%*" PRIu64, width, *((uint64_t *)val));
break;
case TSDB_DATA_TYPE_FLOAT:
- printf("%*.5f", width, GET_FLOAT_VAL(val));
+ printf("%*ef", width, GET_FLOAT_VAL(val));
break;
case TSDB_DATA_TYPE_DOUBLE:
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val));
diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt
index 6ca266c555..eda9c70f15 100644
--- a/utils/test/c/CMakeLists.txt
+++ b/utils/test/c/CMakeLists.txt
@@ -3,6 +3,7 @@ add_dependencies(tmq_demo taos)
add_executable(tmq_sim tmqSim.c)
add_executable(create_table createTable.c)
add_executable(tmq_taosx_ci tmq_taosx_ci.c)
+add_executable(write_raw_block_test write_raw_block_test.c)
add_executable(sml_test sml_test.c)
add_executable(get_db_name_test get_db_name_test.c)
target_link_libraries(
@@ -34,6 +35,14 @@ target_link_libraries(
PUBLIC os
)
+target_link_libraries(
+ write_raw_block_test
+ PUBLIC taos_static
+ PUBLIC util
+ PUBLIC common
+ PUBLIC os
+)
+
target_link_libraries(
sml_test
PUBLIC taos_static
diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c
index 873946121b..b0cc6f749c 100644
--- a/utils/test/c/sml_test.c
+++ b/utils/test/c/sml_test.c
@@ -850,7 +850,7 @@ int smlProcess_18784_Test() {
taos_free_result(pRes);
const char *sql[] = {
- "disk,device=sdc inodes_used=176059i,total=1081101176832i 1661943960000000000",
+ "disk,device=sdc inodes_used=176059i,total=1076048383523889174i 1661943960000000000",
"disk,device=sdc inodes_free=66932805i 1661943960000000000",
};
pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0);
@@ -875,7 +875,7 @@ int smlProcess_18784_Test() {
if (rowIndex == 0) {
ASSERT(ts == 1661943960000);
ASSERT(used == 176059);
- ASSERT(total == 1081101176832);
+ ASSERT(total == 1076048383523889174);
ASSERT(freed == 66932805);
// ASSERT_EQ(latitude, 24.5208);
// ASSERT_EQ(longitude, 28.09377);
@@ -939,6 +939,9 @@ int sml_ts2164_Test() {
// "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27",
"meters,location=la,groupid=ca current=11.8,voltage=221",
"meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27",
+ "ts3038,location=l2a,groupid=ca current=L\"11.8\"",
+ "ts3038,location=l2a,groupid=ca voltage=L\"221\"",
+ "ts3038,location=l2a,groupid=ca phase=L\"221\"",
// "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27",
};
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 1cc2a48469..c4becdd381 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -542,7 +542,6 @@ tmq_t* build_consumer() {
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set(conf, "enable.auto.commit", "true");
- tmq_conf_set(conf, "enable.heartbeat.background", "true");
if (g_conf.snapShot) {
tmq_conf_set(conf, "experimental.snapshot.enable", "true");
diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c
new file mode 100644
index 0000000000..8e5dd62752
--- /dev/null
+++ b/utils/test/c/write_raw_block_test.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+#include
+#include
+#include "taos.h"
+#include "types.h"
+
+int buildStable(TAOS* pConn, TAOS_RES* pRes) {
+ pRes = taos_query(pConn,
+ "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT) TAGS "
+ "(`groupid` INT, `location` VARCHAR(16))");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create super table meters, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "create table d0 using meters tags(1, 'San Francisco')");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create child table d0, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "insert into d0 (ts, current) values (now, 120)");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to insert into table d0, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "create table d1 using meters tags(2, 'San Francisco')");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "create table d2 using meters tags(3, 'San Francisco')");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ return 0;
+}
+
+int32_t init_env() {
+ TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+ if (pConn == NULL) {
+ return -1;
+ }
+
+ TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_raw");
+ if (taos_errno(pRes) != 0) {
+ printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "create database if not exists db_raw vgroups 2");
+ if (taos_errno(pRes) != 0) {
+ printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "use db_raw");
+ if (taos_errno(pRes) != 0) {
+ printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ buildStable(pConn, pRes);
+
+ pRes = taos_query(pConn, "select * from d0");
+ if (taos_errno(pRes) != 0) {
+ printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ void *data = NULL;
+ int32_t numOfRows = 0;
+ int error_code = taos_fetch_raw_block(pRes, &numOfRows, &data);
+ ASSERT(error_code == 0);
+ ASSERT(numOfRows == 1);
+
+ taos_write_raw_block(pConn, numOfRows, data, "d1");
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "select ts,phase from d0");
+ if (taos_errno(pRes) != 0) {
+ printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ error_code = taos_fetch_raw_block(pRes, &numOfRows, &data);
+ ASSERT(error_code == 0);
+ ASSERT(numOfRows == 1);
+
+ int numFields = taos_num_fields(pRes);
+ TAOS_FIELD *fields = taos_fetch_fields(pRes);
+ taos_write_raw_block_with_fields(pConn, numOfRows, data, "d2", fields, numFields);
+ taos_free_result(pRes);
+
+ taos_close(pConn);
+ return 0;
+}
+
+int main(int argc, char* argv[]) {
+ if (init_env() < 0) {
+ return -1;
+ }
+}