diff --git a/README.md b/README.md
index 8d2567a816..6aec756ec7 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ TDengine is an open source, high-performance, cloud native [time-series database
- **[Easy Data Analytics](https://tdengine.com/tdengine/time-series-data-analytics-made-easy/)**: Through super tables, storage and compute separation, data partitioning by time interval, pre-computation and other means, TDengine makes it easy to explore, format, and get access to data in a highly efficient way.
-- **[Open Source](https://tdengine.com/tdengine/open-source-time-series-database/)**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 18.8k stars on GitHub. There is an active developer community, and over 139k running instances worldwide.
+- **[Open Source](https://tdengine.com/tdengine/open-source-time-series-database/)**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 19.9k stars on GitHub. There is an active developer community, and over 139k running instances worldwide.
For a full list of TDengine competitive advantages, please [check here](https://tdengine.com/tdengine/). The easiest way to experience TDengine is through [TDengine Cloud](https://cloud.tdengine.com).
diff --git a/cmake/cmake.define b/cmake/cmake.define
index 3f152f1f09..e34785cba6 100644
--- a/cmake/cmake.define
+++ b/cmake/cmake.define
@@ -123,14 +123,20 @@ ELSE ()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
ENDIF ()
- MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")
IF (TD_INTEL_64 OR TD_INTEL_32)
ADD_DEFINITIONS("-msse4.2")
IF("${FMA_SUPPORT}" MATCHES "true")
- MESSAGE(STATUS "turn fma function support on")
+ MESSAGE(STATUS "fma function supported")
ADD_DEFINITIONS("-mfma")
ELSE ()
- MESSAGE(STATUS "turn fma function support off")
+ MESSAGE(STATUS "fma function NOT supported")
+ ENDIF()
+
+ IF("${SIMD_SUPPORT}" MATCHES "true")
+ ADD_DEFINITIONS("-mavx -mavx2")
+ MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED")
+ ELSE()
+ MESSAGE(STATUS "SIMD instruction (AVX/AVX2)is NOT ACTIVATED")
ENDIF()
ENDIF ()
diff --git a/cmake/cmake.platform b/cmake/cmake.platform
index eb8b63b4c2..e822cc5e2c 100644
--- a/cmake/cmake.platform
+++ b/cmake/cmake.platform
@@ -1,20 +1,17 @@
cmake_minimum_required(VERSION 3.0)
-MESSAGE("Current system is ${CMAKE_SYSTEM_NAME}")
-
# init
SET(TD_LINUX FALSE)
SET(TD_WINDOWS FALSE)
SET(TD_DARWIN FALSE)
-MESSAGE("Compiler ID: ${CMAKE_CXX_COMPILER_ID}")
if(CMAKE_COMPILER_IS_GNUCXX MATCHES 1)
set(CXX_COMPILER_IS_GNU TRUE)
else()
set(CXX_COMPILER_IS_GNU FALSE)
endif()
-MESSAGE("Current system name is ${CMAKE_SYSTEM_NAME}.")
+MESSAGE("Current system: ${CMAKE_SYSTEM_NAME}")
IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
@@ -26,6 +23,8 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
ENDIF ()
+ MESSAGE("Current system processor: ${CMAKE_SYSTEM_PROCESSOR}")
+
IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
SET(TD_LINUX TRUE)
@@ -44,7 +43,6 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
SET(OSTYPE "macOS")
ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare")
- MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.")
IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64")
MESSAGE("Current system arch is arm64")
SET(TD_DARWIN_64 TRUE)
@@ -80,24 +78,22 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
ENDIF()
IF ("${CPUTYPE}" STREQUAL "")
- MESSAGE(STATUS "The current platform " ${CMAKE_SYSTEM_PROCESSOR} " is detected")
-
IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)")
- MESSAGE(STATUS "The current platform is amd64")
+ MESSAGE(STATUS "Current platform is amd64")
SET(PLATFORM_ARCH_STR "amd64")
SET(TD_INTEL_64 TRUE)
ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)")
- MESSAGE(STATUS "The current platform is x86")
+ MESSAGE(STATUS "Current platform is x86")
SET(PLATFORM_ARCH_STR "i386")
SET(TD_INTEL_32 TRUE)
ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l")
- MESSAGE(STATUS "The current platform is aarch32")
+ MESSAGE(STATUS "Current platform is aarch32")
SET(PLATFORM_ARCH_STR "arm")
SET(TD_ARM_32 TRUE)
ADD_DEFINITIONS("-D_TD_ARM_")
ADD_DEFINITIONS("-D_TD_ARM_32")
ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
- MESSAGE(STATUS "The current platform is aarch64")
+ MESSAGE(STATUS "Current platform is aarch64")
SET(PLATFORM_ARCH_STR "arm64")
SET(TD_ARM_64 TRUE)
ADD_DEFINITIONS("-D_TD_ARM_")
@@ -151,5 +147,5 @@ ENDIF ()
MESSAGE(STATUS "platform arch:" ${PLATFORM_ARCH_STR})
-MESSAGE("C Compiler ID: ${CMAKE_C_COMPILER_ID}")
-MESSAGE("CXX Compiler ID: ${CMAKE_CXX_COMPILER_ID}")
+MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})")
+MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})")
diff --git a/cmake/cmake.version b/cmake/cmake.version
index c379d5e739..9faa7c75dd 100644
--- a/cmake/cmake.version
+++ b/cmake/cmake.version
@@ -26,7 +26,7 @@ ELSEIF (HAVE_GIT)
SET(TD_VER_GIT "no git commit id")
ENDIF ()
ELSE ()
- message(STATUS "no git cmd")
+ message(STATUS "no git found")
SET(TD_VER_GIT "no git commit id")
ENDIF ()
@@ -70,9 +70,9 @@ MESSAGE(STATUS "compatible: " ${TD_VER_COMPATIBLE})
MESSAGE(STATUS "commit id: " ${TD_VER_GIT})
MESSAGE(STATUS "build date: " ${TD_VER_DATE})
MESSAGE(STATUS "build type: " ${CMAKE_BUILD_TYPE})
-MESSAGE(STATUS "type: " ${TD_VER_VERTYPE})
-MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE})
-MESSAGE(STATUS "os: " ${TD_VER_OSTYPE})
+MESSAGE(STATUS "type: " ${TD_VER_VERTYPE})
+MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE})
+MESSAGE(STATUS "os: " ${TD_VER_OSTYPE})
MESSAGE(STATUS "============= compile version parameter information end ============= ")
STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER})
diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 30e7b85495..577166353b 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
- GIT_TAG 4cbb9ac
+ GIT_TAG 7e9ce09
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/examples/JDBC/taosdemo/pom.xml b/examples/JDBC/taosdemo/pom.xml
index 724ecc7407..68224bbad5 100644
--- a/examples/JDBC/taosdemo/pom.xml
+++ b/examples/JDBC/taosdemo/pom.xml
@@ -88,7 +88,7 @@
org.apache.logging.log4j
log4j-core
- 2.17.1
+ 2.17.2
diff --git a/examples/JDBC/taosdemo/readme.md b/examples/JDBC/taosdemo/readme.md
index e5f4eb132b..edac970399 100644
--- a/examples/JDBC/taosdemo/readme.md
+++ b/examples/JDBC/taosdemo/readme.md
@@ -2,12 +2,10 @@
cd tests/examples/JDBC/taosdemo
mvn clean package -Dmaven.test.skip=true
# 先建表，再插入的
-java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable true -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100
+java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host -database -doCreateTable true -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100
# 不建表，直接插入的
-java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100
+java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host -database -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100
```
-需求：
-1. 可以读lowa的配置文件
-2. 支持JDBC-JNI和JDBC-restful
-3. 读取配置文件，持续执行查询
\ No newline at end of file
+如果发生错误 Exception in thread "main" java.lang.UnsatisfiedLinkError: no taos in java.library.path
+请检查是否安装 TDengine 客户端安装包或编译 TDengine 安装。如果确定已经安装过还出现这个错误，可以在命令行 java 后加 -Djava.library.path=/usr/local/lib 来指定寻找共享库的路径。
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 674bdcf171..9c1f2063a7 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -225,13 +225,13 @@ typedef struct SVarColAttr {
// pBlockAgg->numOfNull == info.rows, all data are null
// pBlockAgg->numOfNull == 0, no data are null.
typedef struct SColumnInfoData {
- char* pData; // the corresponding block data in memory
+ char* pData; // the corresponding block data in memory
union {
char* nullbitmap; // bitmap, one bit for each item in the list
SVarColAttr varmeta;
};
- SColumnInfo info; // column info
- bool hasNull; // if current column data has null value.
+ SColumnInfo info; // column info
+ bool hasNull; // if current column data has null value.
} SColumnInfoData;
typedef struct SQueryTableDataCond {
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 9982a187d7..db6ca65cf0 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -24,11 +24,6 @@
extern "C" {
#endif
-typedef struct SCorEpSet {
- int32_t version;
- SEpSet epSet;
-} SCorEpSet;
-
typedef struct SBlockOrderInfo {
bool nullFirst;
int32_t order;
@@ -36,14 +31,6 @@ typedef struct SBlockOrderInfo {
SColumnInfoData* pColData;
} SBlockOrderInfo;
-int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
-void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
-
-bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
-
-void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet);
-SEpSet getEpSet_s(SCorEpSet* pEpSet);
-
#define NBIT (3u)
#define BitPos(_n) ((_n) & ((1 << NBIT) - 1))
#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT])
diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 2076906f70..0ded2bc4b5 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -44,6 +44,8 @@ extern int32_t tsCompatibleModel;
extern bool tsPrintAuth;
extern int64_t tsTickPerMin[3];
extern int32_t tsCountAlwaysReturnValue;
+extern float tsSelectivityRatio;
+extern int32_t tsTagFilterResCacheSize;
// queue & threads
extern int32_t tsNumOfRpcThreads;
diff --git a/include/common/tmisce.h b/include/common/tmisce.h
new file mode 100644
index 0000000000..b9f5cf5b91
--- /dev/null
+++ b/include/common/tmisce.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_TMISCE_H
+#define TDENGINE_TMISCE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tmsg.h"
+
+typedef struct SCorEpSet {
+ int32_t version;
+ SEpSet epSet;
+} SCorEpSet;
+
+int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
+void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
+
+bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
+
+void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet);
+SEpSet getEpSet_s(SCorEpSet* pEpSet);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TDENGINE_TMISCE_H
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 2b1af66cc6..cde3315ffb 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -647,34 +647,6 @@ int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pR
int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp);
-typedef struct {
- int16_t lowerRelOptr;
- int16_t upperRelOptr;
- int16_t filterstr; // denote if current column is char(binary/nchar)
-
- union {
- struct {
- int64_t lowerBndi;
- int64_t upperBndi;
- };
- struct {
- double lowerBndd;
- double upperBndd;
- };
- struct {
- int64_t pz;
- int64_t len;
- };
- };
-} SColumnFilterInfo;
-
-typedef struct {
- int16_t numOfFilters;
- union {
- int64_t placeholder;
- SColumnFilterInfo* filterInfo;
- };
-} SColumnFilterList;
/*
* for client side struct, only column id, type, bytes are necessary
* But for data in vnode side, we need all the following information.
@@ -685,10 +657,10 @@ typedef struct {
int16_t slotId;
};
- int8_t type;
- int32_t bytes;
uint8_t precision;
uint8_t scale;
+ int32_t bytes;
+ int8_t type;
} SColumnInfo;
typedef struct STimeWindow {
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index 1fe30a2d66..0bca254e14 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -142,8 +142,11 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
*/
int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal);
+
int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds);
+void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo);
+
/**
* kill the ongoing query asynchronously
* @param tinfo qhandle
diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 6f2a675466..240772bfc2 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -115,7 +115,7 @@ typedef struct SInputColumnInfoData {
int32_t startRowIndex; // handle started row index
int32_t numOfRows; // the number of rows needs to be handled
int32_t numOfInputCols; // PTS is not included
- bool colDataAggIsSet; // if agg is set or not
+ bool colDataSMAIsSet; // if agg is set or not
SColumnInfoData *pPTS; // primary timestamp column
SColumnInfoData **pData;
SColumnDataAgg **pColumnDataAgg;
diff --git a/include/os/os.h b/include/os/os.h
index e780611c41..ab4d0a406e 100644
--- a/include/os/os.h
+++ b/include/os/os.h
@@ -43,6 +43,7 @@ extern "C" {
#include
#include
#include
+#include
#if defined(DARWIN)
#else
@@ -82,6 +83,12 @@ extern "C" {
#include
#include
+#if __AVX__
+#include
+#elif __SSE4_2__
+#include
+#endif
+
#include "osThread.h"
#include "osAtomic.h"
diff --git a/include/os/osDef.h b/include/os/osDef.h
index 297d19e21a..0bf9c6184e 100644
--- a/include/os/osDef.h
+++ b/include/os/osDef.h
@@ -168,22 +168,22 @@ void syslog(int unused, const char *format, ...);
} \
} while (0)
-#define DEFAULT_DOUBLE_COMP(x, y) \
- do { \
- if (isnan(x) && isnan(y)) { \
- return 0; \
- } \
- if (isnan(x)) { \
- return -1; \
- } \
- if (isnan(y)) { \
- return 1; \
- } \
- if ((x) == (y)) { \
- return 0; \
- } else { \
- return (x) < (y) ? -1 : 1; \
- } \
+#define DEFAULT_DOUBLE_COMP(x, y) \
+ do { \
+ if (isnan(x) && isnan(y)) { \
+ return 0; \
+ } \
+ if (isnan(x)) { \
+ return -1; \
+ } \
+ if (isnan(y)) { \
+ return 1; \
+ } \
+ if (fabs((x) - (y)) <= DBL_EPSILON) { \
+ return 0; \
+ } else { \
+ return (x) < (y) ? -1 : 1; \
+ } \
} while (0)
#define DEFAULT_FLOAT_COMP(x, y) DEFAULT_DOUBLE_COMP(x, y)
diff --git a/include/os/osEnv.h b/include/os/osEnv.h
index c1fdc9e404..a3bd209693 100644
--- a/include/os/osEnv.h
+++ b/include/os/osEnv.h
@@ -36,6 +36,11 @@ extern int64_t tsStreamMax;
extern float tsNumOfCores;
extern int64_t tsTotalMemoryKB;
extern char *tsProcPath;
+extern char tsSIMDEnable;
+extern char tsSSE42Enable;
+extern char tsAVXEnable;
+extern char tsAVX2Enable;
+extern char tsFMAEnable;
extern char configDir[];
extern char tsDataDir[];
diff --git a/include/os/osMemory.h b/include/os/osMemory.h
index 14d53a7a06..4681ff6674 100644
--- a/include/os/osMemory.h
+++ b/include/os/osMemory.h
@@ -37,6 +37,7 @@ void taosMemoryFree(void *ptr);
int64_t taosMemorySize(void *ptr);
void taosPrintBackTrace();
void taosMemoryTrim(int32_t size);
+void *taosMemoryMallocAlign(uint32_t alignment, int64_t size);
#define taosMemoryFreeClear(ptr) \
do { \
diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h
index 47cdb02a6f..7765a60f88 100644
--- a/include/os/osSysinfo.h
+++ b/include/os/osSysinfo.h
@@ -40,6 +40,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
int32_t taosGetCpuCores(float *numOfCores);
void taosGetCpuUsage(double *cpu_system, double *cpu_engine);
+int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma);
int32_t taosGetTotalMemory(int64_t *totalKB);
int32_t taosGetProcMemory(int64_t *usedKB);
int32_t taosGetSysMemory(int64_t *usedKB);
diff --git a/include/util/tlist.h b/include/util/tlist.h
index 1954bda145..3dbdb72f9e 100644
--- a/include/util/tlist.h
+++ b/include/util/tlist.h
@@ -225,7 +225,7 @@ void *tdListFree(SList *list);
void tdListPrependNode(SList *list, SListNode *node);
void tdListAppendNode(SList *list, SListNode *node);
int32_t tdListPrepend(SList *list, void *data);
-int32_t tdListAppend(SList *list, void *data);
+int32_t tdListAppend(SList *list, const void *data);
SListNode *tdListPopHead(SList *list);
SListNode *tdListPopTail(SList *list);
SListNode *tdListGetHead(SList *list);
diff --git a/include/util/tpagedbuf.h b/include/util/tpagedbuf.h
index 5f81ccfcfd..73af65997d 100644
--- a/include/util/tpagedbuf.h
+++ b/include/util/tpagedbuf.h
@@ -24,12 +24,9 @@
extern "C" {
#endif
-typedef struct SArray* SIDList;
typedef struct SPageInfo SPageInfo;
typedef struct SDiskbasedBuf SDiskbasedBuf;
-#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024LL) // in bytes
-
typedef struct SFilePage {
int32_t num;
char data[];
@@ -69,7 +66,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId);
* @param pBuf
* @return
*/
-SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf);
+SArray* getDataBufPagesIdList(SDiskbasedBuf* pBuf);
/**
* get the specified buffer page by id
@@ -111,7 +108,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf);
* @param pList
* @return
*/
-SPageInfo* getLastPageInfo(SIDList pList);
+SPageInfo* getLastPageInfo(SArray* pList);
/**
*
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index e569a97723..aae20c587d 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -26,7 +26,7 @@ extern "C" {
#include "query.h"
#include "taos.h"
#include "tcommon.h"
-#include "tdatablock.h"
+#include "tmisce.h"
#include "tdef.h"
#include "thash.h"
#include "tlist.h"
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 423cfcdc5b..976d1dd1b0 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -20,13 +20,13 @@
#include "functionMgt.h"
#include "os.h"
#include "query.h"
-#include "qworker.h"
#include "scheduler.h"
#include "tglobal.h"
#include "tmsg.h"
#include "tref.h"
#include "trpc.h"
#include "version.h"
+#include "tdatablock.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c
index b228deb8ed..4bd74a842f 100644
--- a/source/client/src/clientMsgHandler.c
+++ b/source/client/src/clientMsgHandler.c
@@ -20,6 +20,7 @@
#include "query.h"
#include "tdef.h"
#include "tname.h"
+#include "tdatablock.h"
#include "systable.h"
static void setErrno(SRequestObj* pRequest, int32_t code) {
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 1ae8dafcf3..cfa2964e16 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -19,6 +19,8 @@
#include "tlog.h"
#include "tname.h"
+#define MALLOC_ALIGN_BYTES 32
+
int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {
ASSERT(pColumnInfoData != NULL);
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
@@ -1174,6 +1176,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo*
pColumn->varmeta.offset = (int32_t*)tmp;
memset(&pColumn->varmeta.offset[existedRows], 0, sizeof(int32_t) * (numOfRows - existedRows));
} else {
+ // prepare for the null bitmap
char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows));
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@@ -1184,11 +1187,19 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo*
memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen);
ASSERT(pColumn->info.bytes);
- tmp = taosMemoryRealloc(pColumn->pData, numOfRows * pColumn->info.bytes);
+
+ // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned
+ tmp = taosMemoryMallocAlign(MALLOC_ALIGN_BYTES, numOfRows * pColumn->info.bytes);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
+ // copy back the existed data
+ if (pColumn->pData != NULL) {
+ memcpy(tmp, pColumn->pData, existedRows * pColumn->info.bytes);
+ taosMemoryFreeClear(pColumn->pData);
+ }
+
pColumn->pData = tmp;
if (clearPayload) {
memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows));
@@ -1311,12 +1322,9 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
- if (pSrc->pData == NULL) {
- continue;
- }
-
colDataAssign(pDst, pSrc, src->info.rows, &src->info);
}
+
uint32_t cap = dst->info.capacity;
dst->info = src->info;
dst->info.capacity = cap;
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index f2d8b9aa7c..2e3389fb62 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -15,9 +15,8 @@
#define _DEFAULT_SOURCE
#include "tglobal.h"
-#include "tcompare.h"
#include "tconfig.h"
-#include "tdatablock.h"
+#include "tmisce.h"
#include "tgrant.h"
#include "tlog.h"
@@ -120,6 +119,9 @@ int32_t tsMinIntervalTime = 1;
// maximum memory allowed to be allocated for a single csv load (in MB)
int32_t tsMaxMemUsedByInsert = 1024;
+float tsSelectivityRatio = 1.0;
+int32_t tsTagFilterResCacheSize = 1024*10;
+
// the maximum allowed query buffer size during query processing for each data node.
// -1 no limit (default)
// 0 no query allowed, queries are disabled
@@ -318,7 +320,14 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1;
if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1;
if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1;
- if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1;
+ if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1;
+
+ if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, 0) != 0) return -1;
+ if (cfgAddBool(pCfg, "AVX", tsAVXEnable, 0) != 0) return -1;
+ if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, 0) != 0) return -1;
+ if (cfgAddBool(pCfg, "FMA", tsFMAEnable, 0) != 0) return -1;
+ if (cfgAddBool(pCfg, "SIMD-Supported", tsSIMDEnable, 0) != 0) return -1;
+
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1;
diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c
index dfaebc99f6..dfb1917fcf 100644
--- a/source/common/src/tmisce.c
+++ b/source/common/src/tmisce.c
@@ -14,7 +14,7 @@
*/
#define _DEFAULT_SOURCE
-#include "tdatablock.h"
+#include "tmisce.h"
#include "tglobal.h"
#include "tlog.h"
#include "tname.h"
diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c
index 7fe7d44827..4bc2e64fad 100644
--- a/source/dnode/mgmt/node_util/src/dmEps.c
+++ b/source/dnode/mgmt/node_util/src/dmEps.c
@@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "dmUtil.h"
+#include "tmisce.h"
static void dmPrintEps(SDnodeData *pData);
static bool dmIsEpChanged(SDnodeData *pData, int32_t dnodeId, const char *ep);
diff --git a/source/dnode/mgmt/test/sut/src/client.cpp b/source/dnode/mgmt/test/sut/src/client.cpp
index 6b4c23c0de..a27a511651 100644
--- a/source/dnode/mgmt/test/sut/src/client.cpp
+++ b/source/dnode/mgmt/test/sut/src/client.cpp
@@ -15,6 +15,7 @@
#include "sut.h"
#include "tdatablock.h"
+#include "tmisce.h"
static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
TestClient* client = (TestClient*)parent;
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index 521f924fad..2a3ecf1924 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -24,6 +24,7 @@
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
+#include "tmisce.h"
#define TSDB_DNODE_VER_NUMBER 1
#define TSDB_DNODE_RESERVE_SIZE 64
diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c
index 563b3cd3cf..b02ca4c773 100644
--- a/source/dnode/mnode/impl/src/mndMnode.c
+++ b/source/dnode/mnode/impl/src/mndMnode.c
@@ -20,7 +20,7 @@
#include "mndShow.h"
#include "mndSync.h"
#include "mndTrans.h"
-#include "mndUser.h"
+#include "tmisce.h"
#define MNODE_VER_NUMBER 1
#define MNODE_RESERVE_SIZE 64
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index 3c1d3f09bf..f71bd1c626 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -585,6 +585,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
} else {
pVgEp->qmsg = strdup("");
}
+
+ sdbRelease(pSdb, pVgroup);
}
ASSERT(pSub->unassignedVgs->size > 0);
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 62247f2c2a..d8cf7a837e 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -525,6 +525,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
tFreeSMCreateStbReq(&createReq);
mndFreeStb(&stbObj);
+ mndReleaseStb(pMnode, pStb);
mndReleaseDb(pMnode, pDb);
return 0;
diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c
index 522036afa2..6412761f0b 100644
--- a/source/dnode/mnode/impl/src/mndTopic.c
+++ b/source/dnode/mnode/impl/src/mndTopic.c
@@ -434,6 +434,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
return -1;
}
topicObj.stbUid = pStb->uid;
+ mndReleaseStb(pMnode, pStb);
}
/*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/
/*topicObj.ast = NULL;*/
@@ -512,6 +513,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
mndTransDrop(pTrans);
return -1;
}
+
+ sdbRelease(pSdb, pVgroup);
}
}
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index 7901093315..b549ed1a68 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -22,6 +22,7 @@
#include "mndShow.h"
#include "mndTrans.h"
#include "mndUser.h"
+#include "tmisce.h"
#define VGROUP_VER_NUMBER 1
#define VGROUP_RESERVE_SIZE 64
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 27084700b0..7ef3207b4d 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -99,15 +99,21 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
void metaReaderReleaseLock(SMetaReader *pReader);
void metaReaderClear(SMetaReader *pReader);
int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
+int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
int metaGetTableEntryByName(SMetaReader *pReader, const char *name);
int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags);
int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, SHashObj *tags);
int32_t metaReadNext(SMetaReader *pReader);
const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
-int metaGetTableUidByName(void *meta, char *tbName, int64_t *uid);
+int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
+int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
+ bool *acquired);
+int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+ int32_t payloadLen, double selectivityRatio);
+int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
typedef struct SMetaFltParam {
tb_uid_t suid;
diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h
index 2ceae91f7c..dbfe9e1671 100644
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@@ -70,7 +70,8 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid);
int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo);
int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid);
int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo);
-void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta);
+void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta);
+int32_t metaUidFilterCacheGet(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, LRUHandle** pHandle);
struct SMeta {
TdThreadRwlock lock;
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index ac9fabf052..f229b3b127 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -142,7 +142,7 @@ typedef struct SMetaInfo {
int64_t version;
int32_t skmVer;
} SMetaInfo;
-int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo);
+int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader);
typedef struct {
int64_t uid;
diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c
index 356aa78c22..6a704d0425 100644
--- a/source/dnode/vnode/src/meta/metaCache.c
+++ b/source/dnode/vnode/src/meta/metaCache.c
@@ -31,6 +31,12 @@ typedef struct SMetaStbStatsEntry {
SMetaStbStats info;
} SMetaStbStatsEntry;
+typedef struct STagFilterResEntry {
+ uint64_t suid; // uid for super table
+ SList list; // the linked list of md5 digest, extracted from the serialized tag query condition
+ uint32_t qTimes;// queried times for current super table
+} STagFilterResEntry;
+
struct SMetaCache {
// child, normal, super, table entry cache
struct SEntryCache {
@@ -47,6 +53,11 @@ struct SMetaCache {
} sStbStatsCache;
// query cache
+ struct STagFilterResCache {
+ SHashObj* pTableEntry;
+ SLRUCache* pUidResCache;
+ uint64_t keyBuf[3];
+ } sTagFilterResCache;
};
static void entryCacheClose(SMeta* pMeta) {
@@ -79,6 +90,12 @@ static void statsCacheClose(SMeta* pMeta) {
}
}
+static void freeCacheEntryFp(void* param) {
+ STagFilterResEntry** p = param;
+ tdListEmpty(&(*p)->list);
+ taosMemoryFreeClear(*p);
+}
+
int32_t metaCacheOpen(SMeta* pMeta) {
int32_t code = 0;
SMetaCache* pCache = NULL;
@@ -109,9 +126,20 @@ int32_t metaCacheOpen(SMeta* pMeta) {
goto _err2;
}
- pMeta->pCache = pCache;
+ pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5*1024*1024, -1, 0.5);
+ if (pCache->sTagFilterResCache.pUidResCache == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err2;
+ }
-_exit:
+ pCache->sTagFilterResCache.pTableEntry = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
+ if (pCache->sTagFilterResCache.pTableEntry == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err2;
+ }
+
+ taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp);
+ pMeta->pCache = pCache;
return code;
_err2:
@@ -119,7 +147,6 @@ _err2:
_err:
taosMemoryFree(pCache);
-
metaError("vgId:%d, meta open cache failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
return code;
}
@@ -128,6 +155,9 @@ void metaCacheClose(SMeta* pMeta) {
if (pMeta->pCache) {
entryCacheClose(pMeta);
statsCacheClose(pMeta);
+
+ taosHashCleanup(pMeta->pCache->sTagFilterResCache.pTableEntry);
+ taosLRUCacheCleanup(pMeta->pCache->sTagFilterResCache.pUidResCache);
taosMemoryFree(pMeta->pCache);
pMeta->pCache = NULL;
}
@@ -388,3 +418,142 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) {
return code;
}
+
+int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes) {
+ uint64_t* pBuf = pMeta->pCache->sTagFilterResCache.keyBuf;
+
+ // generate the composed key for LRU cache
+ SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache;
+
+ pBuf[0] = suid;
+ memcpy(&pBuf[1], pKey, keyLen);
+
+ int32_t len = keyLen + sizeof(uint64_t);
+ LRUHandle *pHandle = taosLRUCacheLookup(pCache, pBuf, len);
+ if (pHandle == NULL) {
+ *acquireRes = 0;
+ return TSDB_CODE_SUCCESS;
+ } else { // do some book mark work after acquiring the filter result from cache
+ STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t));
+ ASSERT(pEntry != NULL);
+ *acquireRes = 1;
+
+ const char* p = taosLRUCacheValue(pMeta->pCache->sTagFilterResCache.pUidResCache, pHandle);
+ int32_t size = *(int32_t*) p;
+ taosArrayAddBatch(pList1, p + sizeof(int32_t), size);
+
+ (*pEntry)->qTimes += 1;
+ taosLRUCacheRelease(pCache, pHandle, false);
+
+ // check if scanning all items are necessary or not
+ if ((*pEntry)->qTimes >= 5000 && TD_DLIST_NELES(&(*pEntry)->list) > 10) {
+ SArray* pList = taosArrayInit(64, POINTER_BYTES);
+
+ SListIter iter = {0};
+ tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD);
+
+ SListNode* pNode = NULL;
+ while ((pNode = tdListNext(&iter)) != NULL) {
+ memcpy(pBuf + sizeof(suid), pNode->data, keyLen);
+
+ // check whether it is existed in LRU cache, and remove it from linked list if not.
+ LRUHandle* pRes = taosLRUCacheLookup(pCache, pBuf, len);
+ if (pRes == NULL) { // remove the item in the linked list
+ taosArrayPush(pList, &pNode);
+ } else {
+ taosLRUCacheRelease(pCache, pRes, false);
+ }
+ }
+
+ // remove the keys, of which query uid lists have been replaced already.
+ size_t s = taosArrayGetSize(pList);
+ for(int32_t i = 0; i < s; ++i) {
+ SListNode** p1 = taosArrayGet(pList, i);
+ tdListPopNode(&(*pEntry)->list, *p1);
+ }
+
+ (*pEntry)->qTimes = 0; // reset the query times
+ }
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static void freePayload(const void* key, size_t keyLen, void* value) {
+ if (value == NULL) {
+ return;
+ }
+ taosMemoryFree(value);
+}
+
+// check both the payload size and selectivity ratio
+int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio) {
+ if (selectivityRatio > tsSelectivityRatio) {
+ metaDebug("vgId:%d, suid:%" PRIu64
+ " failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f",
+ TD_VID(pMeta->pVnode), suid, selectivityRatio, tsSelectivityRatio);
+ taosMemoryFree(pPayload);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ if (payloadLen > tsTagFilterResCacheSize) {
+ metaDebug("vgId:%d, suid:%" PRIu64
+ " failed to add to uid list cache, due to payload length %d greater than threshold %d",
+ TD_VID(pMeta->pVnode), suid, payloadLen, tsTagFilterResCacheSize);
+ taosMemoryFree(pPayload);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache;
+ SHashObj* pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry;
+
+ STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t));
+ if (pEntry == NULL) {
+ STagFilterResEntry* p = taosMemoryMalloc(sizeof(STagFilterResEntry));
+ p->qTimes = 0;
+ tdListInit(&p->list, keyLen);
+ taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES);
+ tdListAppend(&p->list, pKey);
+ } else {
+ tdListAppend(&(*pEntry)->list, pKey);
+ }
+
+ uint64_t* pBuf = pMeta->pCache->sTagFilterResCache.keyBuf;
+ pBuf[0] = suid;
+
+ memcpy(&pBuf[1], pKey, keyLen);
+ ASSERT(sizeof(uint64_t) + keyLen == 24);
+
+ // add to cache.
+ taosLRUCacheInsert(pCache, pBuf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, TAOS_LRU_PRIORITY_LOW);
+ metaDebug("vgId:%d, suid:%"PRIu64" list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode),
+ suid, (int32_t) taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry));
+
+ return TSDB_CODE_SUCCESS;
+}
+
+// remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables
+int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) {
+ STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t));
+ if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ int32_t keyLen = sizeof(uint64_t) * 3;
+ uint64_t p[3] = {0};
+ p[0] = suid;
+
+ SListIter iter = {0};
+ tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD);
+
+ SListNode* pNode = NULL;
+ while ((pNode = tdListNext(&iter)) != NULL) {
+ memcpy(&p[1], pNode->data, 16);
+ taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, keyLen);
+ }
+
+ (*pEntry)->qTimes = 0;
+ tdListEmpty(&(*pEntry)->list);
+
+ return TSDB_CODE_SUCCESS;
+}
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index 6158221fa6..6d0dac88fb 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -152,7 +152,7 @@ bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) {
}
int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
- SMeta *pMeta = pReader->pMeta;
+ SMeta *pMeta = pReader->pMeta;
int64_t version1;
// query uid.idx
@@ -165,6 +165,18 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
return metaGetTableEntryByVersion(pReader, version1, uid);
}
+int metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) {
+ SMeta *pMeta = pReader->pMeta;
+
+ SMetaInfo info;
+ if (metaGetInfo(pMeta, uid, &info, pReader) == TSDB_CODE_NOT_FOUND) {
+ terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
+ return -1;
+ }
+
+ return metaGetTableEntryByVersion(pReader, info.version, uid);
+}
+
int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
SMeta *pMeta = pReader->pMeta;
tb_uid_t uid;
@@ -211,7 +223,7 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) {
return 0;
}
-int metaGetTableUidByName(void *meta, char *tbName, int64_t *uid) {
+int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) {
int code = 0;
SMetaReader mr = {0};
metaReaderInit(&mr, (SMeta *)meta, 0);
@@ -605,7 +617,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
SSkmDbKey skmDbKey;
if (sver <= 0) {
SMetaInfo info;
- if (metaGetInfo(pMeta, suid ? suid : uid, &info) == 0) {
+ if (metaGetInfo(pMeta, suid ? suid : uid, &info, NULL) == 0) {
sver = info.skmVer;
} else {
TBC *pSkmDbC = NULL;
@@ -1107,8 +1119,8 @@ int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
valid = tdbTbcGet(pCursor->pCur, (const void **)pEntryKey, &nEntryKey, (const void **)&pEntryVal, &nEntryVal);
if (valid < 0) break;
- char *pTableKey = (char *)pEntryKey;
- int32_t cmp = (*param->filterFunc)(pTableKey, pName, pCursor->type);
+ char *pTableKey = (char *)pEntryKey;
+ cmp = (*param->filterFunc)(pTableKey, pName, pCursor->type);
if (cmp == 0) {
tb_uid_t tuid = *(tb_uid_t *)pEntryVal;
taosArrayPush(pUids, &tuid);
@@ -1360,10 +1372,11 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj
int32_t metaCacheGet(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo);
-int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo) {
+int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo, SMetaReader *pReader) {
int32_t code = 0;
void *pData = NULL;
int nData = 0;
+ int lock = 0;
metaRLock(pMeta);
@@ -1388,11 +1401,22 @@ int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo) {
pInfo->version = ((SUidIdxVal *)pData)->version;
pInfo->skmVer = ((SUidIdxVal *)pData)->skmVer;
+ if (pReader != NULL) {
+ lock = !(pReader->flags & META_READER_NOLOCK);
+ if (lock) {
+ metaULock(pReader->pMeta);
+ // metaReaderReleaseLock(pReader);
+ }
+ }
// upsert the cache
metaWLock(pMeta);
metaCacheUpsert(pMeta, pInfo);
metaULock(pMeta);
+ if (lock) {
+ metaRLock(pReader->pMeta);
+ }
+
_exit:
tdbFree(pData);
return code;
diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c
index 3ada7d1814..52452bf710 100644
--- a/source/dnode/vnode/src/meta/metaSma.c
+++ b/source/dnode/vnode/src/meta/metaSma.c
@@ -36,7 +36,7 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
// validate req
// save smaIndex
metaReaderInit(&mr, pMeta, 0);
- if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) {
+ if (metaGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) {
#if 1
terrno = TSDB_CODE_TSMA_ALREADY_EXIST;
metaReaderClear(&mr);
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 841999ef9f..5c97ee5633 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -207,7 +207,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
tb_uid_t uid = *(tb_uid_t *)pData;
tdbFree(pData);
SMetaInfo info;
- metaGetInfo(pMeta, uid, &info);
+ metaGetInfo(pMeta, uid, &info, NULL);
if (info.uid == info.suid) {
return 0;
} else {
@@ -458,6 +458,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe
metaWLock(pMeta);
metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1);
+ metaUidCacheClear(pMeta, me.ctbEntry.suid);
metaULock(pMeta);
} else {
me.ntbEntry.ctime = pReq->ctime;
@@ -681,6 +682,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
--pMeta->pVnode->config.vndStats.numOfCTables;
metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1);
+ metaUidCacheClear(pMeta, e.ctbEntry.suid);
} else if (e.type == TSDB_NORMAL_TABLE) {
// drop schema.db (todo)
@@ -691,6 +693,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
// drop schema.db (todo)
metaStatsCacheDrop(pMeta, uid);
+ metaUidCacheClear(pMeta, uid);
--pMeta->pVnode->config.vndStats.numOfSTables;
}
@@ -1074,6 +1077,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags,
((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn);
+ metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid);
+
metaULock(pMeta);
tDecoderClear(&dc1);
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 75fb566438..51842a8ae4 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -739,11 +739,13 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
}
}
- tdBlockDataDestroy(pResList);
+ taosArrayDestroy(pResList);
+ qCleanExecTaskBlockBuf(taskInfo);
return TSDB_CODE_SUCCESS;
_err:
- tdBlockDataDestroy(pResList);
+ taosArrayDestroy(pResList);
+ qCleanExecTaskBlockBuf(taskInfo);
return TSDB_CODE_FAILED;
}
@@ -921,7 +923,7 @@ static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) {
SMetaReader mr = {0};
metaReaderInit(&mr, SMA_META(pSma), 0);
smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid);
- if (metaGetTableEntryByUid(&mr, pInfo->suid) < 0) {
+ if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) {
smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid,
terrstr());
goto _err;
@@ -1125,7 +1127,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
for (int64_t i = 0; i < arrSize; ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid);
- if (metaGetTableEntryByUid(&mr, suid) < 0) {
+ if (metaGetTableEntryByUidCache(&mr, suid) < 0) {
smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid,
terrstr());
goto _err;
diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c
index 3887f72740..8bcfe53712 100644
--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqExec.c
@@ -48,7 +48,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i
SMetaReader mr = {0};
metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
// TODO add reference to gurantee success
- if (metaGetTableEntryByUid(&mr, uid) < 0) {
+ if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
metaReaderClear(&mr);
return -1;
}
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index afb7ac39de..e41b1d8aa8 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -766,7 +766,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) {
uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i);
- int32_t code = metaGetTableEntryByUid(&mr, *id);
+ int32_t code = metaGetTableEntryByUidCache(&mr, *id);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
continue;
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index 291a5ab1eb..d71eb33951 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -392,7 +392,7 @@ static tb_uid_t getTableSuidByUid(tb_uid_t uid, STsdb *pTsdb) {
SMetaReader mr = {0};
metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
- if (metaGetTableEntryByUid(&mr, uid) < 0) {
+ if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
metaReaderClear(&mr); // table not esist
return 0;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
index ddf2949607..0a7f59e429 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@@ -104,7 +104,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
tb_uid_t uid = pMsgIter->uid;
SMetaInfo info;
- code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info);
+ code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info, NULL);
if (code) {
code = TSDB_CODE_TDB_TABLE_NOT_EXIST;
goto _err;
@@ -114,7 +114,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
goto _err;
}
if (info.suid) {
- metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info);
+ metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info, NULL);
}
if (pMsgIter->sversion != info.skmVer) {
tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64, TD_VID(pTsdb->pVnode),
@@ -153,7 +153,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
// check if table exists
SMetaInfo info;
- code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info);
+ code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info, NULL);
if (code) {
code = TSDB_CODE_TDB_TABLE_NOT_EXIST;
goto _err;
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 534fe89818..96cfa1752d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -38,7 +38,7 @@ typedef struct {
typedef struct SBlockIndex {
int32_t ordinalIndex;
int64_t inFileOffset;
- STimeWindow window;
+ STimeWindow window; // todo replace it with overlap flag.
} SBlockIndex;
typedef struct STableBlockScanInfo {
@@ -551,7 +551,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity)
}
for (int32_t i = 0; i < pCond->numOfCols; ++i) {
- SColumnInfoData colInfo = {0, {0}};
+ SColumnInfoData colInfo = {0};
colInfo.info = pCond->colList[i];
blockDataAppendColInfo(pResBlock, &colInfo);
}
@@ -709,11 +709,13 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
tMapDataReset(&pScanInfo->mapData);
tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData);
+ taosArrayEnsureCap(pScanInfo->pBlockList, pScanInfo->mapData.nItem);
sizeInDisk += pScanInfo->mapData.nData;
+
+ SDataBlk block = {0};
for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) {
- SDataBlk block = {0};
- tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetDataBlk);
+ tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block);
// 1. time range check
if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) {
@@ -728,8 +730,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
SBlockIndex bIndex = {.ordinalIndex = j, .inFileOffset = block.aSubBlock->offset};
bIndex.window = (STimeWindow){.skey = block.minKey.ts, .ekey = block.maxKey.ts};
- void* p = taosArrayPush(pScanInfo->pBlockList, &bIndex);
- if (p == NULL) {
+ void* p1 = taosArrayPush(pScanInfo->pBlockList, &bIndex);
+ if (p1 == NULL) {
tMapDataClear(&pScanInfo->mapData);
return TSDB_CODE_OUT_OF_MEMORY;
}
@@ -752,6 +754,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el,
pReader->idStr);
+
pReader->cost.numOfBlocks += total;
pReader->cost.headFileLoadTime += el;
@@ -1455,6 +1458,7 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t
return TSDB_CODE_SUCCESS;
}
+// todo: this attribute could be acquired during extractin the global ordered block list.
static bool overlapWithNeighborBlock(SDataBlk* pBlock, SBlockIndex* pNeighborBlockIndex, int32_t order) {
// it is the last block in current file, no chance to overlap with neighbor blocks.
if (ASCENDING_TRAVERSE(order)) {
@@ -4346,7 +4350,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
SMetaReader mr = {0};
metaReaderInit(&mr, pVnode->pMeta, 0);
- int32_t code = metaGetTableEntryByUid(&mr, uid);
+ int32_t code = metaGetTableEntryByUidCache(&mr, uid);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
metaReaderClear(&mr);
@@ -4358,7 +4362,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
if (mr.me.type == TSDB_CHILD_TABLE) {
tDecoderClear(&mr.coder);
*suid = mr.me.ctbEntry.suid;
- code = metaGetTableEntryByUid(&mr, *suid);
+ code = metaGetTableEntryByUidCache(&mr, *suid);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
metaReaderClear(&mr);
diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h
index fd8a357a8b..875528576d 100644
--- a/source/libs/executor/inc/executil.h
+++ b/source/libs/executor/inc/executil.h
@@ -38,16 +38,7 @@
memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \
} while (0)
-#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \
- do { \
- assert(sizeof(_uid) == sizeof(uint64_t)); \
- *(void**)(_k) = (_buf); \
- *(uint64_t*)((_k) + POINTER_BYTES) = (_uid); \
- memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \
- } while (0)
-
#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
-#define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES)
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
@@ -104,16 +95,17 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, SExecTaskInfo* pTaskInfo);
STableListInfo* tableListCreate();
-void* tableListDestroy(STableListInfo* pTableListInfo);
-void tableListClear(STableListInfo* pTableListInfo);
-int32_t tableListGetOutputGroups(const STableListInfo* pTableList);
-bool oneTableForEachGroup(const STableListInfo* pTableList);
-uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid);
-int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t gid);
-int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalIndex, STableKeyInfo** pKeyInfo, int32_t* num);
-uint64_t tableListGetSize(const STableListInfo* pTableList);
-uint64_t tableListGetSuid(const STableListInfo* pTableList);
-STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index);
+void* tableListDestroy(STableListInfo* pTableListInfo);
+void tableListClear(STableListInfo* pTableListInfo);
+int32_t tableListGetOutputGroups(const STableListInfo* pTableList);
+bool oneTableForEachGroup(const STableListInfo* pTableList);
+uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid);
+int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t gid);
+int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalIndex, STableKeyInfo** pKeyInfo,
+ int32_t* num);
+uint64_t tableListGetSize(const STableListInfo* pTableList);
+uint64_t tableListGetSuid(const STableListInfo* pTableList);
+STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index);
size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
void initResultRowInfo(SResultRowInfo* pResultRowInfo);
@@ -140,7 +132,7 @@ bool hasRemainResults(SGroupResInfo* pGroupResInfo);
int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
-SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode);
+SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode);
EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId);
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index b442034860..8163217039 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -153,12 +153,12 @@ typedef struct {
SSchemaWrapper* qsw;
} SSchemaInfo;
-typedef struct {
+typedef struct SExchangeOpStopInfo {
int32_t operatorType;
int64_t refId;
} SExchangeOpStopInfo;
-typedef struct {
+typedef struct STaskStopInfo {
SRWLatch lock;
SArray* pStopInfo;
} STaskStopInfo;
@@ -181,6 +181,7 @@ struct SExecTaskInfo {
SSubplan* pSubplan;
struct SOperatorInfo* pRoot;
SLocalFetch localFetch;
+ SArray* pResultBlockList;// result block list
STaskStopInfo stopInfo;
};
@@ -252,21 +253,22 @@ typedef struct SLimitInfo {
} SLimitInfo;
typedef struct SExchangeInfo {
- SArray* pSources;
- SArray* pSourceDataInfo;
- tsem_t ready;
- void* pTransporter;
+ SArray* pSources;
+ SArray* pSourceDataInfo;
+ tsem_t ready;
+ void* pTransporter;
+
// SArray, result block list, used to keep the multi-block that
// passed by downstream operator
SArray* pResultBlockList;
- int32_t rspBlockIndex; // indicate the return block index in pResultBlockList
+ SArray* pRecycledBlocks;// build a pool for small data block to avoid to repeatly create and then destroy.
SSDataBlock* pDummyBlock; // dummy block, not keep data
bool seqLoadData; // sequential load data or not, false by default
int32_t current;
SLoadRemoteDataInfo loadInfo;
uint64_t self;
SLimitInfo limitInfo;
- int64_t openedTs; // start exec time stamp
+ int64_t openedTs; // start exec time stamp, todo: move to SLoadRemoteDataInfo
} SExchangeInfo;
typedef struct SScanInfo {
@@ -301,9 +303,9 @@ typedef struct {
} SAggOptrPushDownInfo;
typedef struct STableMetaCacheInfo {
- SLRUCache* pTableMetaEntryCache; // 100 by default
- uint64_t metaFetch;
- uint64_t cacheHit;
+ SLRUCache* pTableMetaEntryCache; // 100 by default
+ uint64_t metaFetch;
+ uint64_t cacheHit;
} STableMetaCacheInfo;
typedef struct STableScanBase {
@@ -535,15 +537,6 @@ typedef struct SStreamIntervalOperatorInfo {
SWinKey delKey;
} SStreamIntervalOperatorInfo;
-typedef struct SAggOperatorInfo {
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
- STableQueryInfo* current;
- uint64_t groupId;
- SGroupResInfo groupResInfo;
- SExprSupp scalarExprSup;
-} SAggOperatorInfo;
-
typedef struct SFillOperatorInfo {
struct SFillInfo* pFillInfo;
SSDataBlock* pRes;
@@ -575,18 +568,6 @@ typedef struct SWindowRowsSup {
uint64_t groupId;
} SWindowRowsSup;
-typedef struct SSessionAggOperatorInfo {
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
-
- SGroupResInfo groupResInfo;
- SWindowRowsSup winSup;
- bool reptScan; // next round scan
- int64_t gap; // session window gap
- int32_t tsSlotId; // primary timestamp slot id
- STimeWindowAggSupp twAggSup;
-} SSessionAggOperatorInfo;
-
typedef struct SResultWindowInfo {
void* pOutputBuf;
SSessionKey sessionWin;
@@ -679,53 +660,30 @@ typedef struct SStreamFillOperatorInfo {
SStreamFillInfo* pFillInfo;
} SStreamFillOperatorInfo;
-typedef struct STimeSliceOperatorInfo {
- SSDataBlock* pRes;
- STimeWindow win;
- SInterval interval;
- int64_t current;
- SArray* pPrevRow; // SArray
- SArray* pNextRow; // SArray
- SArray* pLinearInfo; // SArray
- bool isPrevRowSet;
- bool isNextRowSet;
- int32_t fillType; // fill type
- SColumn tsCol; // primary timestamp column
- SExprSupp scalarSup; // scalar calculation
- struct SFillColInfo* pFillColInfo; // fill column info
-} STimeSliceOperatorInfo;
-
-typedef struct SStateWindowOperatorInfo {
- // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
- SExprSupp scalarSup;
-
- SGroupResInfo groupResInfo;
- SWindowRowsSup winSup;
- SColumn stateCol; // start row index
- bool hasKey;
- SStateKeys stateKey;
- int32_t tsSlotId; // primary timestamp column slot id
- STimeWindowAggSupp twAggSup;
-} SStateWindowOperatorInfo;
-
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED)
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
__optr_close_fn_t closeFn, __optr_explain_fn_t explain);
-
-int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);
-int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num);
+int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);
+int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num);
+void setOperatorCompleted(SOperatorInfo* pOperator);
+void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status,
+ void* pInfo, SExecTaskInfo* pTaskInfo);
+void destroyOperatorInfo(SOperatorInfo* pOperator);
void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
void cleanupBasicInfo(SOptrBasicInfo* pInfo);
+
int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr);
void cleanupExprSupp(SExprSupp* pSup);
+
void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs);
-int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
- const char* pkey);
+
+int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
+ const char* pkey);
+void cleanupAggSup(SAggSupporter* pAggSup);
+
void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
@@ -733,12 +691,12 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
SDiskbasedBuf* pBuf);
-bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
-void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
+bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
+void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator);
-void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset,
- int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput);
+void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData,
+ int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput);
int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart);
void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs,
@@ -749,130 +707,108 @@ STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInter
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag);
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
-void doDestroyExchangeOperatorInfo(void* param);
+extern void doDestroyExchangeOperatorInfo(void* param);
-void setOperatorCompleted(SOperatorInfo* pOperator);
-void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status,
- void* pInfo, SExecTaskInfo* pTaskInfo);
void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo);
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
int32_t rows, const char* idStr, STableMetaCacheInfo* pCache);
-void cleanupAggSup(SAggSupporter* pAggSup);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name);
-SSDataBlock* loadNextDataBlock(void* param);
-
void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset);
SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData,
int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo,
bool isIntervalQuery, SAggSupporter* pSup);
-
+// operator creater functions
+// clang-format off
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
- STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode,
- const char* pUser, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode,
- SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo);
+
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams,
- SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode,
- SExecTaskInfo* pTaskInfo, bool isStream);
-SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
- SExecTaskInfo* pTaskInfo, int32_t numOfChild);
-SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
- SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream);
+
+SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild);
+
+SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode, SExecTaskInfo* pTaskInfo);
+
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
- SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode,
- SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode,
- SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
- SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
- SExecTaskInfo* pTaskInfo, int32_t numOfChild);
-SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
- SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode,
- SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild);
+
+SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo);
+// clang-format on
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList);
void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol);
-bool isTaskKilled(SExecTaskInfo* pTaskInfo);
int32_t checkForQueryBuf(size_t numOfTables);
+bool isTaskKilled(SExecTaskInfo* pTaskInfo);
void setTaskKilled(SExecTaskInfo* pTaskInfo);
-void queryCostStatis(SExecTaskInfo* pTaskInfo);
void doDestroyTask(SExecTaskInfo* pTaskInfo);
-void destroyOperatorInfo(SOperatorInfo* pOperator);
-int32_t getMaximumIdleDurationSec();
-
-/*
- * ops: root operator
- * data: *data save the result of encode, need to be freed by caller
- * length: *length save the length of *data
- * nOptrWithVal: *nOptrWithVal save the number of optr with value
- * return: result code, 0 means success
- */
-int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t* length, int32_t* nOptrWithVal);
-
-/*
- * ops: root operator, created by caller
- * data: save the result of decode
- * length: the length of data
- * return: result code, 0 means success
- */
-int32_t decodeOperator(SOperatorInfo* ops, const char* data, int32_t length);
-
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
+
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
char* sql, EOPTR_EXEC_MODEL model);
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
+void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo);
+
+int32_t getMaximumIdleDurationSec();
+
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
int32_t order);
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
@@ -895,15 +831,7 @@ void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock);
int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup,
SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
- SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
- SExecTaskInfo* pTaskInfo);
-
-void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
-
bool groupbyTbname(SNodeList* pGroupList);
-void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo);
int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size);
diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c
index 6b5f773fe3..cdd744bded 100644
--- a/source/libs/executor/src/cachescanoperator.c
+++ b/source/libs/executor/src/cachescanoperator.c
@@ -44,6 +44,8 @@ static void destroyCacheScanOperator(void* param);
static int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds);
static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo);
+#define SCAN_ROW_TYPE(_t) ((_t)? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)
+
SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
@@ -57,7 +59,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
pInfo->readHandle = *readHandle;
SDataBlockDescNode* pDescNode = pScanNode->scan.node.pOutputDataBlockDesc;
- pInfo->pRes = createResDataBlock(pDescNode);
+ pInfo->pRes = createDataBlockFromDescNode(pDescNode);
int32_t numOfCols = 0;
code =
@@ -75,32 +77,36 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
STableListInfo* pTableList = pTaskInfo->pTableInfoList;
- initResultSizeInfo(&pOperator->resultInfo, 4096);
- blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+ int32_t totalTables = tableListGetSize(pTableList);
+ int32_t capacity = 0;
+
pInfo->pUidList = taosArrayInit(4, sizeof(int64_t));
- // partition by tbname, todo opt perf
- if (oneTableForEachGroup(pTableList) || (tableListGetSize(pTableList) == 1)) {
- pInfo->retrieveType =
- CACHESCAN_RETRIEVE_TYPE_ALL | (pScanNode->ignoreNull ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW);
+ // partition by tbname
+ if (oneTableForEachGroup(pTableList) || (totalTables == 1)) {
+ pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL | SCAN_ROW_TYPE(pScanNode->ignoreNull);
STableKeyInfo* pList = tableListGetInfo(pTableList, 0);
- size_t num = tableListGetSize(pTableList);
uint64_t suid = tableListGetSuid(pTableList);
- code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
+ code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
+ capacity = TMIN(totalTables, 4096);
+
pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false);
- blockDataEnsureCapacity(pInfo->pBufferredRes, pOperator->resultInfo.capacity);
+ blockDataEnsureCapacity(pInfo->pBufferredRes, capacity);
} else { // by tags
- pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE |
- (pScanNode->ignoreNull ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW);
+ pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull);
+ capacity = 1; // only one row output
}
+ initResultSizeInfo(&pOperator->resultInfo, capacity);
+ blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+
if (pScanNode->scan.pScanPseudoCols != NULL) {
SExprSupp* p = &pInfo->pseudoExprSup;
p->pExprInfo = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &p->numOfExprs);
diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index b01a4b7871..280880c077 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -181,10 +181,17 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
return NULL;
}
- size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList);
- if (size == 0 || pExchangeInfo->rspBlockIndex >= size) {
- pExchangeInfo->rspBlockIndex = 0;
- taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock);
+ // we have buffered retrieved datablock, return it directly
+ SSDataBlock* p = NULL;
+ if (taosArrayGetSize(pExchangeInfo->pResultBlockList) > 0) {
+ p = taosArrayGetP(pExchangeInfo->pResultBlockList, 0);
+ taosArrayRemove(pExchangeInfo->pResultBlockList, 0);
+ }
+
+ if (p != NULL) {
+ taosArrayPush(pExchangeInfo->pRecycledBlocks, &p);
+ return p;
+ } else {
if (pExchangeInfo->seqLoadData) {
seqLoadRemoteData(pOperator);
} else {
@@ -193,11 +200,13 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) {
return NULL;
+ } else {
+ p = taosArrayGetP(pExchangeInfo->pResultBlockList, 0);
+ taosArrayRemove(pExchangeInfo->pResultBlockList, 0);
+ taosArrayPush(pExchangeInfo->pRecycledBlocks, &p);
+ return p;
}
}
-
- // we have buffered retrieved datablock, return it directly
- return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++);
}
static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) {
@@ -294,8 +303,9 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
}
tsem_init(&pInfo->ready, 0, 0);
- pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc);
- pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES);
+ pInfo->pDummyBlock = createDataBlockFromDescNode(pExNode->node.pOutputDataBlockDesc);
+ pInfo->pResultBlockList = taosArrayInit(64, POINTER_BYTES);
+ pInfo->pRecycledBlocks = taosArrayInit(64, POINTER_BYTES);
SExchangeOpStopInfo stopInfo = {QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, pInfo->self};
qAppendTaskStopInfo(pTaskInfo, &stopInfo);
@@ -342,10 +352,8 @@ void doDestroyExchangeOperatorInfo(void* param) {
taosArrayDestroy(pExInfo->pSources);
taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo);
- if (pExInfo->pResultBlockList != NULL) {
- taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock);
- pExInfo->pResultBlockList = NULL;
- }
+ taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock);
+ taosArrayDestroyEx(pExInfo->pRecycledBlocks, freeBlock);
blockDataDestroy(pExInfo->pDummyBlock);
@@ -638,6 +646,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
SRetrieveTableRsp* pRsp = pDataInfo->pRsp;
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
+
if (pRsp->numOfRows == 0) {
qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 " try next",
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index d1046ff02c..08e6e4792b 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -208,7 +208,7 @@ SArray* createSortInfo(SNodeList* pNodeList) {
return pList;
}
-SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
+SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode) {
int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
SSDataBlock* pBlock = createDataBlock();
@@ -290,7 +290,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle,
SMetaReader mr = {0};
metaReaderInit(&mr, metaHandle, 0);
- code = metaGetTableEntryByUid(&mr, info->uid);
+ code = metaGetTableEntryByUidCache(&mr, info->uid);
if (TSDB_CODE_SUCCESS != code) {
metaReaderClear(&mr);
*pQualified = false;
@@ -405,7 +405,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
- ctx.index = 0;
+
ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo));
if (ctx.cInfoList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -421,7 +421,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
}
for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) {
- SColumnInfoData colInfo = {0, {0}};
+ SColumnInfoData colInfo = {0};
colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i);
blockDataAppendColInfo(pResBlock, &colInfo);
}
@@ -582,7 +582,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
}
for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) {
- SColumnInfoData colInfo = {0, {0}};
+ SColumnInfoData colInfo = {0};
colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i);
blockDataAppendColInfo(pResBlock, &colInfo);
}
@@ -964,67 +964,132 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
return -1;
}
+static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) {
+ if (pTagCond == NULL) {
+ return;
+ }
+
+ char* payload = NULL;
+ int32_t len = 0;
+ nodesNodeToMsg(pTagCond, &payload, &len);
+
+ tMD5Init(pContext);
+ tMD5Update(pContext, (uint8_t*)payload, (uint32_t)len);
+ tMD5Final(pContext);
+
+ taosMemoryFree(payload);
+}
+
+static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* res, SNode* pTagCond, void* metaHandle) {
+ if (pTagCond == NULL) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ terrno = TDB_CODE_SUCCESS;
+ SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond);
+ if (terrno != TDB_CODE_SUCCESS) {
+ colDataDestroy(pColInfoData);
+ taosMemoryFreeClear(pColInfoData);
+ taosArrayDestroy(res);
+ qError("failed to getColInfoResult, code: %s", tstrerror(terrno));
+ return terrno;
+ }
+
+ int32_t i = 0;
+ int32_t len = taosArrayGetSize(res);
+
+ if (pColInfoData != NULL) {
+ bool* pResult = (bool*)pColInfoData->pData;
+ SArray* p = taosArrayInit(taosArrayGetSize(res), sizeof(uint64_t));
+
+ while (i < len && pColInfoData) {
+ int64_t* uid = taosArrayGet(res, i);
+ qDebug("tagfilter get uid:%" PRId64 ", res:%d", *uid, pResult[i]);
+
+ if (pResult[i]) {
+ taosArrayPush(p, uid);
+ }
+ i += 1;
+ }
+
+ taosArraySwap(res, p);
+ taosArrayDestroy(p);
+ }
+
+ colDataDestroy(pColInfoData);
+ taosMemoryFreeClear(pColInfoData);
+
+ return TSDB_CODE_SUCCESS;
+}
+
int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
STableListInfo* pListInfo) {
int32_t code = TSDB_CODE_SUCCESS;
+ size_t numOfTables = 0;
uint64_t tableUid = pScanNode->uid;
pListInfo->suid = pScanNode->suid;
SArray* res = taosArrayInit(8, sizeof(uint64_t));
- if (pScanNode->tableType == TSDB_SUPER_TABLE) {
- if (pTagIndexCond) {
- SIndexMetaArg metaArg = {
- .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
-
- // int64_t stt = taosGetTimestampUs();
- SIdxFltStatus status = SFLT_NOT_INDEX;
- code = doFilterTag(pTagIndexCond, &metaArg, res, &status);
- if (code != 0 || status == SFLT_NOT_INDEX) {
- qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid);
- code = TDB_CODE_SUCCESS;
- }
- } else if (!pTagCond) {
- vnodeGetCtbIdList(pVnode, pScanNode->suid, res);
- }
- } else { // Create one table group.
+ if (pScanNode->tableType != TSDB_SUPER_TABLE) {
if (metaIsTableExist(metaHandle, tableUid)) {
taosArrayPush(res, &tableUid);
}
- }
- if (pTagCond) {
- terrno = TDB_CODE_SUCCESS;
- SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond);
- if (terrno != TDB_CODE_SUCCESS) {
- colDataDestroy(pColInfoData);
- taosMemoryFreeClear(pColInfoData);
- taosArrayDestroy(res);
- qError("failed to getColInfoResult, code: %s", tstrerror(terrno));
- return terrno;
+ code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+ } else {
+ // try to retrieve the result from meta cache
+ T_MD5_CTX context = {0};
+ genTagFilterDigest(pTagCond, &context);
+
+ bool acquired = false;
+ metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), res, &acquired);
+ if (acquired) {
+ qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(res));
+ goto _end;
}
- int32_t i = 0;
- int32_t j = 0;
- int32_t len = taosArrayGetSize(res);
- while (i < taosArrayGetSize(res) && j < len && pColInfoData) {
- void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
+ if (!pTagCond) { // no tag condition exists, let's fetch all tables of this super table
+ ASSERT(pTagIndexCond == NULL);
+ vnodeGetCtbIdList(pVnode, pScanNode->suid, res);
+ } else {
+ // failed to find the result in the cache, let try to calculate the results
+ if (pTagIndexCond) {
+ SIndexMetaArg metaArg = {
+ .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
- int64_t* uid = taosArrayGet(res, i);
- qDebug("tagfilter get uid:%" PRId64 ", res:%d", *uid, *(bool*)var);
- if (*(bool*)var == false) {
- taosArrayRemove(res, i);
- j++;
- continue;
+ SIdxFltStatus status = SFLT_NOT_INDEX;
+ code = doFilterTag(pTagIndexCond, &metaArg, res, &status);
+ if (code != 0 || status == SFLT_NOT_INDEX) {
+ qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid);
+ code = TDB_CODE_SUCCESS;
+ }
}
- i++;
- j++;
}
- colDataDestroy(pColInfoData);
- taosMemoryFreeClear(pColInfoData);
+
+ code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+
+ // let's add the filter results into meta-cache
+ numOfTables = taosArrayGetSize(res);
+ size_t size = numOfTables * sizeof(uint64_t) + sizeof(int32_t);
+ char* pPayload = taosMemoryMalloc(size);
+ *(int32_t*)pPayload = numOfTables;
+
+ if (numOfTables > 0) {
+ memcpy(pPayload + sizeof(int32_t), taosArrayGet(res, 0), numOfTables * sizeof(uint64_t));
+ }
+
+ metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1);
}
- size_t numOfTables = taosArrayGetSize(res);
+_end:
+ numOfTables = taosArrayGetSize(res);
for (int i = 0; i < numOfTables; i++) {
STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0};
@@ -1034,7 +1099,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
return TSDB_CODE_OUT_OF_MEMORY;
}
- qDebug("tagfilter get uid:%" PRIu64 "", info.uid);
+ qTrace("tagfilter get uid:%" PRIu64 "", info.uid);
}
taosArrayDestroy(res);
@@ -1057,7 +1122,7 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) {
int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) {
SMetaReader mr = {0};
metaReaderInit(&mr, pMeta, 0);
- if (metaGetTableEntryByUid(&mr, uid) != 0) { // table not exist
+ if (metaGetTableEntryByUidCache(&mr, uid) != 0) { // table not exist
metaReaderClear(&mr);
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
}
@@ -1770,7 +1835,7 @@ STableListInfo* tableListCreate() {
goto _error;
}
- pListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
+ pListInfo->map = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (pListInfo->map == NULL) {
goto _error;
}
@@ -1914,13 +1979,15 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
return code;
}
+ int32_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList);
ASSERT(pTableListInfo->numOfOuputGroups == 1);
int64_t st1 = taosGetTimestampUs();
pTaskInfo->cost.extractListTime = (st1 - st) / 1000.0;
- qDebug("extract queried table list completed, elapsed time:%.2f ms %s", pTaskInfo->cost.extractListTime, idStr);
+ qDebug("extract queried table list completed, %d tables, elapsed time:%.2f ms %s", numOfTables,
+ pTaskInfo->cost.extractListTime, idStr);
- if (taosArrayGetSize(pTableListInfo->pTableList) == 0) {
+ if (numOfTables == 0) {
qDebug("no table qualified for query, %s" PRIx64, idStr);
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index c1974ff30e..34bd9cf8ca 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -493,7 +493,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
memcpy(&pTaskInfo->localFetch, pLocal, sizeof(*pLocal));
}
- taosArrayClearEx(pResList, freeBlock);
+ taosArrayClear(pResList);
int64_t curOwner = 0;
if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) {
@@ -531,8 +531,20 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
int64_t st = taosGetTimestampUs();
+ int32_t blockIndex = 0;
while ((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) {
- SSDataBlock* p = createOneDataBlock(pRes, true);
+ SSDataBlock* p = NULL;
+ if (blockIndex >= taosArrayGetSize(pTaskInfo->pResultBlockList)) {
+ SSDataBlock* p1 = createOneDataBlock(pRes, true);
+ taosArrayPush(pTaskInfo->pResultBlockList, &p1);
+ p = p1;
+ } else {
+ p = *(SSDataBlock**) taosArrayGet(pTaskInfo->pResultBlockList, blockIndex);
+ copyDataBlock(p, pRes);
+ }
+
+ blockIndex += 1;
+
current += p->info.rows;
ASSERT(p->info.rows > 0);
taosArrayPush(pResList, &p);
@@ -560,6 +572,18 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
return pTaskInfo->code;
}
+void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo) {
+ SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+ SArray* pList = pTaskInfo->pResultBlockList;
+ size_t num = taosArrayGetSize(pList);
+ for(int32_t i = 0; i < num; ++i) {
+ SSDataBlock** p = taosArrayGet(pTaskInfo->pResultBlockList, i);
+ blockDataDestroy(*p);
+ }
+
+ taosArrayClear(pTaskInfo->pResultBlockList);
+}
+
int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
int64_t threadId = taosGetSelfPthreadId();
@@ -688,7 +712,7 @@ void qDestroyTask(qTaskInfo_t qTaskHandle) {
qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows);
- queryCostStatis(pTaskInfo); // print the query cost summary
+ printTaskExecCostInLog(pTaskInfo); // print the query cost summary
doDestroyTask(pTaskInfo);
}
@@ -704,12 +728,12 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
}
int32_t nOptrWithVal = 0;
- int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
- if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) {
- taosMemoryFreeClear(*pOutput);
- *len = 0;
- }
- return code;
+// int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
+// if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) {
+// taosMemoryFreeClear(*pOutput);
+// *len = 0;
+// }
+ return 0;
}
int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) {
@@ -719,7 +743,8 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
return TSDB_CODE_INVALID_PARA;
}
- return decodeOperator(pTaskInfo->pRoot, pInput, len);
+ return 0;
+// return decodeOperator(pTaskInfo->pRoot, pInput, len);
}
int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 036e14a621..5abde1be85 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -76,6 +76,15 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) {
#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st)))
#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0)
+typedef struct SAggOperatorInfo {
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
+ STableQueryInfo* current;
+ uint64_t groupId;
+ SGroupResInfo groupResInfo;
+ SExprSupp scalarExprSup;
+} SAggOperatorInfo;
+
int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; }
static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock);
@@ -246,7 +255,7 @@ static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pRes
// in the first scan, new space needed for results
int32_t pageId = -1;
- SIDList list = getDataBufPagesIdList(pResultBuf);
+ SArray* list = getDataBufPagesIdList(pResultBuf);
if (taosArrayGetSize(list) == 0) {
pData = getNewBufPage(pResultBuf, &pageId);
@@ -305,19 +314,19 @@ typedef struct {
} SFunctionCtxStatus;
static void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
- pStatus->hasAgg = pCtx->input.colDataAggIsSet;
+ pStatus->hasAgg = pCtx->input.colDataSMAIsSet;
pStatus->numOfRows = pCtx->input.numOfRows;
pStatus->startOffset = pCtx->input.startRowIndex;
}
static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
- pCtx->input.colDataAggIsSet = pStatus->hasAgg;
+ pCtx->input.colDataSMAIsSet = pStatus->hasAgg;
pCtx->input.numOfRows = pStatus->numOfRows;
pCtx->input.startRowIndex = pStatus->startOffset;
}
-void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset,
- int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) {
+void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData,
+ int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) {
for (int32_t k = 0; k < numOfOutput; ++k) {
// keep it temporarily
SFunctionCtxStatus status = {0};
@@ -328,8 +337,8 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfo
// not a whole block involved in query processing, statistics data can not be used
// NOTE: the original value of isSet have been changed here
- if (pCtx[k].input.colDataAggIsSet && forwardStep < numOfTotal) {
- pCtx[k].input.colDataAggIsSet = false;
+ if (pCtx[k].input.colDataSMAIsSet && forwardStep < numOfTotal) {
+ pCtx[k].input.colDataSMAIsSet = false;
}
if (fmIsWindowPseudoColumnFunc(pCtx[k].functionId)) {
@@ -439,7 +448,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
SInputColumnInfoData* pInput = &pCtx[i].input;
pInput->uid = pBlock->info.uid;
- pInput->colDataAggIsSet = false;
+ pInput->colDataSMAIsSet = false;
SExprInfo* pOneExpr = &pExprSup->pExprInfo[i];
for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) {
@@ -755,7 +764,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
pInput->totalRows = numOfRows;
if (pBlock->pBlockAgg != NULL) {
- pInput->colDataAggIsSet = true;
+ pInput->colDataSMAIsSet = true;
for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) {
SFunctParam* pFuncParam = &pExprInfo->base.pParam[j];
@@ -764,7 +773,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
int32_t slotId = pFuncParam->pCol->slotId;
pInput->pColumnDataAgg[j] = pBlock->pBlockAgg[slotId];
if (pInput->pColumnDataAgg[j] == NULL) {
- pInput->colDataAggIsSet = false;
+ pInput->colDataSMAIsSet = false;
}
// Here we set the column info data since the data type for each column data is required, but
@@ -775,7 +784,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
}
}
} else {
- pInput->colDataAggIsSet = false;
+ pInput->colDataSMAIsSet = false;
}
}
@@ -1326,7 +1335,7 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
}
}
-void queryCostStatis(SExecTaskInfo* pTaskInfo) {
+void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) {
STaskCostInfo* pSummary = &pTaskInfo->cost;
SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder;
@@ -1949,7 +1958,7 @@ void cleanupAggSup(SAggSupporter* pAggSup) {
destroyDiskbasedBuf(pAggSup->pResultBuf);
}
-int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
+int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey) {
int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
if (code != TSDB_CODE_SUCCESS) {
@@ -2039,7 +2048,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
goto _error;
}
- SSDataBlock* pResBlock = createResDataBlock(pAggNode->node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
@@ -2047,7 +2056,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
- int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
+ int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -2213,7 +2222,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
goto _error;
}
- pInfo->pRes = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
+ pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr);
pOperator->exprSupp.pExprInfo = pExprInfo;
@@ -2300,6 +2309,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT
pTaskInfo->execModel = model;
pTaskInfo->pTableInfoList = tableListCreate();
pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo));
+ pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES);
char* p = taosMemoryCalloc(1, 128);
snprintf(p, 128, "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, queryId);
@@ -2313,7 +2323,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode);
int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) {
SMetaReader mr = {0};
metaReaderInit(&mr, pHandle->meta, 0);
- int32_t code = metaGetTableEntryByUid(&mr, pScanNode->uid);
+ int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid,
GET_TASKID(pTaskInfo));
@@ -2332,7 +2342,7 @@ int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode,
tDecoderClear(&mr.coder);
tb_uid_t suid = mr.me.ctbEntry.suid;
- metaGetTableEntryByUid(&mr, suid);
+ metaGetTableEntryByUidCache(&mr, suid);
pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow);
pSchemaInfo->tversion = mr.me.stbEntry.schemaTag.version;
} else {
@@ -2511,7 +2521,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return NULL;
}
- pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo);
+ pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
@@ -2724,103 +2734,6 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa
}
#endif
-int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) {
- int32_t code = TDB_CODE_SUCCESS;
- char* pCurrent = NULL;
- int32_t currLength = 0;
- if (ops->fpSet.encodeResultRow) {
- if (result == NULL || length == NULL || nOptrWithVal == NULL) {
- return TSDB_CODE_TSC_INVALID_INPUT;
- }
- code = ops->fpSet.encodeResultRow(ops, &pCurrent, &currLength);
-
- if (code != TDB_CODE_SUCCESS) {
- if (*result != NULL) {
- taosMemoryFree(*result);
- *result = NULL;
- }
- return code;
- } else if (currLength == 0) {
- ASSERT(!pCurrent);
- goto _downstream;
- }
-
- ++(*nOptrWithVal);
-
- ASSERT(currLength >= 0);
-
- if (*result == NULL) {
- *result = (char*)taosMemoryCalloc(1, currLength + sizeof(int32_t));
- if (*result == NULL) {
- taosMemoryFree(pCurrent);
- return TSDB_CODE_OUT_OF_MEMORY;
- }
- memcpy(*result + sizeof(int32_t), pCurrent, currLength);
- *(int32_t*)(*result) = currLength + sizeof(int32_t);
- } else {
- int32_t sizePre = *(int32_t*)(*result);
- char* tmp = (char*)taosMemoryRealloc(*result, sizePre + currLength);
- if (tmp == NULL) {
- taosMemoryFree(pCurrent);
- taosMemoryFree(*result);
- *result = NULL;
- return TSDB_CODE_OUT_OF_MEMORY;
- }
- *result = tmp;
- memcpy(*result + sizePre, pCurrent, currLength);
- *(int32_t*)(*result) += currLength;
- }
- taosMemoryFree(pCurrent);
- *length = *(int32_t*)(*result);
- }
-
-_downstream:
- for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
- code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal);
- if (code != TDB_CODE_SUCCESS) {
- return code;
- }
- }
- return TDB_CODE_SUCCESS;
-}
-
-int32_t decodeOperator(SOperatorInfo* ops, const char* result, int32_t length) {
- int32_t code = TDB_CODE_SUCCESS;
- if (ops->fpSet.decodeResultRow) {
- if (result == NULL) {
- return TSDB_CODE_TSC_INVALID_INPUT;
- }
-
- ASSERT(length == *(int32_t*)result);
-
- const char* data = result + sizeof(int32_t);
- code = ops->fpSet.decodeResultRow(ops, (char*)data);
- if (code != TDB_CODE_SUCCESS) {
- return code;
- }
-
- int32_t totalLength = *(int32_t*)result;
- int32_t dataLength = *(int32_t*)data;
-
- if (totalLength == dataLength + sizeof(int32_t)) { // the last data
- result = NULL;
- length = 0;
- } else {
- result += dataLength;
- *(int32_t*)(result) = totalLength - dataLength;
- length = totalLength - dataLength;
- }
- }
-
- for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
- code = decodeOperator(ops->pDownstream[i], result, length);
- if (code != TDB_CODE_SUCCESS) {
- return code;
- }
- }
- return TDB_CODE_SUCCESS;
-}
-
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle) {
SExecTaskInfo* pTask = *(SExecTaskInfo**)pTaskInfo;
@@ -2901,6 +2814,11 @@ _complete:
return terrno;
}
+static void freeBlock(void* pParam) {
+ SSDataBlock* pBlock = *(SSDataBlock**)pParam;
+ blockDataDestroy(pBlock);
+}
+
void doDestroyTask(SExecTaskInfo* pTaskInfo) {
qDebug("%s execTask is freed", GET_TASKID(pTaskInfo));
@@ -2913,6 +2831,7 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) {
nodesDestroyNode((SNode*)pTaskInfo->pSubplan);
}
+ taosArrayDestroyEx(pTaskInfo->pResultBlockList, freeBlock);
taosArrayDestroy(pTaskInfo->stopInfo.pStopInfo);
taosMemoryFreeClear(pTaskInfo->sql);
taosMemoryFreeClear(pTaskInfo->id.str);
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index 82398d6e34..6dc8818900 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -274,10 +274,9 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
// return;
// }
- int32_t len = 0;
- STimeWindow w = TSWINDOW_INITIALIZER;
-
+ int32_t len = 0;
terrno = TSDB_CODE_SUCCESS;
+
int32_t num = 0;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
// Compare with the previous row of this column, and do not set the output buffer again if they are identical.
@@ -315,7 +314,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
}
int32_t rowIndex = j - num;
- doApplyFunctions(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs);
+ applyAggFunctionOnPartialTuples(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs);
// assign the group keys or user input constant values if required
doAssignGroupKeys(pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.rows, rowIndex);
@@ -332,7 +331,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
}
int32_t rowIndex = pBlock->info.rows - num;
- doApplyFunctions(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs);
+ applyAggFunctionOnPartialTuples(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs);
doAssignGroupKeys(pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.rows, rowIndex);
}
}
@@ -432,7 +431,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
goto _error;
}
- SSDataBlock* pResBlock = createResDataBlock(pAggNode->node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
int32_t numOfScalarExpr = 0;
@@ -457,7 +456,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
- code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str);
+ code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -824,7 +823,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
uint32_t defaultPgsz = 0;
uint32_t defaultBufsz = 0;
- pInfo->binfo.pRes = createResDataBlock(pPartNode->node.pOutputDataBlockDesc);
+ pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->node.pOutputDataBlockDesc);
getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz);
if (!osTempSpaceAvailable()) {
@@ -1120,7 +1119,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
}
pInfo->partitionSup.needCalc = true;
- pInfo->binfo.pRes = createResDataBlock(pPartNode->part.node.pOutputDataBlockDesc);
+ pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->part.node.pOutputDataBlockDesc);
if (pInfo->binfo.pRes == NULL) {
goto _error;
}
diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c
index a1b44307d4..3839af9913 100644
--- a/source/libs/executor/src/joinoperator.c
+++ b/source/libs/executor/src/joinoperator.c
@@ -87,7 +87,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
}
int32_t numOfCols = 0;
- SSDataBlock* pResBlock = createResDataBlock(pJoinNode->node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &numOfCols);
initResultSizeInfo(&pOperator->resultInfo, 4096);
diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c
index ada7964c67..4bba3a72e1 100644
--- a/source/libs/executor/src/projectoperator.c
+++ b/source/libs/executor/src/projectoperator.c
@@ -85,7 +85,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &numOfCols);
- SSDataBlock* pResBlock = createResDataBlock(pProjPhyNode->node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pProjPhyNode->node.pOutputDataBlockDesc);
initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo);
pInfo->binfo.pRes = pResBlock;
@@ -102,7 +102,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
}
initResultSizeInfo(&pOperator->resultInfo, numOfRows);
- code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
+ code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -385,7 +385,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
}
}
- SSDataBlock* pResBlock = createResDataBlock(pPhyNode->node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->node.pOutputDataBlockDesc);
int32_t numOfRows = 4096;
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
@@ -400,7 +400,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
initResultSizeInfo(&pOperator->resultInfo, numOfRows);
blockDataEnsureCapacity(pResBlock, numOfRows);
- int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str);
+ int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 4c23d17ac7..c0bea731bd 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -484,7 +484,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
// 1. check if it is existed in meta cache
if (pCache == NULL) {
metaReaderInit(&mr, pHandle->meta, 0);
- code = metaGetTableEntryByUid(&mr, pBlock->info.uid);
+ code = metaGetTableEntryByUidCache(&mr, pBlock->info.uid);
if (code != TSDB_CODE_SUCCESS) {
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid,
@@ -508,7 +508,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(pBlock->info.uid));
if (h == NULL) {
metaReaderInit(&mr, pHandle->meta, 0);
- code = metaGetTableEntryByUid(&mr, pBlock->info.uid);
+ code = metaGetTableEntryByUidCache(&mr, pBlock->info.uid);
if (code != TSDB_CODE_SUCCESS) {
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",
@@ -885,7 +885,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
initResultSizeInfo(&pOperator->resultInfo, 4096);
- pInfo->pResBlock = createResDataBlock(pDescNode);
+ pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
@@ -2352,7 +2352,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
goto _error;
}
- pInfo->pRes = createResDataBlock(pDescNode);
+ pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
pInfo->windowSup = (SWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN};
@@ -2476,8 +2476,7 @@ static void destroyTagScanOperatorInfo(void* param) {
taosMemoryFreeClear(param);
}
-SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
- STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@@ -2499,7 +2498,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
goto _error;
}
- pInfo->pRes = createResDataBlock(pDescNode);
+ pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->readHandle = *pReadHandle;
pInfo->curPos = 0;
@@ -2613,7 +2612,7 @@ SArray* generateSortByTsInfo(SArray* colMatchInfo, int32_t order) {
return pList;
}
-int32_t dumpSQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* dst) {
+int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* dst) {
memcpy((void*)dst, (void*)src, sizeof(SQueryTableDataCond));
dst->colList = taosMemoryCalloc(src->numOfCols, sizeof(SColumnInfo));
for (int i = 0; i < src->numOfCols; i++) {
@@ -2664,7 +2663,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
taosArrayPush(pInfo->sortSourceParams, ¶m);
SQueryTableDataCond cond;
- dumpSQueryTableCond(&pInfo->base.cond, &cond);
+ dumpQueryTableCond(&pInfo->base.cond, &cond);
taosArrayPush(pInfo->queryConds, &cond);
}
@@ -2900,7 +2899,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
}
initResultSizeInfo(&pOperator->resultInfo, 1024);
- pInfo->pResBlock = createResDataBlock(pDescNode);
+ pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index c292d7e9e1..f2c8dc5083 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -47,7 +47,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc;
int32_t numOfCols = 0;
- SSDataBlock* pResBlock = createResDataBlock(pDescNode);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pDescNode);
SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols);
int32_t numOfOutputCols = 0;
@@ -509,7 +509,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
initResultSizeInfo(&pOperator->resultInfo, 1024);
pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
- pInfo->binfo.pRes = createResDataBlock(pDescNode);
+ pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
int32_t numOfOutputCols = 0;
@@ -551,6 +551,7 @@ typedef struct SMultiwayMergeOperatorInfo {
SSortHandle* pSortHandle;
SColMatchInfo matchInfo;
SSDataBlock* pInputBlock;
+ SSDataBlock* pIntermediateBlock; // to hold the intermediate result
int64_t startTs; // sort start time
bool groupSort;
bool hasGroupId;
@@ -651,12 +652,19 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
blockDataCleanup(pDataBlock);
- SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
- if (p == NULL) {
- return NULL;
+ if (pInfo->pIntermediateBlock == NULL) {
+ pInfo->pIntermediateBlock = tsortGetSortedDataBlock(pHandle);
+ if (pInfo->pIntermediateBlock == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+ blockDataEnsureCapacity(pInfo->pIntermediateBlock, capacity);
+ } else {
+ blockDataCleanup(pInfo->pIntermediateBlock);
}
- blockDataEnsureCapacity(p, capacity);
+ SSDataBlock* p = pInfo->pIntermediateBlock;
+
while (1) {
doGetSortedBlockData(pInfo, pHandle, capacity, p);
if (p->info.rows == 0) {
@@ -686,7 +694,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
pDataBlock->info.groupId = pInfo->groupId;
}
- blockDataDestroy(p);
qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
pDataBlock->info.rows);
@@ -722,6 +729,7 @@ void destroyMultiwayMergeOperatorInfo(void* param) {
SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock);
+ pInfo->pIntermediateBlock = blockDataDestroy(pInfo->pIntermediateBlock);
tsortDestroySortHandle(pInfo->pSortHandle);
taosArrayDestroy(pInfo->pSortInfo);
@@ -758,7 +766,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
}
initLimitInfo(pMergePhyNode->node.pLimit, pMergePhyNode->node.pSlimit, &pInfo->limitInfo);
- pInfo->binfo.pRes = createResDataBlock(pDescNode);
+ pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode);
int32_t rowSize = pInfo->binfo.pRes->info.rowSize;
ASSERT(rowSize < 100 * 1024 * 1024);
@@ -771,7 +779,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
}
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
- SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
+ SSDataBlock* pInputBlock = createDataBlockFromDescNode(pChildNode->pOutputDataBlockDesc);
initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c
index c5e1f2c214..7ef2668804 100644
--- a/source/libs/executor/src/sysscanoperator.c
+++ b/source/libs/executor/src/sysscanoperator.c
@@ -441,6 +441,9 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName);
if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByName, therefore, return directly
+ metaReaderClear(&smrChildTable);
+ blockDataDestroy(dataBlock);
+ pInfo->loadInfo.totalRows = 0;
return NULL;
}
@@ -456,12 +459,16 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByUid
+ metaReaderClear(&smrSuperTable);
+ metaReaderClear(&smrChildTable);
+ blockDataDestroy(dataBlock);
return NULL;
}
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
metaReaderClear(&smrSuperTable);
metaReaderClear(&smrChildTable);
+
if (numOfRows > 0) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
numOfRows = 0;
@@ -1404,7 +1411,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
pInfo->pUser = taosMemoryStrDup((void*)pUser);
pInfo->sysInfo = pScanPhyNode->sysInfo;
pInfo->showRewrite = pScanPhyNode->showRewrite;
- pInfo->pRes = createResDataBlock(pDescNode);
+ pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->pCondition = pScanNode->node.pConditions;
code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
@@ -1921,7 +1928,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
pInfo->readHandle = *readHandle;
pInfo->uid = pBlockScanNode->suid;
- pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc);
+ pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pInfo->pResBlock, 1);
int32_t numOfCols = 0;
diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c
index 7674b9e479..ba826a23d2 100644
--- a/source/libs/executor/src/tfill.c
+++ b/source/libs/executor/src/tfill.c
@@ -1651,9 +1651,9 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
}
initResultSizeInfo(&pOperator->resultInfo, 4096);
- pInfo->pRes = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
- pInfo->pSrcBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
- pInfo->pPrevSrcBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
+ pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
+ pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
+ pInfo->pPrevSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity);
blockDataEnsureCapacity(pInfo->pPrevSrcBlock, pOperator->resultInfo.capacity);
diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c
new file mode 100644
index 0000000000..d8cef86971
--- /dev/null
+++ b/source/libs/executor/src/timesliceoperator.c
@@ -0,0 +1,587 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+#include "executorimpl.h"
+#include "filter.h"
+#include "function.h"
+#include "functionMgt.h"
+#include "tcommon.h"
+#include "tcompare.h"
+#include "tdatablock.h"
+#include "tfill.h"
+#include "ttime.h"
+
+typedef struct STimeSliceOperatorInfo {
+ SSDataBlock* pRes;
+ STimeWindow win;
+ SInterval interval;
+ int64_t current;
+ SArray* pPrevRow; // SArray
+ SArray* pNextRow; // SArray
+ SArray* pLinearInfo; // SArray
+ bool isPrevRowSet;
+ bool isNextRowSet;
+ int32_t fillType; // fill type
+ SColumn tsCol; // primary timestamp column
+ SExprSupp scalarSup; // scalar calculation
+ struct SFillColInfo* pFillColInfo; // fill column info
+} STimeSliceOperatorInfo;
+
+static void destroyTimeSliceOperatorInfo(void* param);
+
+static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
+ int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+
+ SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);
+ if (!colDataIsNull_s(pColInfoData, rowIndex)) {
+ pkey->isNull = false;
+ char* val = colDataGetData(pColInfoData, rowIndex);
+ if (!IS_VAR_DATA_TYPE(pkey->type)) {
+ memcpy(pkey->pData, val, pkey->bytes);
+ } else {
+ memcpy(pkey->pData, val, varDataLen(val));
+ }
+ } else {
+ pkey->isNull = true;
+ }
+ }
+
+ pSliceInfo->isPrevRowSet = true;
+}
+
+static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
+ int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+
+ SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i);
+ if (!colDataIsNull_s(pColInfoData, rowIndex)) {
+ pkey->isNull = false;
+ char* val = colDataGetData(pColInfoData, rowIndex);
+ if (!IS_VAR_DATA_TYPE(pkey->type)) {
+ memcpy(pkey->pData, val, pkey->bytes);
+ } else {
+ memcpy(pkey->pData, val, varDataLen(val));
+ }
+ } else {
+ pkey->isNull = true;
+ }
+ }
+
+ pSliceInfo->isNextRowSet = true;
+}
+
+static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
+ int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+ SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
+ SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i);
+
+ // null value is represented by using key = INT64_MIN for now.
+ // TODO: optimize to ignore null values for linear interpolation.
+ if (!pLinearInfo->isStartSet) {
+ if (!colDataIsNull_s(pColInfoData, rowIndex)) {
+ pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
+ memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
+ }
+ pLinearInfo->isStartSet = true;
+ } else if (!pLinearInfo->isEndSet) {
+ if (!colDataIsNull_s(pColInfoData, rowIndex)) {
+ pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
+ memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
+ }
+ pLinearInfo->isEndSet = true;
+ } else {
+ pLinearInfo->start.key = pLinearInfo->end.key;
+ memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes);
+
+ if (!colDataIsNull_s(pColInfoData, rowIndex)) {
+ pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
+ memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
+ } else {
+ pLinearInfo->end.key = INT64_MIN;
+ }
+ }
+ }
+
+}
+
+static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) {
+ int32_t rows = pResBlock->info.rows;
+ blockDataEnsureCapacity(pResBlock, rows + 1);
+ // todo set the correct primary timestamp column
+
+ // output the result
+ bool hasInterp = true;
+ for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
+ SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
+
+ int32_t dstSlot = pExprInfo->base.resSchema.slotId;
+ SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
+
+ if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
+ colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false);
+ continue;
+ }
+
+ int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
+ switch (pSliceInfo->fillType) {
+ case TSDB_FILL_NULL: {
+ colDataAppendNULL(pDst, rows);
+ break;
+ }
+
+ case TSDB_FILL_SET_VALUE: {
+ SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;
+
+ if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
+ float v = 0;
+ GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
+ colDataAppend(pDst, rows, (char*)&v, false);
+ } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
+ double v = 0;
+ GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
+ colDataAppend(pDst, rows, (char*)&v, false);
+ } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
+ int64_t v = 0;
+ GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
+ colDataAppend(pDst, rows, (char*)&v, false);
+ }
+ break;
+ }
+
+ case TSDB_FILL_LINEAR: {
+ SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot);
+
+ SPoint start = pLinearInfo->start;
+ SPoint end = pLinearInfo->end;
+ SPoint current = {.key = pSliceInfo->current};
+
+ // do not interpolate before ts range, only increate pSliceInfo->current
+ if (beforeTs && !pLinearInfo->isEndSet) {
+ return true;
+ }
+
+ if (!pLinearInfo->isStartSet || !pLinearInfo->isEndSet) {
+ hasInterp = false;
+ break;
+ }
+
+ if (start.key == INT64_MIN || end.key == INT64_MIN) {
+ colDataAppendNULL(pDst, rows);
+ break;
+ }
+
+ current.val = taosMemoryCalloc(pLinearInfo->bytes, 1);
+ taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type);
+ colDataAppend(pDst, rows, (char*)current.val, false);
+
+ taosMemoryFree(current.val);
+ break;
+ }
+ case TSDB_FILL_PREV: {
+ if (!pSliceInfo->isPrevRowSet) {
+ hasInterp = false;
+ break;
+ }
+
+ SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
+ if (pkey->isNull == false) {
+ colDataAppend(pDst, rows, pkey->pData, false);
+ } else {
+ colDataAppendNULL(pDst, rows);
+ }
+ break;
+ }
+
+ case TSDB_FILL_NEXT: {
+ if (!pSliceInfo->isNextRowSet) {
+ hasInterp = false;
+ break;
+ }
+
+ SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
+ if (pkey->isNull == false) {
+ colDataAppend(pDst, rows, pkey->pData, false);
+ } else {
+ colDataAppendNULL(pDst, rows);
+ }
+ break;
+ }
+
+ case TSDB_FILL_NONE:
+ default:
+ break;
+ }
+ }
+
+ if (hasInterp) {
+ pResBlock->info.rows += 1;
+ }
+
+ return hasInterp;
+}
+
+static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock,
+ SSDataBlock* pSrcBlock, int32_t index) {
+ blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1);
+ for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
+ SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
+
+ int32_t dstSlot = pExprInfo->base.resSchema.slotId;
+ SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
+
+ if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
+ colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false);
+ } else {
+ int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
+ SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot);
+
+ if (colDataIsNull_s(pSrc, index)) {
+ colDataAppendNULL(pDst, pResBlock->info.rows);
+ continue;
+ }
+
+ char* v = colDataGetData(pSrc, index);
+ colDataAppend(pDst, pResBlock->info.rows, v, false);
+ }
+ }
+
+ pResBlock->info.rows += 1;
+ return;
+}
+
+
+static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
+ if (pInfo->pPrevRow != NULL) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
+ if (pInfo->pPrevRow == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
+
+ SGroupKeys key = {0};
+ key.bytes = pColInfo->info.bytes;
+ key.type = pColInfo->info.type;
+ key.isNull = false;
+ key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
+ taosArrayPush(pInfo->pPrevRow, &key);
+ }
+
+ pInfo->isPrevRowSet = false;
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
+ if (pInfo->pNextRow != NULL) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys));
+ if (pInfo->pNextRow == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
+
+ SGroupKeys key = {0};
+ key.bytes = pColInfo->info.bytes;
+ key.type = pColInfo->info.type;
+ key.isNull = false;
+ key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
+ taosArrayPush(pInfo->pNextRow, &key);
+ }
+
+ pInfo->isNextRowSet = false;
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
+ if (pInfo->pLinearInfo != NULL) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo));
+ if (pInfo->pLinearInfo == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
+
+ SFillLinearInfo linearInfo = {0};
+ linearInfo.start.key = INT64_MIN;
+ linearInfo.end.key = INT64_MIN;
+ linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes);
+ linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes);
+ linearInfo.isStartSet = false;
+ linearInfo.isEndSet = false;
+ linearInfo.type = pColInfo->info.type;
+ linearInfo.bytes = pColInfo->info.bytes;
+ taosArrayPush(pInfo->pLinearInfo, &linearInfo);
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
+ int32_t code;
+ code = initPrevRowsKeeper(pInfo, pBlock);
+ if (code != TSDB_CODE_SUCCESS) {
+ return TSDB_CODE_FAILED;
+ }
+
+ code = initNextRowsKeeper(pInfo, pBlock);
+ if (code != TSDB_CODE_SUCCESS) {
+ return TSDB_CODE_FAILED;
+ }
+
+ code = initFillLinearInfo(pInfo, pBlock);
+ if (code != TSDB_CODE_SUCCESS) {
+ return TSDB_CODE_FAILED;
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
+ if (pOperator->status == OP_EXEC_DONE) {
+ return NULL;
+ }
+
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+
+ STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
+ SSDataBlock* pResBlock = pSliceInfo->pRes;
+ SExprSupp* pSup = &pOperator->exprSupp;
+
+ int32_t order = TSDB_ORDER_ASC;
+ SInterval* pInterval = &pSliceInfo->interval;
+ SOperatorInfo* downstream = pOperator->pDownstream[0];
+
+ blockDataCleanup(pResBlock);
+
+ while (1) {
+ SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
+ if (pBlock == NULL) {
+ break;
+ }
+
+ int32_t code = initKeeperInfo(pSliceInfo, pBlock);
+ if (code != TSDB_CODE_SUCCESS) {
+ T_LONG_JMP(pTaskInfo->env, code);
+ }
+
+ // the pDataBlock are always the same one, no need to call this again
+ setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
+
+ SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
+ for (int32_t i = 0; i < pBlock->info.rows; ++i) {
+ int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
+
+ if (pSliceInfo->current > pSliceInfo->win.ekey) {
+ setOperatorCompleted(pOperator);
+ break;
+ }
+
+ if (ts == pSliceInfo->current) {
+ addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i);
+
+ doKeepPrevRows(pSliceInfo, pBlock, i);
+ doKeepLinearInfo(pSliceInfo, pBlock, i);
+
+ pSliceInfo->current =
+ taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
+ if (pSliceInfo->current > pSliceInfo->win.ekey) {
+ setOperatorCompleted(pOperator);
+ break;
+ }
+ } else if (ts < pSliceInfo->current) {
+ // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
+ doKeepPrevRows(pSliceInfo, pBlock, i);
+ doKeepLinearInfo(pSliceInfo, pBlock, i);
+
+ if (i < pBlock->info.rows - 1) {
+ // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
+ doKeepNextRows(pSliceInfo, pBlock, i + 1);
+ int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
+ if (nextTs > pSliceInfo->current) {
+ while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
+ if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) {
+ break;
+ } else {
+ pSliceInfo->current =
+ taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
+ }
+ }
+
+ if (pSliceInfo->current > pSliceInfo->win.ekey) {
+ setOperatorCompleted(pOperator);
+ break;
+ }
+ } else {
+ // ignore current row, and do nothing
+ }
+ } else { // it is the last row of current block
+ doKeepPrevRows(pSliceInfo, pBlock, i);
+ }
+ } else { // ts > pSliceInfo->current
+ // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
+ doKeepNextRows(pSliceInfo, pBlock, i);
+ doKeepLinearInfo(pSliceInfo, pBlock, i);
+
+ while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
+ if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) {
+ break;
+ } else {
+ pSliceInfo->current =
+ taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
+ }
+ }
+
+ // add current row if timestamp match
+ if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
+ addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i);
+ doKeepPrevRows(pSliceInfo, pBlock, i);
+
+ pSliceInfo->current =
+ taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
+ }
+
+ if (pSliceInfo->current > pSliceInfo->win.ekey) {
+ setOperatorCompleted(pOperator);
+ break;
+ }
+ }
+ }
+ }
+
+ // check if need to interpolate after last datablock
+ // except for fill(next), fill(linear)
+ while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
+ pSliceInfo->fillType != TSDB_FILL_LINEAR) {
+ genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false);
+ pSliceInfo->current =
+ taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
+ }
+
+ // restore the value
+ setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
+ if (pResBlock->info.rows == 0) {
+ pOperator->status = OP_EXEC_DONE;
+ }
+
+ return pResBlock->info.rows == 0 ? NULL : pResBlock;
+}
+
+SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
+ STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
+ SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+ if (pOperator == NULL || pInfo == NULL) {
+ goto _error;
+ }
+
+ SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
+ SExprSupp* pSup = &pOperator->exprSupp;
+
+ int32_t numOfExprs = 0;
+ SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
+ int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
+ if (pInterpPhyNode->pExprs != NULL) {
+ int32_t num = 0;
+ SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
+ code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+ }
+
+ pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
+ pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
+ initResultSizeInfo(&pOperator->resultInfo, 4096);
+
+ pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, NULL, 0, (SNodeListNode*)pInterpPhyNode->pFillValues);
+ pInfo->pLinearInfo = NULL;
+ pInfo->pRes = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
+ pInfo->win = pInterpPhyNode->timeRange;
+ pInfo->interval.interval = pInterpPhyNode->interval;
+ pInfo->current = pInfo->win.skey;
+
+ if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
+ STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info;
+ pScanInfo->base.cond.twindows = pInfo->win;
+ pScanInfo->base.cond.type = TIMEWINDOW_RANGE_EXTERNAL;
+ }
+
+ setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo,
+ pTaskInfo);
+ pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL);
+
+ blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+
+ code = appendDownstream(pOperator, &downstream, 1);
+ return pOperator;
+
+ _error:
+ taosMemoryFree(pInfo);
+ taosMemoryFree(pOperator);
+ pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+}
+
+void destroyTimeSliceOperatorInfo(void* param) {
+ STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;
+
+ pInfo->pRes = blockDataDestroy(pInfo->pRes);
+
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
+ SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);
+ taosMemoryFree(pKey->pData);
+ }
+ taosArrayDestroy(pInfo->pPrevRow);
+
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->pNextRow); ++i) {
+ SGroupKeys* pKey = taosArrayGet(pInfo->pNextRow, i);
+ taosMemoryFree(pKey->pData);
+ }
+ taosArrayDestroy(pInfo->pNextRow);
+
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->pLinearInfo); ++i) {
+ SFillLinearInfo* pKey = taosArrayGet(pInfo->pLinearInfo, i);
+ taosMemoryFree(pKey->start.val);
+ taosMemoryFree(pKey->end.val);
+ }
+ taosArrayDestroy(pInfo->pLinearInfo);
+
+ taosMemoryFree(pInfo->pFillColInfo);
+ taosMemoryFreeClear(param);
+}
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 013b8d39de..0e0ec5b339 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -22,13 +22,37 @@
#include "tfill.h"
#include "ttime.h"
+#define IS_FINAL_OP(op) ((op)->isFinal)
+
+typedef struct SSessionAggOperatorInfo {
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
+ SGroupResInfo groupResInfo;
+ SWindowRowsSup winSup;
+ bool reptScan; // next round scan
+ int64_t gap; // session window gap
+ int32_t tsSlotId; // primary timestamp slot id
+ STimeWindowAggSupp twAggSup;
+} SSessionAggOperatorInfo;
+
+typedef struct SStateWindowOperatorInfo {
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
+ SExprSupp scalarSup;
+ SGroupResInfo groupResInfo;
+ SWindowRowsSup winSup;
+ SColumn stateCol; // start row index
+ bool hasKey;
+ SStateKeys stateKey;
+ int32_t tsSlotId; // primary timestamp column slot id
+ STimeWindowAggSupp twAggSup;
+} SStateWindowOperatorInfo;
+
typedef enum SResultTsInterpType {
RESULT_ROW_START_INTERP = 1,
RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;
-#define IS_FINAL_OP(op) ((op)->isFinal)
-
typedef struct SPullWindowInfo {
STimeWindow window;
uint64_t groupId;
@@ -640,7 +664,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows,
numOfExprs);
if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
@@ -937,7 +961,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
numOfOutput);
doCloseWindow(pResultRowInfo, pInfo, pResult);
@@ -972,7 +996,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
}
#endif
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
numOfOutput);
doCloseWindow(pResultRowInfo, pInfo, pResult);
}
@@ -1140,7 +1164,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
// here we start a new session window
@@ -1165,7 +1189,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
pBlock->info.rows, numOfOutput);
}
@@ -1706,7 +1730,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
goto _error;
}
- SSDataBlock* pResBlock = createResDataBlock(pPhyNode->window.node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
SExprSupp* pSup = &pOperator->exprSupp;
@@ -1717,7 +1741,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
- int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
+ int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -1845,7 +1869,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator
// pInfo->numOfRows data belong to the current session window
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
// here we start a new session window
@@ -1863,7 +1887,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
pBlock->info.rows, numOfOutput);
}
@@ -1938,552 +1962,6 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
}
-static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
- int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
- for (int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-
- SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);
- if (!colDataIsNull_s(pColInfoData, rowIndex)) {
- pkey->isNull = false;
- char* val = colDataGetData(pColInfoData, rowIndex);
- if (!IS_VAR_DATA_TYPE(pkey->type)) {
- memcpy(pkey->pData, val, pkey->bytes);
- } else {
- memcpy(pkey->pData, val, varDataLen(val));
- }
- } else {
- pkey->isNull = true;
- }
- }
-
- pSliceInfo->isPrevRowSet = true;
-}
-
-static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
- int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
- for (int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-
- SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i);
- if (!colDataIsNull_s(pColInfoData, rowIndex)) {
- pkey->isNull = false;
- char* val = colDataGetData(pColInfoData, rowIndex);
- if (!IS_VAR_DATA_TYPE(pkey->type)) {
- memcpy(pkey->pData, val, pkey->bytes);
- } else {
- memcpy(pkey->pData, val, varDataLen(val));
- }
- } else {
- pkey->isNull = true;
- }
- }
-
- pSliceInfo->isNextRowSet = true;
-}
-
-static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
- int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
- for (int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
- SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
- SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i);
-
- // null value is represented by using key = INT64_MIN for now.
- // TODO: optimize to ignore null values for linear interpolation.
- if (!pLinearInfo->isStartSet) {
- if (!colDataIsNull_s(pColInfoData, rowIndex)) {
- pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
- memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
- }
- pLinearInfo->isStartSet = true;
- } else if (!pLinearInfo->isEndSet) {
- if (!colDataIsNull_s(pColInfoData, rowIndex)) {
- pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
- memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
- }
- pLinearInfo->isEndSet = true;
- } else {
- pLinearInfo->start.key = pLinearInfo->end.key;
- memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes);
-
- if (!colDataIsNull_s(pColInfoData, rowIndex)) {
- pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
- memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
- } else {
- pLinearInfo->end.key = INT64_MIN;
- }
- }
- }
-
-}
-
-static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) {
- int32_t rows = pResBlock->info.rows;
- blockDataEnsureCapacity(pResBlock, rows + 1);
- // todo set the correct primary timestamp column
-
- // output the result
- bool hasInterp = true;
- for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
- SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
-
- int32_t dstSlot = pExprInfo->base.resSchema.slotId;
- SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
-
- if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
- colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false);
- continue;
- }
-
- int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
- switch (pSliceInfo->fillType) {
- case TSDB_FILL_NULL: {
- colDataAppendNULL(pDst, rows);
- break;
- }
-
- case TSDB_FILL_SET_VALUE: {
- SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;
-
- if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
- float v = 0;
- GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
- colDataAppend(pDst, rows, (char*)&v, false);
- } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
- double v = 0;
- GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
- colDataAppend(pDst, rows, (char*)&v, false);
- } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
- int64_t v = 0;
- GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
- colDataAppend(pDst, rows, (char*)&v, false);
- }
- break;
- }
-
- case TSDB_FILL_LINEAR: {
- SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot);
-
- SPoint start = pLinearInfo->start;
- SPoint end = pLinearInfo->end;
- SPoint current = {.key = pSliceInfo->current};
-
- // do not interpolate before ts range, only increate pSliceInfo->current
- if (beforeTs && !pLinearInfo->isEndSet) {
- return true;
- }
-
- if (!pLinearInfo->isStartSet || !pLinearInfo->isEndSet) {
- hasInterp = false;
- break;
- }
-
- if (start.key == INT64_MIN || end.key == INT64_MIN) {
- colDataAppendNULL(pDst, rows);
- break;
- }
-
- current.val = taosMemoryCalloc(pLinearInfo->bytes, 1);
- taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type);
- colDataAppend(pDst, rows, (char*)current.val, false);
-
- taosMemoryFree(current.val);
- break;
- }
- case TSDB_FILL_PREV: {
- if (!pSliceInfo->isPrevRowSet) {
- hasInterp = false;
- break;
- }
-
- SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
- if (pkey->isNull == false) {
- colDataAppend(pDst, rows, pkey->pData, false);
- } else {
- colDataAppendNULL(pDst, rows);
- }
- break;
- }
-
- case TSDB_FILL_NEXT: {
- if (!pSliceInfo->isNextRowSet) {
- hasInterp = false;
- break;
- }
-
- SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
- if (pkey->isNull == false) {
- colDataAppend(pDst, rows, pkey->pData, false);
- } else {
- colDataAppendNULL(pDst, rows);
- }
- break;
- }
-
- case TSDB_FILL_NONE:
- default:
- break;
- }
- }
-
- if (hasInterp) {
- pResBlock->info.rows += 1;
- }
-
- return hasInterp;
-}
-
-static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock,
- SSDataBlock* pSrcBlock, int32_t index) {
- blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1);
- for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
- SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
-
- int32_t dstSlot = pExprInfo->base.resSchema.slotId;
- SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
-
- if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
- colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false);
- } else {
- int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
- SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot);
-
- if (colDataIsNull_s(pSrc, index)) {
- colDataAppendNULL(pDst, pResBlock->info.rows);
- continue;
- }
-
- char* v = colDataGetData(pSrc, index);
- colDataAppend(pDst, pResBlock->info.rows, v, false);
- }
- }
-
- pResBlock->info.rows += 1;
- return;
-}
-
-
-static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
- if (pInfo->pPrevRow != NULL) {
- return TSDB_CODE_SUCCESS;
- }
-
- pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
- if (pInfo->pPrevRow == NULL) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
-
- int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
- for (int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
-
- SGroupKeys key = {0};
- key.bytes = pColInfo->info.bytes;
- key.type = pColInfo->info.type;
- key.isNull = false;
- key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
- taosArrayPush(pInfo->pPrevRow, &key);
- }
-
- pInfo->isPrevRowSet = false;
-
- return TSDB_CODE_SUCCESS;
-}
-
-static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
- if (pInfo->pNextRow != NULL) {
- return TSDB_CODE_SUCCESS;
- }
-
- pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys));
- if (pInfo->pNextRow == NULL) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
-
- int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
- for (int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
-
- SGroupKeys key = {0};
- key.bytes = pColInfo->info.bytes;
- key.type = pColInfo->info.type;
- key.isNull = false;
- key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
- taosArrayPush(pInfo->pNextRow, &key);
- }
-
- pInfo->isNextRowSet = false;
-
- return TSDB_CODE_SUCCESS;
-}
-
-static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
- if (pInfo->pLinearInfo != NULL) {
- return TSDB_CODE_SUCCESS;
- }
-
- pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo));
- if (pInfo->pLinearInfo == NULL) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
-
- int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
- for (int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
-
- SFillLinearInfo linearInfo = {0};
- linearInfo.start.key = INT64_MIN;
- linearInfo.end.key = INT64_MIN;
- linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes);
- linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes);
- linearInfo.isStartSet = false;
- linearInfo.isEndSet = false;
- linearInfo.type = pColInfo->info.type;
- linearInfo.bytes = pColInfo->info.bytes;
- taosArrayPush(pInfo->pLinearInfo, &linearInfo);
- }
-
- return TSDB_CODE_SUCCESS;
-}
-
-static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
- int32_t code;
- code = initPrevRowsKeeper(pInfo, pBlock);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_FAILED;
- }
-
- code = initNextRowsKeeper(pInfo, pBlock);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_FAILED;
- }
-
- code = initFillLinearInfo(pInfo, pBlock);
- if (code != TSDB_CODE_SUCCESS) {
- return TSDB_CODE_FAILED;
- }
-
- return TSDB_CODE_SUCCESS;
-}
-
-static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
- if (pOperator->status == OP_EXEC_DONE) {
- return NULL;
- }
-
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
-
- STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
- SSDataBlock* pResBlock = pSliceInfo->pRes;
- SExprSupp* pSup = &pOperator->exprSupp;
-
- int32_t order = TSDB_ORDER_ASC;
- SInterval* pInterval = &pSliceInfo->interval;
- SOperatorInfo* downstream = pOperator->pDownstream[0];
-
- blockDataCleanup(pResBlock);
-
- while (1) {
- SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
- if (pBlock == NULL) {
- break;
- }
-
- int32_t code = initKeeperInfo(pSliceInfo, pBlock);
- if (code != TSDB_CODE_SUCCESS) {
- T_LONG_JMP(pTaskInfo->env, code);
- }
-
- // the pDataBlock are always the same one, no need to call this again
- setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
-
- SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
- for (int32_t i = 0; i < pBlock->info.rows; ++i) {
- int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
-
- if (pSliceInfo->current > pSliceInfo->win.ekey) {
- setOperatorCompleted(pOperator);
- break;
- }
-
- if (ts == pSliceInfo->current) {
- addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i);
-
- doKeepPrevRows(pSliceInfo, pBlock, i);
- doKeepLinearInfo(pSliceInfo, pBlock, i);
-
- pSliceInfo->current =
- taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
- if (pSliceInfo->current > pSliceInfo->win.ekey) {
- setOperatorCompleted(pOperator);
- break;
- }
- } else if (ts < pSliceInfo->current) {
- // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
- doKeepPrevRows(pSliceInfo, pBlock, i);
- doKeepLinearInfo(pSliceInfo, pBlock, i);
-
- if (i < pBlock->info.rows - 1) {
- // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
- doKeepNextRows(pSliceInfo, pBlock, i + 1);
- int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
- if (nextTs > pSliceInfo->current) {
- while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
- if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) {
- break;
- } else {
- pSliceInfo->current =
- taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
- }
- }
-
- if (pSliceInfo->current > pSliceInfo->win.ekey) {
- setOperatorCompleted(pOperator);
- break;
- }
- } else {
- // ignore current row, and do nothing
- }
- } else { // it is the last row of current block
- doKeepPrevRows(pSliceInfo, pBlock, i);
- }
- } else { // ts > pSliceInfo->current
- // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
- doKeepNextRows(pSliceInfo, pBlock, i);
- doKeepLinearInfo(pSliceInfo, pBlock, i);
-
- while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
- if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) {
- break;
- } else {
- pSliceInfo->current =
- taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
- }
- }
-
- // add current row if timestamp match
- if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
- addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i);
- doKeepPrevRows(pSliceInfo, pBlock, i);
-
- pSliceInfo->current =
- taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
- }
-
- if (pSliceInfo->current > pSliceInfo->win.ekey) {
- setOperatorCompleted(pOperator);
- break;
- }
- }
- }
- }
-
- // check if need to interpolate after last datablock
- // except for fill(next), fill(linear)
- while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
- pSliceInfo->fillType != TSDB_FILL_LINEAR) {
- genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false);
- pSliceInfo->current =
- taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
- }
-
- // restore the value
- setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
- if (pResBlock->info.rows == 0) {
- pOperator->status = OP_EXEC_DONE;
- }
-
- return pResBlock->info.rows == 0 ? NULL : pResBlock;
-}
-
-void destroyTimeSliceOperatorInfo(void* param) {
- STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;
-
- pInfo->pRes = blockDataDestroy(pInfo->pRes);
-
- for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
- SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);
- taosMemoryFree(pKey->pData);
- }
- taosArrayDestroy(pInfo->pPrevRow);
-
- for (int32_t i = 0; i < taosArrayGetSize(pInfo->pNextRow); ++i) {
- SGroupKeys* pKey = taosArrayGet(pInfo->pNextRow, i);
- taosMemoryFree(pKey->pData);
- }
- taosArrayDestroy(pInfo->pNextRow);
-
- for (int32_t i = 0; i < taosArrayGetSize(pInfo->pLinearInfo); ++i) {
- SFillLinearInfo* pKey = taosArrayGet(pInfo->pLinearInfo, i);
- taosMemoryFree(pKey->start.val);
- taosMemoryFree(pKey->end.val);
- }
- taosArrayDestroy(pInfo->pLinearInfo);
-
- taosMemoryFree(pInfo->pFillColInfo);
- taosMemoryFreeClear(param);
-}
-
-SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
- STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
- SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
- if (pOperator == NULL || pInfo == NULL) {
- goto _error;
- }
-
- SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
- SExprSupp* pSup = &pOperator->exprSupp;
-
- int32_t numOfExprs = 0;
- SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
- int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs);
- if (code != TSDB_CODE_SUCCESS) {
- goto _error;
- }
-
- if (pInterpPhyNode->pExprs != NULL) {
- int32_t num = 0;
- SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
- code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
- if (code != TSDB_CODE_SUCCESS) {
- goto _error;
- }
- }
-
- pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
- pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
- initResultSizeInfo(&pOperator->resultInfo, 4096);
-
- pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, NULL, 0, (SNodeListNode*)pInterpPhyNode->pFillValues);
- pInfo->pLinearInfo = NULL;
- pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
- pInfo->win = pInterpPhyNode->timeRange;
- pInfo->interval.interval = pInterpPhyNode->interval;
- pInfo->current = pInfo->win.skey;
-
- if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
- STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info;
- pScanInfo->base.cond.twindows = pInfo->win;
- pScanInfo->base.cond.type = TIMEWINDOW_RANGE_EXTERNAL;
- }
-
- setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo,
- pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL);
-
- blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
-
- code = appendDownstream(pOperator, &downstream, 1);
- return pOperator;
-
-_error:
- taosMemoryFree(pInfo);
- taosMemoryFree(pOperator);
- pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
- return NULL;
-}
-
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
SExecTaskInfo* pTaskInfo) {
SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
@@ -2523,12 +2001,12 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
initResultSizeInfo(&pOperator->resultInfo, 4096);
- code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
+ code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
- SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pStateNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
initResultRowInfo(&pInfo->binfo.resultRowInfo);
@@ -2588,10 +2066,10 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
- SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pSessionNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
- int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
+ int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -2765,26 +2243,6 @@ static void clearSpecialDataBlock(SSDataBlock* pBlock) {
blockDataCleanup(pBlock);
}
-void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
- // ASSERT(pDest->info.capacity >= pSource->info.rows);
- blockDataEnsureCapacity(pDest, pSource->info.rows);
- clearSpecialDataBlock(pDest);
- SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
- SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
-
- // copy timestamp column
- colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
- for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
- SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
- colDataAppendNNULL(pCol, 0, pSource->info.rows);
- }
-
- pDest->info.rows = pSource->info.rows;
- pDest->info.groupId = pSource->info.groupId;
- pDest->info.type = pSource->info.type;
- blockDataUpdateTsWindow(pDest, 0);
-}
-
static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
clearSpecialDataBlock(pBlock);
int32_t size = taosArrayGetSize(array);
@@ -2975,7 +2433,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0);
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
pSDataBlock->info.rows, numOfOutput);
SWinKey key = {
.ts = nextWin.skey,
@@ -3226,10 +2684,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
- SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
- int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
+ int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -3551,7 +3009,7 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, false);
- doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput);
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput);
return TSDB_CODE_SUCCESS;
}
@@ -4066,7 +3524,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
SExprSupp* pSup = &pOperator->exprSupp;
SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
- SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
@@ -4586,7 +4044,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
SExprSupp* pSup = &pOperator->exprSupp;
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);
- SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
@@ -4695,7 +4153,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
}
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
pBlock->info.rows, pSup->numOfExprs);
finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
@@ -4715,7 +4173,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
}
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
- doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
pBlock->info.rows, pSup->numOfExprs);
}
@@ -4868,12 +4326,12 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
- code = initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
+ code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
- SSDataBlock* pResBlock = createResDataBlock(pNode->window.node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&iaInfo->binfo, pResBlock);
initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
@@ -5022,7 +4480,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo*
}
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
- doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, iaInfo, pResult);
@@ -5054,7 +4512,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo*
doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup);
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
- doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
+ applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, iaInfo, pResult);
@@ -5173,12 +4631,12 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
initResultSizeInfo(&pOperator->resultInfo, 4096);
- int32_t code = initAggInfo(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
+ int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
- SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pIntervalPhyNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&pIntervalInfo->binfo, pResBlock);
initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win);
@@ -5334,7 +4792,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
ASSERT(numOfCols > 0);
- SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
+ SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
SInterval interval = {
.interval = pIntervalPhyNode->interval,
.sliding = pIntervalPhyNode->sliding,
@@ -5369,7 +4827,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
initResultSizeInfo(&pOperator->resultInfo, 4096);
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
- code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
+ code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -5423,3 +4881,4 @@ _error:
pTaskInfo->code = code;
return NULL;
}
+
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 02f2b15a8f..e0a0b9442e 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -801,7 +801,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) {
index = tMergeTreeGetChosenIndex(pHandle->pMergeTree);
pSource = pHandle->cmpParam.pSources[index];
- assert(pSource->src.pBlock != NULL);
+ ASSERT(pSource->src.pBlock != NULL);
pHandle->tupleHandle.rowIndex = pSource->src.rowIndex;
pHandle->tupleHandle.pBlock = pSource->src.pBlock;
diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt
index fa241dc6ef..9d11d7b376 100644
--- a/source/libs/function/CMakeLists.txt
+++ b/source/libs/function/CMakeLists.txt
@@ -1,6 +1,7 @@
aux_source_directory(src FUNCTION_SRC)
+aux_source_directory(src/detail FUNCTION_SRC_DETAIL)
list(REMOVE_ITEM FUNCTION_SRC src/udfd.c)
-add_library(function STATIC ${FUNCTION_SRC})
+add_library(function STATIC ${FUNCTION_SRC} ${FUNCTION_SRC_DETAIL})
target_include_directories(
function
PUBLIC
diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h
index 89e9673b06..307a82e256 100644
--- a/source/libs/function/inc/builtinsimpl.h
+++ b/source/libs/function/inc/builtinsimpl.h
@@ -23,6 +23,32 @@ extern "C" {
#include "function.h"
#include "functionMgt.h"
+typedef struct SSumRes {
+ union {
+ int64_t isum;
+ uint64_t usum;
+ double dsum;
+ };
+ int16_t type;
+ int64_t prevTs;
+ bool isPrevTsSet;
+} SSumRes;
+
+typedef struct SMinmaxResInfo {
+ bool assign; // assign the first value or not
+ int64_t v;
+ STuplePos tuplePos;
+
+ STuplePos nullTuplePos;
+ bool nullTupleSaved;
+ int16_t type;
+} SMinmaxResInfo;
+int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc);
+
+STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey);
+int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
+const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos);
+
bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
@@ -119,15 +145,10 @@ EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow);
int32_t lastRowFunction(SqlFunctionCtx* pCtx);
bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
-bool getTopBotMergeFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
bool topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t topFunction(SqlFunctionCtx* pCtx);
-int32_t topFunctionMerge(SqlFunctionCtx* pCtx);
int32_t bottomFunction(SqlFunctionCtx* pCtx);
-int32_t bottomFunctionMerge(SqlFunctionCtx* pCtx);
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
-int32_t topBotPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
-int32_t topBotMergeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t topCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t getTopBotInfoSize(int64_t numOfItems);
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 622baa76c9..e572ce7a40 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -41,34 +41,15 @@
#define HLL_BUCKET_MASK (HLL_BUCKETS - 1)
#define HLL_ALPHA_INF 0.721347520444481703680 // constant for 0.5/ln(2)
-typedef struct SSumRes {
- union {
- int64_t isum;
- uint64_t usum;
- double dsum;
- };
- int16_t type;
- int64_t prevTs; // used for csum only
- bool isPrevTsSet; // used for csum only
-
-} SSumRes;
-
-typedef struct SAvgRes {
- double result;
- SSumRes sum;
- int64_t count;
- int16_t type; // store the original input type, used in merge function
-} SAvgRes;
-
-typedef struct SMinmaxResInfo {
- bool assign; // assign the first value or not
- int64_t v;
- STuplePos tuplePos;
-
- STuplePos nullTuplePos;
- bool nullTupleSaved;
- int16_t type;
-} SMinmaxResInfo;
+//typedef struct SMinmaxResInfo {
+// bool assign; // assign the first value or not
+// int64_t v;
+// STuplePos tuplePos;
+//
+// STuplePos nullTuplePos;
+// bool nullTupleSaved;
+// int16_t type;
+//} SMinmaxResInfo;
typedef struct STopBotResItem {
SVariant v;
@@ -369,19 +350,19 @@ typedef struct SGroupKeyInfo {
} \
} while (0)
-#define LIST_AVG_N(sumT, T) \
- do { \
- T* plist = (T*)pCol->pData; \
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { \
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { \
- continue; \
- } \
- \
- numOfElem += 1; \
- pAvgRes->count -= 1; \
- sumT -= plist[i]; \
- } \
- } while (0)
+//#define LIST_AVG_N(sumT, T) \
+// do { \
+// T* plist = (T*)pCol->pData; \
+// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { \
+// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { \
+// continue; \
+// } \
+// \
+// numOfElem += 1; \
+// pAvgRes->count -= 1; \
+// sumT -= plist[i]; \
+// } \
+// } while (0)
#define LIST_STDDEV_SUB_N(sumT, T) \
do { \
@@ -514,13 +495,13 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) {
int32_t numOfElem = 0;
/*
- * 1. column data missing (schema modified) causes pInputCol->hasNull == true. pInput->colDataAggIsSet == true;
- * 2. for general non-primary key columns, pInputCol->hasNull may be true or false, pInput->colDataAggIsSet == true;
- * 3. for primary key column, pInputCol->hasNull always be false, pInput->colDataAggIsSet == false;
+ * 1. column data missing (schema modified) causes pInputCol->hasNull == true. pInput->colDataSMAIsSet == true;
+ * 2. for general non-primary key columns, pInputCol->hasNull may be true or false, pInput->colDataSMAIsSet == true;
+ * 3. for primary key column, pInputCol->hasNull always be false, pInput->colDataSMAIsSet == false;
*/
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
- if (pInput->colDataAggIsSet && pInput->totalRows == pInput->numOfRows) {
+ if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) {
numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull;
ASSERT(numOfElem >= 0);
} else {
@@ -609,7 +590,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
goto _sum_over;
}
- if (pInput->colDataAggIsSet) {
+ if (pInput->colDataSMAIsSet) {
numOfElem = pInput->numOfRows - pAgg->numOfNull;
ASSERT(numOfElem >= 0);
@@ -674,7 +655,7 @@ int32_t sumInvertFunction(SqlFunctionCtx* pCtx) {
SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
- if (pInput->colDataAggIsSet) {
+ if (pInput->colDataSMAIsSet) {
numOfElem = pInput->numOfRows - pAgg->numOfNull;
ASSERT(numOfElem >= 0);
@@ -748,378 +729,6 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true;
}
-int32_t getAvgInfoSize() { return (int32_t)sizeof(SAvgRes); }
-
-bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
- pEnv->calcMemSize = sizeof(SAvgRes);
- return true;
-}
-
-bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
- if (!functionSetup(pCtx, pResultInfo)) {
- return false;
- }
-
- SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo);
- memset(pRes, 0, sizeof(SAvgRes));
- return true;
-}
-
-int32_t avgFunction(SqlFunctionCtx* pCtx) {
- int32_t numOfElem = 0;
-
- SInputColumnInfoData* pInput = &pCtx->input;
- SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0];
- int32_t type = pInput->pData[0]->info.type;
-
- SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
- pAvgRes->type = type;
-
- // computing based on the true data block
- SColumnInfoData* pCol = pInput->pData[0];
-
- int32_t start = pInput->startRowIndex;
- int32_t numOfRows = pInput->numOfRows;
-
- if (IS_NULL_TYPE(type)) {
- numOfElem = 0;
- goto _avg_over;
- }
-
- if (pInput->colDataAggIsSet) {
- numOfElem = numOfRows - pAgg->numOfNull;
- ASSERT(numOfElem >= 0);
-
- pAvgRes->count += numOfElem;
- if (IS_SIGNED_NUMERIC_TYPE(type)) {
- pAvgRes->sum.isum += pAgg->sum;
- } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
- pAvgRes->sum.usum += pAgg->sum;
- } else if (IS_FLOAT_TYPE(type)) {
- pAvgRes->sum.dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum));
- }
- } else { // computing based on the true data block
- switch (type) {
- case TSDB_DATA_TYPE_TINYINT: {
- int8_t* plist = (int8_t*)pCol->pData;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.isum += plist[i];
- }
-
- break;
- }
-
- case TSDB_DATA_TYPE_SMALLINT: {
- int16_t* plist = (int16_t*)pCol->pData;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.isum += plist[i];
- }
- break;
- }
-
- case TSDB_DATA_TYPE_INT: {
- int32_t* plist = (int32_t*)pCol->pData;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.isum += plist[i];
- }
-
- break;
- }
-
- case TSDB_DATA_TYPE_BIGINT: {
- int64_t* plist = (int64_t*)pCol->pData;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.isum += plist[i];
- }
- break;
- }
-
- case TSDB_DATA_TYPE_UTINYINT: {
- uint8_t* plist = (uint8_t*)pCol->pData;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.usum += plist[i];
- }
-
- break;
- }
-
- case TSDB_DATA_TYPE_USMALLINT: {
- uint16_t* plist = (uint16_t*)pCol->pData;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.usum += plist[i];
- }
- break;
- }
-
- case TSDB_DATA_TYPE_UINT: {
- uint32_t* plist = (uint32_t*)pCol->pData;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.usum += plist[i];
- }
-
- break;
- }
-
- case TSDB_DATA_TYPE_UBIGINT: {
- uint64_t* plist = (uint64_t*)pCol->pData;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.usum += plist[i];
- }
- break;
- }
-
- case TSDB_DATA_TYPE_FLOAT: {
- float* plist = (float*)pCol->pData;
- // float val = 0;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.dsum += plist[i];
- }
- // pAvgRes->sum.dsum = val;
- break;
- }
-
- case TSDB_DATA_TYPE_DOUBLE: {
- double* plist = (double*)pCol->pData;
- for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
- if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- numOfElem += 1;
- pAvgRes->count += 1;
- pAvgRes->sum.dsum += plist[i];
- }
- break;
- }
-
- default:
- break;
- }
- }
-
-_avg_over:
- // data in the check operation are all null, not output
- SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
- return TSDB_CODE_SUCCESS;
-}
-
-static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) {
- pOutput->type = pInput->type;
- if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) {
- pOutput->sum.isum += pInput->sum.isum;
- } else if (IS_UNSIGNED_NUMERIC_TYPE(pOutput->type)) {
- pOutput->sum.usum += pInput->sum.usum;
- } else {
- pOutput->sum.dsum += pInput->sum.dsum;
- }
-
- pOutput->count += pInput->count;
-
- return;
-}
-
-int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) {
- SInputColumnInfoData* pInput = &pCtx->input;
- SColumnInfoData* pCol = pInput->pData[0];
- ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
-
- SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
-
- int32_t start = pInput->startRowIndex;
-
- for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
- char* data = colDataGetData(pCol, i);
- SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data);
- avgTransferInfo(pInputInfo, pInfo);
- }
-
- SET_VAL(GET_RES_INFO(pCtx), 1, 1);
-
- return TSDB_CODE_SUCCESS;
-}
-
-int32_t avgInvertFunction(SqlFunctionCtx* pCtx) {
- int32_t numOfElem = 0;
-
- // Only the pre-computing information loaded and actual data does not loaded
- SInputColumnInfoData* pInput = &pCtx->input;
- int32_t type = pInput->pData[0]->info.type;
-
- SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
-
- // computing based on the true data block
- SColumnInfoData* pCol = pInput->pData[0];
-
- int32_t start = pInput->startRowIndex;
- int32_t numOfRows = pInput->numOfRows;
-
- switch (type) {
- case TSDB_DATA_TYPE_TINYINT: {
- LIST_AVG_N(pAvgRes->sum.isum, int8_t);
- break;
- }
- case TSDB_DATA_TYPE_SMALLINT: {
- LIST_AVG_N(pAvgRes->sum.isum, int16_t);
- break;
- }
- case TSDB_DATA_TYPE_INT: {
- LIST_AVG_N(pAvgRes->sum.isum, int32_t);
- break;
- }
- case TSDB_DATA_TYPE_BIGINT: {
- LIST_AVG_N(pAvgRes->sum.isum, int64_t);
- break;
- }
- case TSDB_DATA_TYPE_UTINYINT: {
- LIST_AVG_N(pAvgRes->sum.usum, uint8_t);
- break;
- }
- case TSDB_DATA_TYPE_USMALLINT: {
- LIST_AVG_N(pAvgRes->sum.usum, uint16_t);
- break;
- }
- case TSDB_DATA_TYPE_UINT: {
- LIST_AVG_N(pAvgRes->sum.usum, uint32_t);
- break;
- }
- case TSDB_DATA_TYPE_UBIGINT: {
- LIST_AVG_N(pAvgRes->sum.usum, uint64_t);
- break;
- }
- case TSDB_DATA_TYPE_FLOAT: {
- LIST_AVG_N(pAvgRes->sum.dsum, float);
- break;
- }
- case TSDB_DATA_TYPE_DOUBLE: {
- LIST_AVG_N(pAvgRes->sum.dsum, double);
- break;
- }
- default:
- break;
- }
-
- // data in the check operation are all null, not output
- SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
- return TSDB_CODE_SUCCESS;
-}
-
-int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
- SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
- SAvgRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
-
- SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
- SAvgRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
- int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type;
-
- if (IS_SIGNED_NUMERIC_TYPE(type)) {
- pDBuf->sum.isum += pSBuf->sum.isum;
- } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
- pDBuf->sum.usum += pSBuf->sum.usum;
- } else {
- pDBuf->sum.dsum += pSBuf->sum.dsum;
- }
- pDBuf->count += pSBuf->count;
-
- return TSDB_CODE_SUCCESS;
-}
-
-int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
- SInputColumnInfoData* pInput = &pCtx->input;
-
- SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
- int32_t type = pAvgRes->type;
-
- if (pAvgRes->count == 0) {
- // [ASAN] runtime error: division by zero
- GET_RES_INFO(pCtx)->numOfRes = 0;
- } else if (IS_SIGNED_NUMERIC_TYPE(type)) {
- pAvgRes->result = pAvgRes->sum.isum / ((double)pAvgRes->count);
- } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
- pAvgRes->result = pAvgRes->sum.usum / ((double)pAvgRes->count);
- } else {
- pAvgRes->result = pAvgRes->sum.dsum / ((double)pAvgRes->count);
- }
-
- // check for overflow
- if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) {
- GET_RES_INFO(pCtx)->numOfRes = 0;
- }
-
- return functionFinalize(pCtx, pBlock);
-}
-
-int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
- SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
- SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
- int32_t resultBytes = getAvgInfoSize();
- char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
-
- memcpy(varDataVal(res), pInfo, resultBytes);
- varDataSetLen(res, resultBytes);
-
- int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
- SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
-
- colDataAppend(pCol, pBlock->info.rows, res, false);
-
- taosMemoryFree(res);
- return pResInfo->numOfRes;
-}
-
EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
return FUNC_DATA_REQUIRED_SMA_LOAD;
}
@@ -1143,534 +752,6 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true;
}
-static STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock,
- const STupleKey* pKey);
-static int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
-static const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos);
-
-static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) {
- // the data is loaded, not only the block SMA value
- for (int32_t i = start; i < num + start; ++i) {
- char* p = colDataGetData(pCol, i);
- if (memcmp((void*)tval, p, pCol->info.bytes) == 0) {
- return i;
- }
- }
-
- // if reach here means real data of block SMA is not set in pCtx->input.
- return -1;
-}
-
-int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
- int32_t numOfElems = 0;
-
- SInputColumnInfoData* pInput = &pCtx->input;
- SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0];
-
- SColumnInfoData* pCol = pInput->pData[0];
- int32_t type = pCol->info.type;
-
- SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
- SMinmaxResInfo* pBuf = GET_ROWCELL_INTERBUF(pResInfo);
- pBuf->type = type;
-
- if (IS_NULL_TYPE(type)) {
- numOfElems = 0;
- goto _min_max_over;
- }
-
- // data in current data block are qualified to the query
- if (pInput->colDataAggIsSet) {
- numOfElems = pInput->numOfRows - pAgg->numOfNull;
- ASSERT(pInput->numOfRows == pInput->totalRows && numOfElems >= 0);
- if (numOfElems == 0) {
- return numOfElems;
- }
-
- void* tval = NULL;
- int16_t index = 0;
-
- if (isMinFunc) {
- tval = &pInput->pColumnDataAgg[0]->min;
- } else {
- tval = &pInput->pColumnDataAgg[0]->max;
- }
-
- if (!pBuf->assign) {
- if (type == TSDB_DATA_TYPE_FLOAT) {
- *(float*)&pBuf->v = GET_DOUBLE_VAL(tval);
- } else {
- pBuf->v = *(int64_t*)tval;
- }
- if (pCtx->subsidiaries.num > 0) {
- index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
- if (index >= 0) {
- pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
- }
- }
- } else {
- if (IS_SIGNED_NUMERIC_TYPE(type)) {
- int64_t prev = 0;
- GET_TYPED_DATA(prev, int64_t, type, &pBuf->v);
-
- int64_t val = GET_INT64_VAL(tval);
- if ((prev < val) ^ isMinFunc) {
- *(int64_t*)&pBuf->v = val;
- if (pCtx->subsidiaries.num > 0) {
- index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
- if (index >= 0) {
- pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
- }
- }
- }
- } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
- uint64_t prev = 0;
- GET_TYPED_DATA(prev, uint64_t, type, &pBuf->v);
-
- uint64_t val = GET_UINT64_VAL(tval);
- if ((prev < val) ^ isMinFunc) {
- *(uint64_t*)&pBuf->v = val;
- if (pCtx->subsidiaries.num > 0) {
- index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
- if (index >= 0) {
- pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
- }
- }
- }
- } else if (type == TSDB_DATA_TYPE_DOUBLE) {
- double prev = 0;
- GET_TYPED_DATA(prev, double, type, &pBuf->v);
-
- double val = GET_DOUBLE_VAL(tval);
- if ((prev < val) ^ isMinFunc) {
- *(double*)&pBuf->v = val;
- if (pCtx->subsidiaries.num > 0) {
- index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
- if (index >= 0) {
- pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
- }
- }
- }
- } else if (type == TSDB_DATA_TYPE_FLOAT) {
- float prev = 0;
- GET_TYPED_DATA(prev, float, type, &pBuf->v);
-
- float val = GET_DOUBLE_VAL(tval);
- if ((prev < val) ^ isMinFunc) {
- *(float*)&pBuf->v = val;
- }
-
- if (pCtx->subsidiaries.num > 0) {
- index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
- if (index >= 0) {
- pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
- }
- }
- }
- }
-
- pBuf->assign = true;
- return numOfElems;
- }
-
- int32_t start = pInput->startRowIndex;
- int32_t numOfRows = pInput->numOfRows;
-
- if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
- if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) {
- int8_t* pData = (int8_t*)pCol->pData;
- int8_t* val = (int8_t*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
- // ignore the equivalent data value
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- } else if (type == TSDB_DATA_TYPE_SMALLINT) {
- int16_t* pData = (int16_t*)pCol->pData;
- int16_t* val = (int16_t*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
- // ignore the equivalent data value
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- } else if (type == TSDB_DATA_TYPE_INT) {
- int32_t* pData = (int32_t*)pCol->pData;
- int32_t* val = (int32_t*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
- // ignore the equivalent data value
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- } else if (type == TSDB_DATA_TYPE_BIGINT) {
- int64_t* pData = (int64_t*)pCol->pData;
- int64_t* val = (int64_t*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
- // ignore the equivalent data value
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- }
- } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
- if (type == TSDB_DATA_TYPE_UTINYINT) {
- uint8_t* pData = (uint8_t*)pCol->pData;
- uint8_t* val = (uint8_t*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
- // ignore the equivalent data value
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- } else if (type == TSDB_DATA_TYPE_USMALLINT) {
- uint16_t* pData = (uint16_t*)pCol->pData;
- uint16_t* val = (uint16_t*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
- // ignore the equivalent data value
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- } else if (type == TSDB_DATA_TYPE_UINT) {
- uint32_t* pData = (uint32_t*)pCol->pData;
- uint32_t* val = (uint32_t*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
- // ignore the equivalent data value
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- } else if (type == TSDB_DATA_TYPE_UBIGINT) {
- uint64_t* pData = (uint64_t*)pCol->pData;
- uint64_t* val = (uint64_t*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
- // ignore the equivalent data value
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- }
- } else if (type == TSDB_DATA_TYPE_DOUBLE) {
- double* pData = (double*)pCol->pData;
- double* val = (double*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
- // ignore the equivalent data value
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- } else if (type == TSDB_DATA_TYPE_FLOAT) {
- float* pData = (float*)pCol->pData;
- float* val = (float*)&pBuf->v;
-
- for (int32_t i = start; i < start + numOfRows; ++i) {
- if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
- continue;
- }
-
- if (!pBuf->assign) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
- }
- pBuf->assign = true;
- } else {
-#if 0
- if ((*val) == pData[i]) {
- continue;
- }
-
- if ((*val < pData[i]) ^ isMinFunc) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
-#endif
- // NOTE: An faster version to avoid one additional comparison with FPU.
- if (isMinFunc) { // min
- if (*val > pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- } else { // max
- if (*val < pData[i]) {
- *val = pData[i];
- if (pCtx->subsidiaries.num > 0) {
- updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
- }
- }
- }
- }
-
- numOfElems += 1;
- }
- }
-
-_min_max_over:
- if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) {
- pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
- pBuf->nullTupleSaved = true;
- }
- return numOfElems;
-}
-
int32_t minFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElems = doMinMaxHelper(pCtx, 1);
SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1);
@@ -2499,7 +1580,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) {
// the first stage, only acquire the min/max value
if (pInfo->stage == 0) {
- if (pCtx->input.colDataAggIsSet) {
+ if (pCtx->input.colDataSMAIsSet) {
double tmin = 0.0, tmax = 0.0;
if (IS_SIGNED_NUMERIC_TYPE(type)) {
tmin = (double)GET_INT64_VAL(&pAgg->min);
@@ -2963,14 +2044,14 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
pInfo->bytes = pInputCol->info.bytes;
// All null data column, return directly.
- if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) {
+ if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) {
ASSERT(pInputCol->hasNull == true);
// save selectivity value for column consisted of all null values
firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo);
return 0;
}
- SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL;
+ SColumnDataAgg* pColAgg = (pInput->colDataSMAIsSet) ? pInput->pColumnDataAgg[0] : NULL;
TSKEY startKey = getRowPTs(pInput->pPTS, 0);
TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1);
@@ -3067,14 +2148,14 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
pInfo->bytes = bytes;
// All null data column, return directly.
- if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) {
+ if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) {
ASSERT(pInputCol->hasNull == true);
// save selectivity value for column consisted of all null values
firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo);
return 0;
}
- SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL;
+ SColumnDataAgg* pColAgg = (pInput->colDataSMAIsSet) ? pInput->pColumnDataAgg[0] : NULL;
TSKEY startKey = getRowPTs(pInput->pPTS, 0);
TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1);
@@ -3138,10 +2219,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
int32_t round = pInput->numOfRows >> 2;
int32_t reminder = pInput->numOfRows & 0x03;
- int32_t tick = 0;
- for (int32_t i = pInput->startRowIndex; tick < round; i += 4, tick += 1) {
- int64_t cts = pts[i];
- int32_t chosen = i;
+ for (int32_t i = pInput->startRowIndex, tick = 0; tick < round; i += 4, tick += 1) {
+ int64_t cts = pts[i];
+ int32_t chosen = i;
if (cts < pts[i + 1]) {
cts = pts[i + 1];
@@ -3165,18 +2245,18 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
}
}
- for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
- if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) {
- char* data = colDataGetData(pInputCol, i);
- doSaveCurrentVal(pCtx, i, pts[i], type, data);
- pResInfo->numOfRes = 1;
- }
- }
- } else {
- for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
- if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
- continue;
+ for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
+ if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) {
+ char* data = colDataGetData(pInputCol, i);
+ doSaveCurrentVal(pCtx, i, pts[i], type, data);
+ pResInfo->numOfRes = 1;
+ }
}
+ } else {
+ for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
+ if (colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
+ continue;
+ }
numOfElems++;
@@ -3895,7 +2975,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf
return TSDB_CODE_SUCCESS;
}
-static int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
+int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf);
doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos);
return TSDB_CODE_SUCCESS;
@@ -3915,7 +2995,7 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo
}
}
-static const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) {
+const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) {
return doLoadTupleData(&pCtx->saveHandle, pPos);
}
@@ -4036,7 +3116,7 @@ int32_t spreadFunction(SqlFunctionCtx* pCtx) {
SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
- if (pInput->colDataAggIsSet) {
+ if (pInput->colDataSMAIsSet) {
numOfElems = pInput->numOfRows - pAgg->numOfNull;
if (numOfElems == 0) {
goto _spread_over;
@@ -4211,7 +3291,7 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) {
goto _elapsed_over;
}
- if (pInput->colDataAggIsSet) {
+ if (pInput->colDataSMAIsSet) {
if (pInfo->min == TSKEY_MAX) {
pInfo->min = GET_INT64_VAL(&pAgg->min);
pInfo->max = GET_INT64_VAL(&pAgg->max);
diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c
new file mode 100644
index 0000000000..7d018a8dc7
--- /dev/null
+++ b/source/libs/function/src/detail/tavgfunction.c
@@ -0,0 +1,751 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "builtinsimpl.h"
+#include "function.h"
+#include "tdatablock.h"
+#include "tfunctionInt.h"
+#include "tglobal.h"
+
+#define SET_VAL(_info, numOfElem, res) \
+ do { \
+ if ((numOfElem) <= 0) { \
+ break; \
+ } \
+ (_info)->numOfRes = (res); \
+ } while (0)
+
+#define LIST_AVG_N(sumT, T) \
+ do { \
+ T* plist = (T*)pCol->pData; \
+ for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { \
+ if (colDataIsNull_f(pCol->nullbitmap, i)) { \
+ continue; \
+ } \
+ \
+ numOfElem += 1; \
+ pAvgRes->count -= 1; \
+ sumT -= plist[i]; \
+ } \
+ } while (0)
+
+typedef struct SAvgRes {
+ double result;
+ SSumRes sum;
+ int64_t count;
+ int16_t type; // store the original input type, used in merge function
+} SAvgRes;
+
+static void floatVectorSumAVX(const float* plist, int32_t numOfRows, SAvgRes* pRes) {
+ const int32_t bitWidth = 256;
+
+#if __AVX__
+ // find the start position that are aligned to 32bytes address in memory
+ int32_t width = (bitWidth>>3u) / sizeof(float);
+
+ int32_t remainder = numOfRows % width;
+ int32_t rounds = numOfRows / width;
+
+ const float* p = plist;
+
+ __m256 val;
+ __m256 sum = _mm256_setzero_ps();
+
+ for (int32_t i = 0; i < rounds; ++i) {
+ val = _mm256_loadu_ps(p);
+ sum = _mm256_add_ps(sum, val);
+ p += width;
+ }
+
+ // let sum up the final results
+ const float* q = (const float*)∑
+ pRes->sum.dsum += q[0] + q[1] + q[2] + q[3] + q[4] + q[5] + q[6] + q[7];
+
+ int32_t startIndex = rounds * width;
+ for (int32_t j = 0; j < remainder; ++j) {
+ pRes->sum.dsum += plist[j + startIndex];
+ }
+#endif
+}
+
+static void doubleVectorSumAVX(const double* plist, int32_t numOfRows, SAvgRes* pRes) {
+ const int32_t bitWidth = 256;
+
+#if __AVX__
+ // find the start position that are aligned to 32bytes address in memory
+ int32_t width = (bitWidth>>3u) / sizeof(int64_t);
+
+ int32_t remainder = numOfRows % width;
+ int32_t rounds = numOfRows / width;
+
+ const double* p = plist;
+
+ __m256d val;
+ __m256d sum = _mm256_setzero_pd();
+
+ for (int32_t i = 0; i < rounds; ++i) {
+ val = _mm256_loadu_pd(p);
+ sum = _mm256_add_pd(sum, val);
+ p += width;
+ }
+
+ // let sum up the final results
+ const double* q = (const double*)∑
+ pRes->sum.dsum += q[0] + q[1] + q[2] + q[3];
+
+ int32_t startIndex = rounds * width;
+ for (int32_t j = 0; j < remainder; ++j) {
+ pRes->sum.dsum += plist[j + startIndex];
+ }
+#endif
+}
+
+static void i8VectorSumAVX2(const int8_t* plist, int32_t numOfRows, int32_t type, SAvgRes* pRes) {
+ const int32_t bitWidth = 256;
+
+#if __AVX2__
+ // find the start position that are aligned to 32bytes address in memory
+ int32_t width = (bitWidth>>3u) / sizeof(int64_t);
+
+ int32_t remainder = numOfRows % width;
+ int32_t rounds = numOfRows / width;
+
+ __m256i sum = _mm256_setzero_si256();
+
+ if (type == TSDB_DATA_TYPE_TINYINT) {
+ const int8_t* p = plist;
+
+ for (int32_t i = 0; i < rounds; ++i) {
+ __m128i val = _mm_lddqu_si128((__m128i*)p);
+ __m256i extVal = _mm256_cvtepi8_epi64(val); // only four items will be converted into __m256i
+ sum = _mm256_add_epi64(sum, extVal);
+ p += width;
+ }
+ } else {
+ const uint8_t* p = (const uint8_t*)plist;
+
+ for(int32_t i = 0; i < rounds; ++i) {
+ __m128i val = _mm_lddqu_si128((__m128i*)p);
+ __m256i extVal = _mm256_cvtepu8_epi64(val); // only four items will be converted into __m256i
+ sum = _mm256_add_epi64(sum, extVal);
+ p += width;
+ }
+ }
+
+ // let sum up the final results
+ const int64_t* q = (const int64_t*)∑
+ pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+
+ int32_t startIndex = rounds * width;
+ for (int32_t j = 0; j < remainder; ++j) {
+ pRes->sum.isum += plist[j + startIndex];
+ }
+#endif
+}
+
+static void i16VectorSumAVX2(const int16_t* plist, int32_t numOfRows, int32_t type, SAvgRes* pRes) {
+ const int32_t bitWidth = 256;
+
+#if __AVX2__
+ // find the start position that are aligned to 32bytes address in memory
+ int32_t width = (bitWidth>>3u) / sizeof(int64_t);
+
+ int32_t remainder = numOfRows % width;
+ int32_t rounds = numOfRows / width;
+
+ __m256i sum = _mm256_setzero_si256();
+
+ if (type == TSDB_DATA_TYPE_SMALLINT) {
+ const int16_t* p = plist;
+
+ for (int32_t i = 0; i < rounds; ++i) {
+ __m128i val = _mm_lddqu_si128((__m128i*)p);
+ __m256i extVal = _mm256_cvtepi16_epi64(val); // only four items will be converted into __m256i
+ sum = _mm256_add_epi64(sum, extVal);
+ p += width;
+ }
+ } else {
+ const uint8_t* p = (const uint8_t*)plist;
+
+ for(int32_t i = 0; i < rounds; ++i) {
+ __m128i val = _mm_lddqu_si128((__m128i*)p);
+ __m256i extVal = _mm256_cvtepu16_epi64(val); // only four items will be converted into __m256i
+ sum = _mm256_add_epi64(sum, extVal);
+ p += width;
+ }
+ }
+
+ // let sum up the final results
+ const int64_t* q = (const int64_t*)∑
+ pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+
+ int32_t startIndex = rounds * width;
+ for (int32_t j = 0; j < remainder; ++j) {
+ pRes->sum.isum += plist[j + startIndex];
+ }
+#endif
+}
+
+static void i32VectorSumAVX2(const int32_t* plist, int32_t numOfRows, int32_t type, SAvgRes* pRes) {
+ const int32_t bitWidth = 256;
+
+#if __AVX2__
+ // find the start position that are aligned to 32bytes address in memory
+ int32_t width = (bitWidth>>3u) / sizeof(int64_t);
+
+ int32_t remainder = numOfRows % width;
+ int32_t rounds = numOfRows / width;
+
+ __m256i sum = _mm256_setzero_si256();
+
+ if (type == TSDB_DATA_TYPE_INT) {
+ const int32_t* p = plist;
+
+ for (int32_t i = 0; i < rounds; ++i) {
+ __m128i val = _mm_lddqu_si128((__m128i*)p);
+ __m256i extVal = _mm256_cvtepi32_epi64(val); // only four items will be converted into __m256i
+ sum = _mm256_add_epi64(sum, extVal);
+ p += width;
+ }
+ } else {
+ const uint32_t* p = (const uint32_t*)plist;
+
+ for(int32_t i = 0; i < rounds; ++i) {
+ __m128i val = _mm_lddqu_si128((__m128i*)p);
+ __m256i extVal = _mm256_cvtepu32_epi64(val); // only four items will be converted into __m256i
+ sum = _mm256_add_epi64(sum, extVal);
+ p += width;
+ }
+ }
+
+ // let sum up the final results
+ const int64_t* q = (const int64_t*)∑
+ pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+
+ int32_t startIndex = rounds * width;
+ for (int32_t j = 0; j < remainder; ++j) {
+ pRes->sum.isum += plist[j + startIndex];
+ }
+#endif
+}
+
+static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* pRes) {
+ const int32_t bitWidth = 256;
+
+#if __AVX2__
+ // find the start position that are aligned to 32bytes address in memory
+ int32_t width = (bitWidth>>3u) / sizeof(int64_t);
+
+ int32_t remainder = numOfRows % width;
+ int32_t rounds = numOfRows / width;
+
+ __m256i sum = _mm256_setzero_si256();
+
+ const int64_t* p = plist;
+
+ for (int32_t i = 0; i < rounds; ++i) {
+ __m256i val = _mm256_lddqu_si256((__m256i*)p);
+ sum = _mm256_add_epi64(sum, val);
+ p += width;
+ }
+
+ // let sum up the final results
+ const int64_t* q = (const int64_t*)∑
+ pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+
+ int32_t startIndex = rounds * width;
+ for (int32_t j = 0; j < remainder; ++j) {
+ pRes->sum.isum += plist[j + startIndex];
+ }
+#endif
+}
+
+int32_t getAvgInfoSize() { return (int32_t)sizeof(SAvgRes); }
+
+bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
+ pEnv->calcMemSize = sizeof(SAvgRes);
+ return true;
+}
+
+bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
+ if (!functionSetup(pCtx, pResultInfo)) {
+ return false;
+ }
+
+ SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo);
+ memset(pRes, 0, sizeof(SAvgRes));
+ return true;
+}
+
+static int32_t calculateAvgBySMAInfo(SAvgRes* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg) {
+ int32_t numOfElem = numOfRows - pAgg->numOfNull;
+ ASSERT(numOfElem >= 0);
+
+ pRes->count += numOfElem;
+ if (IS_SIGNED_NUMERIC_TYPE(type)) {
+ pRes->sum.isum += pAgg->sum;
+ } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+ pRes->sum.usum += pAgg->sum;
+ } else if (IS_FLOAT_TYPE(type)) {
+ pRes->sum.dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum));
+ }
+
+ return numOfElem;
+}
+
+static int32_t doAddNumericVector(SColumnInfoData* pCol, int32_t type, SInputColumnInfoData *pInput, SAvgRes* pRes) {
+ int32_t start = pInput->startRowIndex;
+ int32_t numOfRows = pInput->numOfRows;
+ int32_t numOfElems = 0;
+
+ switch (type) {
+ case TSDB_DATA_TYPE_TINYINT: {
+ int8_t* plist = (int8_t*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.isum += plist[i];
+ }
+
+ break;
+ }
+
+ case TSDB_DATA_TYPE_SMALLINT: {
+ int16_t* plist = (int16_t*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.isum += plist[i];
+ }
+ break;
+ }
+
+ case TSDB_DATA_TYPE_INT: {
+ int32_t* plist = (int32_t*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.isum += plist[i];
+ }
+
+ break;
+ }
+
+ case TSDB_DATA_TYPE_BIGINT: {
+ int64_t* plist = (int64_t*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.isum += plist[i];
+ }
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UTINYINT: {
+ uint8_t* plist = (uint8_t*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.usum += plist[i];
+ }
+
+ break;
+ }
+
+ case TSDB_DATA_TYPE_USMALLINT: {
+ uint16_t* plist = (uint16_t*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.usum += plist[i];
+ }
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UINT: {
+ uint32_t* plist = (uint32_t*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.usum += plist[i];
+ }
+
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UBIGINT: {
+ uint64_t* plist = (uint64_t*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.usum += plist[i];
+ }
+ break;
+ }
+
+ case TSDB_DATA_TYPE_FLOAT: {
+ float* plist = (float*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.dsum += plist[i];
+ }
+ break;
+ }
+
+ case TSDB_DATA_TYPE_DOUBLE: {
+ double* plist = (double*)pCol->pData;
+ for (int32_t i = start; i < numOfRows + start; ++i) {
+ if (colDataIsNull_f(pCol->nullbitmap, i)) {
+ continue;
+ }
+
+ numOfElems += 1;
+ pRes->count += 1;
+ pRes->sum.dsum += plist[i];
+ }
+ break;
+ }
+
+ default:
+ break;
+ }
+
+ return numOfElems;
+}
+
+int32_t avgFunction(SqlFunctionCtx* pCtx) {
+ int32_t numOfElem = 0;
+ const int32_t THRESHOLD_SIZE = 8;
+
+ SInputColumnInfoData* pInput = &pCtx->input;
+ SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0];
+ int32_t type = pInput->pData[0]->info.type;
+
+ SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+ pAvgRes->type = type;
+
+ // computing based on the true data block
+ SColumnInfoData* pCol = pInput->pData[0];
+
+ int32_t start = pInput->startRowIndex;
+ int32_t numOfRows = pInput->numOfRows;
+
+ if (IS_NULL_TYPE(type)) {
+ goto _over;
+ }
+
+ if (pInput->colDataSMAIsSet) { // try to use SMA if available
+ numOfElem = calculateAvgBySMAInfo(pAvgRes, numOfRows, type, pAgg);
+ } else if (!pCol->hasNull) { // try to employ the simd instructions to speed up the loop
+ numOfElem = pInput->numOfRows;
+ pAvgRes->count += pInput->numOfRows;
+
+ bool simdAvailable = tsAVXEnable && tsSIMDEnable && (numOfRows > THRESHOLD_SIZE);
+
+ switch(type) {
+ case TSDB_DATA_TYPE_UTINYINT:
+ case TSDB_DATA_TYPE_TINYINT: {
+ const int8_t* plist = (const int8_t*) pCol->pData;
+
+ // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop
+ if (simdAvailable) {
+ i8VectorSumAVX2(plist, numOfRows, type, pAvgRes);
+ } else {
+ for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+ pAvgRes->sum.usum += plist[i];
+ }
+ }
+ break;
+ }
+
+ case TSDB_DATA_TYPE_USMALLINT:
+ case TSDB_DATA_TYPE_SMALLINT: {
+ const int16_t* plist = (const int16_t*)pCol->pData;
+
+ // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop
+ if (simdAvailable) {
+ i16VectorSumAVX2(plist, numOfRows, type, pAvgRes);
+ } else {
+ for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+ pAvgRes->sum.isum += plist[i];
+ }
+ }
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UINT:
+ case TSDB_DATA_TYPE_INT: {
+ const int32_t* plist = (const int32_t*) pCol->pData;
+
+ // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop
+ if (simdAvailable) {
+ i32VectorSumAVX2(plist, numOfRows, type, pAvgRes);
+ } else {
+ for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+ pAvgRes->sum.isum += plist[i];
+ }
+ }
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UBIGINT:
+ case TSDB_DATA_TYPE_BIGINT: {
+ const int64_t* plist = (const int64_t*) pCol->pData;
+
+ // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop
+ if (simdAvailable) {
+ i64VectorSumAVX2(plist, numOfRows, pAvgRes);
+ } else {
+ for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+ pAvgRes->sum.isum += plist[i];
+ }
+ }
+ break;
+ }
+
+ case TSDB_DATA_TYPE_FLOAT: {
+ const float* plist = (const float*) pCol->pData;
+
+ // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop
+ if (simdAvailable) {
+ floatVectorSumAVX(plist, numOfRows, pAvgRes);
+ } else {
+ for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+ pAvgRes->sum.dsum += plist[i];
+ }
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_DOUBLE: {
+ const double* plist = (const double*)pCol->pData;
+
+ // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop
+ if (simdAvailable) {
+ doubleVectorSumAVX(plist, numOfRows, pAvgRes);
+ } else {
+ for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+ pAvgRes->sum.dsum += plist[i];
+ }
+ }
+ break;
+ }
+ default:
+ ASSERT(0);
+ }
+ } else {
+ numOfElem = doAddNumericVector(pCol, type, pInput, pAvgRes);
+ }
+
+_over:
+ // data in the check operation are all null, not output
+ SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
+ return TSDB_CODE_SUCCESS;
+}
+
+static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) {
+ pOutput->type = pInput->type;
+ if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) {
+ pOutput->sum.isum += pInput->sum.isum;
+ } else if (IS_UNSIGNED_NUMERIC_TYPE(pOutput->type)) {
+ pOutput->sum.usum += pInput->sum.usum;
+ } else {
+ pOutput->sum.dsum += pInput->sum.dsum;
+ }
+
+ pOutput->count += pInput->count;
+}
+
+int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) {
+ SInputColumnInfoData* pInput = &pCtx->input;
+ SColumnInfoData* pCol = pInput->pData[0];
+ ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
+
+ SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+ int32_t start = pInput->startRowIndex;
+
+ for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
+ char* data = colDataGetData(pCol, i);
+ SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data);
+ avgTransferInfo(pInputInfo, pInfo);
+ }
+
+ SET_VAL(GET_RES_INFO(pCtx), 1, 1);
+
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t avgInvertFunction(SqlFunctionCtx* pCtx) {
+ int32_t numOfElem = 0;
+
+ // Only the pre-computing information loaded and actual data does not loaded
+ SInputColumnInfoData* pInput = &pCtx->input;
+ SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+ // computing based on the true data block
+ SColumnInfoData* pCol = pInput->pData[0];
+
+ int32_t start = pInput->startRowIndex;
+ int32_t numOfRows = pInput->numOfRows;
+
+ switch (pCol->info.type) {
+ case TSDB_DATA_TYPE_TINYINT: {
+ LIST_AVG_N(pAvgRes->sum.isum, int8_t);
+ break;
+ }
+ case TSDB_DATA_TYPE_SMALLINT: {
+ LIST_AVG_N(pAvgRes->sum.isum, int16_t);
+ break;
+ }
+ case TSDB_DATA_TYPE_INT: {
+ LIST_AVG_N(pAvgRes->sum.isum, int32_t);
+ break;
+ }
+ case TSDB_DATA_TYPE_BIGINT: {
+ LIST_AVG_N(pAvgRes->sum.isum, int64_t);
+ break;
+ }
+ case TSDB_DATA_TYPE_UTINYINT: {
+ LIST_AVG_N(pAvgRes->sum.usum, uint8_t);
+ break;
+ }
+ case TSDB_DATA_TYPE_USMALLINT: {
+ LIST_AVG_N(pAvgRes->sum.usum, uint16_t);
+ break;
+ }
+ case TSDB_DATA_TYPE_UINT: {
+ LIST_AVG_N(pAvgRes->sum.usum, uint32_t);
+ break;
+ }
+ case TSDB_DATA_TYPE_UBIGINT: {
+ LIST_AVG_N(pAvgRes->sum.usum, uint64_t);
+ break;
+ }
+ case TSDB_DATA_TYPE_FLOAT: {
+ LIST_AVG_N(pAvgRes->sum.dsum, float);
+ break;
+ }
+ case TSDB_DATA_TYPE_DOUBLE: {
+ LIST_AVG_N(pAvgRes->sum.dsum, double);
+ break;
+ }
+ default:
+ break;
+ }
+
+ // data in the check operation are all null, not output
+ SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
+ SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
+ SAvgRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
+
+ SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
+ SAvgRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
+ int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type;
+
+ if (IS_SIGNED_NUMERIC_TYPE(type)) {
+ pDBuf->sum.isum += pSBuf->sum.isum;
+ } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+ pDBuf->sum.usum += pSBuf->sum.usum;
+ } else {
+ pDBuf->sum.dsum += pSBuf->sum.dsum;
+ }
+ pDBuf->count += pSBuf->count;
+
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+ SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx);
+
+ SAvgRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo);
+ int32_t type = pRes->type;
+
+ if (pRes->count > 0) {
+ if (IS_SIGNED_NUMERIC_TYPE(type)) {
+ pRes->result = pRes->sum.isum / ((double)pRes->count);
+ } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+ pRes->result = pRes->sum.usum / ((double)pRes->count);
+ } else {
+ pRes->result = pRes->sum.dsum / ((double)pRes->count);
+ }
+ }
+
+ if (pRes->count == 0 || isinf(pRes->result) || isnan(pRes->result)) {
+ pEntryInfo->numOfRes = 0;
+ } else {
+ pEntryInfo->numOfRes = 1;
+ }
+
+ return functionFinalize(pCtx, pBlock);
+}
+
+int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+ SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+ SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+ int32_t resultBytes = getAvgInfoSize();
+ char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
+
+ memcpy(varDataVal(res), pInfo, resultBytes);
+ varDataSetLen(res, resultBytes);
+
+ int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
+ SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+ colDataAppend(pCol, pBlock->info.rows, res, false);
+
+ taosMemoryFree(res);
+ return pResInfo->numOfRes;
+}
\ No newline at end of file
diff --git a/source/libs/function/src/detail/tminmax.c b/source/libs/function/src/detail/tminmax.c
new file mode 100644
index 0000000000..b2cb36cba0
--- /dev/null
+++ b/source/libs/function/src/detail/tminmax.c
@@ -0,0 +1,891 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "builtinsimpl.h"
+#include "function.h"
+#include "tdatablock.h"
+#include "tfunctionInt.h"
+#include "tglobal.h"
+
+#define __COMPARE_ACQUIRED_MAX(i, end, bm, _data, ctx, val, pos) \
+ for (; i < (end); ++i) { \
+ if (colDataIsNull_f(bm, i)) { \
+ continue; \
+ } \
+ \
+ if ((val) < (_data)[i]) { \
+ (val) = (_data)[i]; \
+ if ((ctx)->subsidiaries.num > 0) { \
+ updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \
+ } \
+ } \
+ }
+
+#define __COMPARE_ACQUIRED_MIN(i, end, bm, _data, ctx, val, pos) \
+ for (; i < (end); ++i) { \
+ if (colDataIsNull_f(bm, i)) { \
+ continue; \
+ } \
+ \
+ if ((val) > (_data)[i]) { \
+ (val) = (_data)[i]; \
+ if ((ctx)->subsidiaries.num > 0) { \
+ updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \
+ } \
+ } \
+ }
+
+#define __COMPARE_EXTRACT_MIN(start, end, val, _data) \
+ for (int32_t i = (start); i < (end); ++i) { \
+ if ((val) > (_data)[i]) { \
+ (val) = (_data)[i]; \
+ } \
+ }
+
+#define __COMPARE_EXTRACT_MAX(start, end, val, _data) \
+ for (int32_t i = (start); i < (end); ++i) { \
+ if ((val) < (_data)[i]) { \
+ (val) = (_data)[i]; \
+ } \
+ }
+
+static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder, int32_t* rounds, int32_t* width) {
+ const int32_t bitWidth = 256;
+
+ *width = (bitWidth>>3u) / bytes;
+ *remainder = numOfRows % (*width);
+ *rounds = numOfRows / (*width);
+}
+
+#define EXTRACT_MAX_VAL(_first, _sec, _width, _remain, _v) \
+ (_v) = TMAX((_first)[0], (_first)[1]); \
+ for (int32_t k = 1; k < (_width); ++k) { \
+ (_v) = TMAX((_v), (_first)[k]); \
+ } \
+ \
+ for (int32_t j = 0; j < (_remain); ++j) { \
+ if ((_v) < (_sec)[j]) { \
+ (_v) = (_sec)[j]; \
+ } \
+ }
+
+#define EXTRACT_MIN_VAL(_first, _sec, _width, _remain, _v) \
+ (_v) = TMIN((_first)[0], (_first)[1]); \
+ for (int32_t k = 1; k < (_width); ++k) { \
+ (_v) = TMIN((_v), (_first)[k]); \
+ } \
+ \
+ for (int32_t j = 0; j < (_remain); ++j) { \
+ if ((_v) > (_sec)[j]) { \
+ (_v) = (_sec)[j]; \
+ } \
+ }
+
+
+static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal) {
+ int8_t v = 0;
+ const int8_t* p = pData;
+
+ int32_t width, remain, rounds;
+ calculateRounds(numOfRows, sizeof(int8_t), &remain, &rounds, &width);
+
+#if __AVX2__
+ __m256i next;
+ __m256i initVal = _mm256_lddqu_si256((__m256i*)p);
+ p += width;
+
+ if (!isMinFunc) { // max function
+ if (signVal) {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_max_epi8(initVal, next);
+ p += width;
+ }
+
+ const int8_t* q = (const int8_t*)&initVal;
+ EXTRACT_MAX_VAL(q, p, width, remain, v)
+ } else { // unsigned value
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_max_epu8(initVal, next);
+ p += width;
+ }
+
+ const uint8_t* q = (const uint8_t*)&initVal;
+ EXTRACT_MAX_VAL(q, p, width, remain, v)
+ }
+
+ } else { // min function
+ if (signVal) {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_min_epi8(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const int8_t* q = (const int8_t*)&initVal;
+ EXTRACT_MIN_VAL(q, p, width, remain, v)
+ } else {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_min_epu8(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const uint8_t* q = (const uint8_t*)&initVal;
+ EXTRACT_MIN_VAL(q, p, width, remain, v)
+ }
+ }
+#endif
+
+ return v;
+}
+
+static int16_t i16VectorCmpAVX2(const int16_t* pData, int32_t numOfRows, bool isMinFunc, bool signVal) {
+ int16_t v = 0;
+ const int16_t* p = pData;
+
+ int32_t width, remain, rounds;
+ calculateRounds(numOfRows, sizeof(int16_t), &remain, &rounds, &width);
+
+#if __AVX2__
+ __m256i next;
+ __m256i initVal = _mm256_lddqu_si256((__m256i*)p);
+ p += width;
+
+ if (!isMinFunc) { // max function
+ if (signVal) {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_max_epi16(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const int16_t* q = (const int16_t*)&initVal;
+ EXTRACT_MAX_VAL(q, p, width, remain, v)
+ } else {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_max_epu16(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const uint16_t* q = (const uint16_t*)&initVal;
+ EXTRACT_MAX_VAL(q, p, width, remain, v)
+ }
+
+ } else { // min function
+ if (signVal) {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_min_epi16(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const int16_t* q = (const int16_t*)&initVal;
+ EXTRACT_MIN_VAL(q, p, width, remain, v)
+ } else {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_min_epi16(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const uint16_t* q = (const uint16_t*)&initVal;
+ EXTRACT_MIN_VAL(q, p, width, remain, v)
+ }
+ }
+#endif
+
+ return v;
+}
+
+static int32_t i32VectorCmpAVX2(const int32_t* pData, int32_t numOfRows, bool isMinFunc, bool signVal) {
+ int32_t v = 0;
+ const int32_t* p = pData;
+
+ int32_t width, remain, rounds;
+ calculateRounds(numOfRows, sizeof(int32_t), &remain, &rounds, &width);
+
+#if __AVX2__
+ __m256i next;
+ __m256i initVal = _mm256_lddqu_si256((__m256i*)p);
+ p += width;
+
+ if (!isMinFunc) { // max function
+ if (signVal) {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_max_epi32(initVal, next);
+ p += width;
+ }
+
+ // let compare the final results
+ const int32_t* q = (const int32_t*)&initVal;
+ EXTRACT_MAX_VAL(q, p, width, remain, v)
+ } else { // unsigned value
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_max_epi32(initVal, next);
+ p += width;
+ }
+
+ // let compare the final results
+ const uint32_t* q = (const uint32_t*)&initVal;
+ EXTRACT_MAX_VAL(q, p, width, remain, v)
+ }
+ } else { // min function
+ if (signVal) {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_min_epi32(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const int32_t* q = (const int32_t*)&initVal;
+ EXTRACT_MIN_VAL(q, p, width, remain, v)
+ } else {
+ for (int32_t i = 0; i < rounds; ++i) {
+ next = _mm256_lddqu_si256((__m256i*)p);
+ initVal = _mm256_min_epu32(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const uint32_t* q = (const uint32_t*)&initVal;
+ EXTRACT_MIN_VAL(q, p, width, remain, v)
+ }
+ }
+#endif
+
+ return v;
+}
+
+static float floatVectorCmpAVX(const float* pData, int32_t numOfRows, bool isMinFunc) {
+ float v = 0;
+ const float* p = pData;
+
+ int32_t width, remain, rounds;
+ calculateRounds(numOfRows, sizeof(float), &remain, &rounds, &width);
+
+#if __AVX__
+
+ __m256 next;
+ __m256 initVal = _mm256_loadu_ps(p);
+ p += width;
+
+ if (!isMinFunc) { // max function
+ for (int32_t i = 1; i < rounds; ++i) {
+ next = _mm256_loadu_ps(p);
+ initVal = _mm256_max_ps(initVal, next);
+ p += width;
+ }
+
+ const float* q = (const float*)&initVal;
+ EXTRACT_MAX_VAL(q, p, width, remain, v)
+ } else { // min function
+ for (int32_t i = 1; i < rounds; ++i) {
+ next = _mm256_loadu_ps(p);
+ initVal = _mm256_min_ps(initVal, next);
+ p += width;
+ }
+
+ const float* q = (const float*)&initVal;
+ EXTRACT_MIN_VAL(q, p, width, remain, v)
+ }
+#endif
+
+ return v;
+}
+
+static double doubleVectorCmpAVX(const double* pData, int32_t numOfRows, bool isMinFunc) {
+ double v = 0;
+ const double* p = pData;
+
+ int32_t width, remain, rounds;
+ calculateRounds(numOfRows, sizeof(double), &remain, &rounds, &width);
+
+#if __AVX__
+
+ __m256d next;
+ __m256d initVal = _mm256_loadu_pd(p);
+ p += width;
+
+ if (!isMinFunc) { // max function
+ for (int32_t i = 1; i < rounds; ++i) {
+ next = _mm256_loadu_pd(p);
+ initVal = _mm256_max_pd(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const double* q = (const double*)&initVal;
+ EXTRACT_MAX_VAL(q, p, width, remain, v)
+ } else { // min function
+ for (int32_t i = 1; i < rounds; ++i) {
+ next = _mm256_loadu_pd(p);
+ initVal = _mm256_min_pd(initVal, next);
+ p += width;
+ }
+
+ // let sum up the final results
+ const double* q = (const double*)&initVal;
+ EXTRACT_MIN_VAL(q, p, width, remain, v)
+ }
+#endif
+
+ return v;
+}
+
+static int32_t findFirstValPosition(const SColumnInfoData* pCol, int32_t start, int32_t numOfRows) {
+ int32_t i = start;
+
+ while (i < (start + numOfRows) && (colDataIsNull_f(pCol->nullbitmap, i) == true)) {
+ i += 1;
+ }
+
+ return i;
+}
+
+static void handleInt8Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc,
+ bool signVal) {
+ // AVX2 version to speedup the loop
+ if (tsAVX2Enable && tsSIMDEnable) {
+ pBuf->v = i8VectorCmpAVX2(data, numOfRows, isMinFunc, signVal);
+ } else {
+ if (!pBuf->assign) {
+ pBuf->v = ((int8_t*)data)[0];
+ }
+
+ if (signVal) {
+ const int8_t* p = (const int8_t*)data;
+ int8_t* v = (int8_t*)&pBuf->v;
+
+ if (isMinFunc) {
+ __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p);
+ } else {
+ __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p);
+ }
+ } else {
+ const uint8_t* p = (const uint8_t*)data;
+ uint8_t* v = (uint8_t*)&pBuf->v;
+
+ if (isMinFunc) {
+ __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p);
+ } else {
+ __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p);
+ }
+ }
+ }
+
+ pBuf->assign = true;
+}
+
+static void handleInt16Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc,
+ bool signVal) {
+ // AVX2 version to speedup the loop
+ if (tsAVX2Enable && tsSIMDEnable) {
+ pBuf->v = i16VectorCmpAVX2(data, numOfRows, isMinFunc, signVal);
+ } else {
+ if (!pBuf->assign) {
+ pBuf->v = ((int16_t*)data)[0];
+ }
+
+ if (signVal) {
+ const int16_t* p = (const int16_t*)data;
+ int16_t* v = (int16_t*)&pBuf->v;
+
+ if (isMinFunc) {
+ __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p);
+ } else {
+ __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p);
+ }
+ } else {
+ const uint16_t* p = (const uint16_t*)data;
+ uint16_t* v = (uint16_t*)&pBuf->v;
+
+ if (isMinFunc) {
+ __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p);
+ } else {
+ __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p);
+ }
+ }
+ }
+
+ pBuf->assign = true;
+}
+
+static void handleInt32Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc,
+ bool signVal) {
+ // AVX2 version to speedup the loop
+ if (tsAVX2Enable && tsSIMDEnable) {
+ pBuf->v = i32VectorCmpAVX2(data, numOfRows, isMinFunc, signVal);
+ } else {
+ if (!pBuf->assign) {
+ pBuf->v = ((int32_t*)data)[0];
+ }
+
+ if (signVal) {
+ const int32_t* p = (const int32_t*)data;
+ int32_t* v = (int32_t*)&pBuf->v;
+
+ if (isMinFunc) {
+ __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p);
+ } else {
+ __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p);
+ }
+ } else {
+ const uint32_t* p = (const uint32_t*)data;
+ uint32_t* v = (uint32_t*)&pBuf->v;
+
+ if (isMinFunc) {
+ __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p);
+ } else {
+ __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p);
+ }
+ }
+ }
+
+ pBuf->assign = true;
+}
+
+static void handleInt64Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc,
+ bool signVal) {
+ if (!pBuf->assign) {
+ pBuf->v = ((int64_t*)data)[0];
+ }
+
+ if (signVal) {
+ const int64_t* p = (const int64_t*)data;
+ int64_t* v = &pBuf->v;
+
+ if (isMinFunc) {
+ __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p);
+ } else {
+ __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p);
+ }
+ } else {
+ const uint64_t* p = (const uint64_t*)data;
+ uint64_t* v = (uint64_t*)&pBuf->v;
+
+ if (isMinFunc) {
+ __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p);
+ } else {
+ __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p);
+ }
+ }
+}
+
+static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc) {
+ float* pData = (float*)pCol->pData;
+ float* val = (float*)&pBuf->v;
+
+ // AVX version to speedup the loop
+ if (tsAVXEnable && tsSIMDEnable) {
+ *val = floatVectorCmpAVX(pData, numOfRows, isMinFunc);
+ } else {
+ if (!pBuf->assign) {
+ *val = pData[0];
+ }
+
+ if (isMinFunc) { // min
+ for (int32_t i = start; i < start + numOfRows; ++i) {
+ if (*val > pData[i]) {
+ *val = pData[i];
+ }
+ }
+ } else { // max
+ for (int32_t i = start; i < start + numOfRows; ++i) {
+ if (*val < pData[i]) {
+ *val = pData[i];
+ }
+ }
+ }
+ }
+
+ pBuf->assign = true;
+}
+
+static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc) {
+ double* pData = (double*)pCol->pData;
+ double* val = (double*)&pBuf->v;
+
+ // AVX version to speedup the loop
+ if (tsAVXEnable && tsSIMDEnable) {
+ *val = (double)doubleVectorCmpAVX(pData, numOfRows, isMinFunc);
+ } else {
+ if (!pBuf->assign) {
+ *val = pData[0];
+ }
+
+ if (isMinFunc) { // min
+ for (int32_t i = start; i < start + numOfRows; ++i) {
+ if (*val > pData[i]) {
+ *val = pData[i];
+ }
+ }
+ } else { // max
+ for (int32_t i = start; i < start + numOfRows; ++i) {
+ if (*val < pData[i]) {
+ *val = pData[i];
+ }
+ }
+ }
+ }
+
+ pBuf->assign = true;
+}
+
+static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) {
+ // the data is loaded, not only the block SMA value
+ for (int32_t i = start; i < num + start; ++i) {
+ char* p = colDataGetData(pCol, i);
+ if (memcmp((void*)tval, p, pCol->info.bytes) == 0) {
+ return i;
+ }
+ }
+
+ // if reach here means real data of block SMA is not set in pCtx->input.
+ return -1;
+}
+
+static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunctionCtx* pCtx, SMinmaxResInfo* pBuf,
+ bool isMinFunc) {
+ if (isMinFunc) {
+ switch (pCol->info.type) {
+ case TSDB_DATA_TYPE_BOOL:
+ case TSDB_DATA_TYPE_TINYINT: {
+ const int8_t* pData = (const int8_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, *(int8_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_SMALLINT: {
+ const int16_t* pData = (const int16_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, *(int16_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_INT: {
+ const int32_t* pData = (const int32_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, *(int32_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_BIGINT: {
+ const int64_t* pData = (const int64_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, (pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UTINYINT: {
+ const uint8_t* pData = (const uint8_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, *(uint8_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_USMALLINT: {
+ const uint16_t* pData = (const uint16_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, *(uint16_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UINT: {
+ const uint32_t* pData = (const uint32_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, *(uint32_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UBIGINT: {
+ const uint64_t* pData = (const uint64_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, *(uint64_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_FLOAT: {
+ const float* pData = (const float*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, *(float*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_DOUBLE: {
+ const double* pData = (const double*)pCol->pData;
+ __COMPARE_ACQUIRED_MIN(i, end, pCol->nullbitmap, pData, pCtx, *(double*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+ }
+ } else {
+ switch (pCol->info.type) {
+ case TSDB_DATA_TYPE_BOOL:
+ case TSDB_DATA_TYPE_TINYINT: {
+ const int8_t* pData = (const int8_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(int8_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_SMALLINT: {
+ const int16_t* pData = (const int16_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(int16_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_INT: {
+ const int32_t* pData = (const int32_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(int32_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_BIGINT: {
+ const int64_t* pData = (const int64_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, (pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UTINYINT: {
+ const uint8_t* pData = (const uint8_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(uint8_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_USMALLINT: {
+ const uint16_t* pData = (const uint16_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(uint16_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UINT: {
+ const uint32_t* pData = (const uint32_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(uint32_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_UBIGINT: {
+ const uint64_t* pData = (const uint64_t*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(uint64_t*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_FLOAT: {
+ const float* pData = (const float*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(float*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+
+ case TSDB_DATA_TYPE_DOUBLE: {
+ const double* pData = (const double*)pCol->pData;
+ __COMPARE_ACQUIRED_MAX(i, end, pCol->nullbitmap, pData, pCtx, *(double*)&(pBuf->v), &pBuf->tuplePos)
+ break;
+ }
+ }
+ }
+}
+
+int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
+ int32_t numOfElems = 0;
+
+ SInputColumnInfoData* pInput = &pCtx->input;
+ SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0];
+
+ SColumnInfoData* pCol = pInput->pData[0];
+ int32_t type = pCol->info.type;
+
+ SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+ SMinmaxResInfo* pBuf = GET_ROWCELL_INTERBUF(pResInfo);
+ pBuf->type = type;
+
+ if (IS_NULL_TYPE(type)) {
+ numOfElems = 0;
+ goto _over;
+ }
+
+ // data in current data block are qualified to the query
+ if (pInput->colDataSMAIsSet) {
+ numOfElems = pInput->numOfRows - pAgg->numOfNull;
+ ASSERT(pInput->numOfRows == pInput->totalRows && numOfElems >= 0);
+
+ if (numOfElems == 0) {
+ goto _over;
+ }
+
+ void* tval = NULL;
+ int16_t index = 0;
+
+ if (isMinFunc) {
+ tval = &pInput->pColumnDataAgg[0]->min;
+ } else {
+ tval = &pInput->pColumnDataAgg[0]->max;
+ }
+
+ if (!pBuf->assign) {
+ pBuf->v = *(int64_t*)tval;
+ if (pCtx->subsidiaries.num > 0) {
+ index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
+ if (index >= 0) {
+ pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
+ }
+ }
+ } else {
+ if (IS_SIGNED_NUMERIC_TYPE(type)) {
+ int64_t prev = 0;
+ GET_TYPED_DATA(prev, int64_t, type, &pBuf->v);
+
+ int64_t val = GET_INT64_VAL(tval);
+ if ((prev < val) ^ isMinFunc) {
+ *(int64_t*)&pBuf->v = val;
+ if (pCtx->subsidiaries.num > 0) {
+ index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
+ if (index >= 0) {
+ pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
+ }
+ }
+ }
+ } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+ uint64_t prev = 0;
+ GET_TYPED_DATA(prev, uint64_t, type, &pBuf->v);
+
+ uint64_t val = GET_UINT64_VAL(tval);
+ if ((prev < val) ^ isMinFunc) {
+ *(uint64_t*)&pBuf->v = val;
+ if (pCtx->subsidiaries.num > 0) {
+ index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
+ if (index >= 0) {
+ pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
+ }
+ }
+ }
+ } else if (type == TSDB_DATA_TYPE_DOUBLE) {
+ double prev = 0;
+ GET_TYPED_DATA(prev, double, type, &pBuf->v);
+
+ double val = GET_DOUBLE_VAL(tval);
+ if ((prev < val) ^ isMinFunc) {
+ *(double*)&pBuf->v = val;
+ if (pCtx->subsidiaries.num > 0) {
+ index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
+ if (index >= 0) {
+ pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
+ }
+ }
+ }
+ } else if (type == TSDB_DATA_TYPE_FLOAT) {
+ float prev = 0;
+ GET_TYPED_DATA(prev, float, type, &pBuf->v);
+
+ float val = GET_DOUBLE_VAL(tval);
+ if ((prev < val) ^ isMinFunc) {
+ *(float*)&pBuf->v = val;
+ }
+
+ if (pCtx->subsidiaries.num > 0) {
+ index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
+ if (index >= 0) {
+ pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
+ }
+ }
+ }
+ }
+
+ pBuf->assign = true;
+ return numOfElems;
+ }
+
+ int32_t start = pInput->startRowIndex;
+ int32_t numOfRows = pInput->numOfRows;
+ int32_t end = start + numOfRows;
+
+ if (pCol->hasNull || numOfRows < 32 || pCtx->subsidiaries.num > 0) {
+ int32_t i = findFirstValPosition(pCol, start, numOfRows);
+
+ if ((i < end) && (!pBuf->assign)) {
+ memcpy(&pBuf->v, pCol->pData + (pCol->info.bytes * i), pCol->info.bytes);
+
+ if (pCtx->subsidiaries.num > 0) {
+ pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
+ }
+ pBuf->assign = true;
+ numOfElems = 1;
+ }
+
+ if (i >= end) {
+ ASSERT(numOfElems == 0);
+ goto _over;
+ }
+
+ doExtractVal(pCol, i, end, pCtx, pBuf, isMinFunc);
+ } else {
+ numOfElems = numOfRows;
+
+ switch (pCol->info.type) {
+ case TSDB_DATA_TYPE_BOOL:
+ case TSDB_DATA_TYPE_TINYINT: {
+ handleInt8Col(pCol->pData, start, numOfRows, pBuf, isMinFunc, true);
+ break;
+ }
+ case TSDB_DATA_TYPE_SMALLINT: {
+ handleInt16Col(pCol->pData, start, numOfRows, pBuf, isMinFunc, true);
+ break;
+ }
+ case TSDB_DATA_TYPE_INT: {
+ handleInt32Col(pCol->pData, start, numOfRows, pBuf, isMinFunc, true);
+ break;
+ }
+ case TSDB_DATA_TYPE_BIGINT: {
+ handleInt64Col(pCol->pData, start, numOfRows, pBuf, isMinFunc, true);
+ break;
+ }
+ case TSDB_DATA_TYPE_UTINYINT: {
+ handleInt8Col(pCol->pData, start, numOfRows, pBuf, isMinFunc, false);
+ break;
+ }
+ case TSDB_DATA_TYPE_USMALLINT: {
+ handleInt16Col(pCol->pData, start, numOfRows, pBuf, isMinFunc, false);
+ break;
+ }
+ case TSDB_DATA_TYPE_UINT: {
+ handleInt32Col(pCol->pData, start, numOfRows, pBuf, isMinFunc, false);
+ break;
+ }
+ case TSDB_DATA_TYPE_UBIGINT: {
+ handleInt64Col(pCol->pData, start, numOfRows, pBuf, isMinFunc, false);
+ break;
+ }
+ case TSDB_DATA_TYPE_FLOAT: {
+ handleFloatCol(pCol, start, numOfRows, pBuf, isMinFunc);
+ break;
+ }
+ case TSDB_DATA_TYPE_DOUBLE: {
+ handleDoubleCol(pCol, start, numOfRows, pBuf, isMinFunc);
+ break;
+ }
+ }
+
+ pBuf->assign = true;
+ }
+
+_over:
+ if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) {
+ pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
+ pBuf->nullTupleSaved = true;
+ }
+
+ return numOfElems;
+}
\ No newline at end of file
diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c
index 0924106476..e5727f1472 100644
--- a/source/libs/function/src/tpercentile.c
+++ b/source/libs/function/src/tpercentile.c
@@ -494,7 +494,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
resetSlotInfo(pMemBucket);
int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1);
- SIDList list = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId));
+ SArray* list = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId));
ASSERT(list != NULL && list->size > 0);
for (int32_t f = 0; f < list->size; ++f) {
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index f5682b69bf..d87df2d0f3 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -27,6 +27,7 @@
#include "tglobal.h"
#include "tmsg.h"
#include "trpc.h"
+#include "tmisce.h"
// clang-foramt on
typedef struct SUdfdContext {
diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp
index 95f7af435d..be2e4b90b9 100644
--- a/source/libs/parser/test/mockCatalogService.cpp
+++ b/source/libs/parser/test/mockCatalogService.cpp
@@ -20,9 +20,9 @@
#include