xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #26734 from taosdata/3.0 

3.0
This commit is contained in:
dm chen 2024-07-23 10:39:48 +08:00 committed by GitHub
parent 005b51efeb 26c7943279
commit eac6597a30
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
123 changed files with 8014 additions and 5193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/rsync.h
View File

@ -12,7 +12,7 @@ extern "C" {
#include "tarray.h"
void stopRsync();
void startRsync();
int32_t startRsync();
int32_t uploadByRsync(const char* id, const char* path);
int32_t downloadRsync(const char* id, const char* path);
int32_t deleteRsync(const char* id);

2
include/common/tdatablock.h
View File

@ -247,7 +247,7 @@ int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockI
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize);
int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n);
int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n);
void blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n);
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc);

2
include/libs/function/functionMgt.h
View File

@ -138,6 +138,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_CACHE_LAST_ROW,
FUNCTION_TYPE_CACHE_LAST,
FUNCTION_TYPE_TABLE_COUNT,
FUNCTION_TYPE_GROUP_CONST_VALUE,
// distributed splitting functions
FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000,
@ -256,6 +257,7 @@ bool fmIsConstantResFunc(SFunctionNode* pFunc);
bool fmIsSkipScanCheckFunc(int32_t funcId);
bool fmIsPrimaryKeyFunc(int32_t funcId);
bool fmIsProcessByRowFunc(int32_t funcId);
bool fmisSelectGroupConstValueFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList);

2
include/libs/qcom/query.h
View File

@ -335,7 +335,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam
void destroyQueryExecRes(SExecResult* pRes);
int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len);
char* parseTagDatatoJson(void* p);
void parseTagDatatoJson(void* p, char** jsonStr);
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType);
int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst);

1
include/os/osString.h
View File

@ -61,6 +61,7 @@ typedef enum { M2C = 0, C2M } ConvType;
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size))
#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src))
char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4);

2
include/util/taoserror.h
View File

@ -53,7 +53,7 @@ int32_t taosGetErrSize();
#define terrln (*taosGetErrln())
#define SET_ERROR_MSG(MSG, ...) \
snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__)
(void)snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__)
#define TSDB_CODE_SUCCESS 0
#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error

4
include/util/tutil.h
View File

@ -135,6 +135,8 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
} \
} while (0)
#define QUERY_CHECK_CODE TSDB_CHECK_CODE
#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \
if ((ptr) == NULL) { \
(CODE) = (ERRNO); \
@ -142,6 +144,8 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
goto LABEL; \
}
#define QUERY_CHECK_NULL TSDB_CHECK_NULL
#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
#define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL)

7
packaging/deb/DEBIAN/prerm
View File

@ -20,6 +20,7 @@ if [ -f ${insmetaPath}/preun.sh ]; then
else
bin_link_dir="/usr/bin"
lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include"
data_link_dir="/usr/local/taos/data"
@ -40,9 +41,11 @@ else
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || :
${csudo}rm -f ${inc_link_dir}/taosws.h || :
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
${csudo}rm -f ${lib_link_dir}/libtaosws.so || :
${csudo}rm -f ${lib64_link_dir}/libtaosws.so || :
${csudo}rm -f ${log_link_dir} || :
${csudo}rm -f ${data_link_dir} || :

10
packaging/rpm/tdengine.spec
View File

@ -223,6 +223,7 @@ if [ $1 -eq 0 ];then
else
bin_link_dir="/usr/bin"
lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include"
data_link_dir="/usr/local/taos/data"
@ -235,13 +236,18 @@ if [ $1 -eq 0 ];then
${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/taosdump || :
${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
${csudo}rm -f ${cfg_link_dir}/* || :
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
${csudo}rm -f ${inc_link_dir}/taows.h || :
${csudo}rm -f ${lib_link_dir}/libtaos.so || :
${csudo}rm -f ${lib_link_dir}/libtaosws.so || :
${csudo}rm -f ${lib64_link_dir}/libtaosws.so || :
${csudo}rm -f ${log_link_dir} || :
${csudo}rm -f ${data_link_dir} || :

5
packaging/tools/remove.sh
View File

@ -12,6 +12,8 @@ RED='\033[0;31m'
GREEN='\033[1;32m'
NC='\033[0m'
PREFIX="taos"
if [ "$osType" != "Darwin" ]; then
installDir="/usr/local/taos"
bin_link_dir="/usr/bin"
@ -32,7 +34,6 @@ else
inc_link_dir="/usr/local/include"
fi
PREFIX="taos"
serverName="${PREFIX}d"
clientName="${PREFIX}"
uninstallScript="rm${PREFIX}"
@ -46,8 +47,6 @@ explorerName="${PREFIX}-explorer"
tarbitratorName="tarbitratord"
productName="TDengine"
installDir="/usr/local/${PREFIX}"
#install main path
install_main_dir=${installDir}
data_link_dir=${installDir}/data

12
packaging/tools/remove_client.sh
View File

@ -63,10 +63,13 @@ function clean_bin() {
}
function clean_lib() {
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
}
function clean_header() {
@ -76,6 +79,7 @@ function clean_header() {
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
${csudo}rm -f ${inc_link_dir}/taosws.h || :
}
function clean_config() {

13
source/client/src/clientEnv.c
View File

@ -35,6 +35,7 @@
#include "tsched.h"
#include "ttime.h"
#include "tversion.h"
#include "tcompare.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@ -875,6 +876,12 @@ void taos_init_imp(void) {
}
rpcInit();
if (InitRegexCache() != 0) {
tscInitRes = -1;
tscError("failed to init regex cache");
return;
}
SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100};
catalogInit(&cfg);
@ -885,7 +892,11 @@ void taos_init_imp(void) {
taosSetCoreDump(true);
#endif
initTaskQueue();
if (initTaskQueue() != 0){
tscInitRes = -1;
tscError("failed to init task queue");
return;
}
fmFuncMgtInit();
nodesInitAllocatorSet();

3
source/client/src/clientImpl.c
View File

@ -2007,7 +2007,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (tTagIsJson(data)) {
char* jsonString = parseTagDatatoJson(data);
char* jsonString = NULL;
parseTagDatatoJson(data, &jsonString);
STR_TO_VARSTR(dst, jsonString);
taosMemoryFree(jsonString);
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"

2
source/client/src/clientMain.c
View File

@ -28,6 +28,7 @@
#include "tref.h"
#include "trpc.h"
#include "version.h"
#include "tcompare.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
@ -78,6 +79,7 @@ void taos_cleanup(void) {
clientConnRefPool = -1;
taosCloseRef(id);
DestroyRegexCache();
rpcCleanup();
tscDebug("rpc cleanup");

1222
source/client/src/clientRawBlockWrite.c
View File

File diff suppressed because it is too large Load Diff

20
source/client/test/CMakeLists.txt
View File

@ -23,11 +23,11 @@ TARGET_LINK_LIBRARIES(
PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry
)
ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp)
TARGET_LINK_LIBRARIES(
clientMonitorTest
PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor
)
#ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp)
#TARGET_LINK_LIBRARIES(
# clientMonitorTest
# PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor
#)
TARGET_INCLUDE_DIRECTORIES(
clientTest
@ -47,11 +47,11 @@ TARGET_INCLUDE_DIRECTORIES(
PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
)
TARGET_INCLUDE_DIRECTORIES(
clientMonitorTest
PUBLIC "${TD_SOURCE_DIR}/include/client/"
PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
)
#TARGET_INCLUDE_DIRECTORIES(
# clientMonitorTest
# PUBLIC "${TD_SOURCE_DIR}/include/client/"
# PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
#)
add_test(
NAME smlTest

26
source/common/src/rsync.c
View File

@ -54,10 +54,11 @@ static void changeDirFromWindowsToLinux(char* from, char* to) {
#endif
static int32_t generateConfigFile(char* confDir) {
int32_t code = 0;
TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA);
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
#ifdef WINDOWS
@ -92,7 +93,8 @@ static int32_t generateConfigFile(char* confDir) {
if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) {
uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
taosCloseFile(&pFile);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
return code;
}
taosCloseFile(&pFile);
@ -129,11 +131,13 @@ void stopRsync() {
taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation.
}
void startRsync() {
int32_t startRsync() {
int32_t code = 0;
if (taosMulMkDir(tsCheckpointBackupDir) != 0) {
uError("[rsync] build checkpoint backup dir failed, path:%s," ERRNO_ERR_FORMAT, tsCheckpointBackupDir,
ERRNO_ERR_DATA);
return;
code = TAOS_SYSTEM_ERROR(errno);
return code;
}
removeEmptyDir();
@ -141,9 +145,9 @@ void startRsync() {
char confDir[PATH_MAX] = {0};
snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir);
int32_t code = generateConfigFile(confDir);
code = generateConfigFile(confDir);
if (code != 0) {
return;
return code;
}
char cmd[PATH_MAX] = {0};
@ -152,9 +156,11 @@ void startRsync() {
code = system(cmd);
if (code != 0) {
uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else {
uInfo("[rsync] cmd:%s start server successful", cmd);
}
return code;
}
int32_t uploadByRsync(const char* id, const char* path) {
@ -199,6 +205,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
if (code != 0) {
uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else {
int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
@ -242,6 +249,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
if (code != 0) {
uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else {
int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
@ -283,13 +291,13 @@ int32_t downloadRsync(const char* id, const char* path) {
uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id,
path, times, code, ERRNO_ERR_DATA);
taosSsleep(1);
code = TAOS_SYSTEM_ERROR(errno);
} else {
int32_t el = taosGetTimestampMs() - st;
uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el);
break;
}
}
return code;
}
@ -298,7 +306,7 @@ int32_t deleteRsync(const char* id) {
int32_t code = taosMkDir(tmp);
if (code != 0) {
uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
char command[PATH_MAX] = {0};
@ -310,7 +318,7 @@ int32_t deleteRsync(const char* id) {
taosRemoveDir(tmp);
if (code != 0) {
uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
uDebug("[rsync] delete data:%s successful", id);

8
source/common/src/tdatablock.c
View File

@ -1990,14 +1990,14 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
}
}
int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
void blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
if (n == 0) {
blockDataEmpty(pBlock);
return TSDB_CODE_SUCCESS;
return ;
}
if (pBlock->info.rows <= n) {
return TSDB_CODE_SUCCESS;
return ;
} else {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
@ -2007,7 +2007,7 @@ int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
pBlock->info.rows = n;
}
return TSDB_CODE_SUCCESS;
return ;
}
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {

58
source/common/src/tglobal.c
View File

@ -1370,58 +1370,68 @@ static int32_t taosCheckGlobalCfg() {
return 0;
}
static int32_t cfgInitWrapper(SConfig **pCfg) {
if (*pCfg == NULL) {
*pCfg = cfgInit();
if (*pCfg == NULL) {
return terrno;
}
}
return 0;
}
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc) {
if (tsCfg != NULL) return 0;
tsCfg = cfgInit();
int32_t code = cfgInitWrapper(&tsCfg);
if (tsc) {
if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
if ((code = taosAddClientCfg(tsCfg)) != 0) return code;
if ((code = taosAddClientLogCfg(tsCfg)) != 0) return code;
} else {
if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddServerCfg(tsCfg) != 0) return -1;
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
if (taosAddServerLogCfg(tsCfg) != 0) return -1;
if ((code = taosAddClientCfg(tsCfg)) != 0) return code;
if ((code = taosAddServerCfg(tsCfg)) != 0) return code;
if ((code = taosAddClientLogCfg(tsCfg)) != 0) return code;
if ((code = taosAddServerLogCfg(tsCfg)) != 0) return code;
}
taosAddSystemCfg(tsCfg);
code = taosAddSystemCfg(tsCfg);
if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
uError("failed to load cfg since %s", terrstr());
if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
uError("failed to load cfg since %s", tstrerror(code));
cfgCleanup(tsCfg);
tsCfg = NULL;
return -1;
return code;
}
if (cfgLoadFromArray(tsCfg, pArgs) != 0) {
uError("failed to load cfg from array since %s", terrstr());
if ((code = cfgLoadFromArray(tsCfg, pArgs)) != 0) {
uError("failed to load cfg from array since %s", tstrerror(code));
cfgCleanup(tsCfg);
tsCfg = NULL;
return -1;
return code;
}
if (tsc) {
if (taosSetClientCfg(tsCfg)) return -1;
if ((code = taosSetClientCfg(tsCfg)) != 0) return code;
} else {
if (taosSetClientCfg(tsCfg)) return -1;
if (taosUpdateServerCfg(tsCfg)) return -1;
if (taosSetServerCfg(tsCfg)) return -1;
if (taosSetReleaseCfg(tsCfg)) return -1;
if (taosSetTfsCfg(tsCfg) != 0) return -1;
if (taosSetS3Cfg(tsCfg) != 0) return -1;
if ((code = taosSetClientCfg(tsCfg)) != 0) return code;
if ((code = taosUpdateServerCfg(tsCfg)) != 0) return code;
if ((code = taosSetServerCfg(tsCfg)) != 0) return code;
if ((code = taosSetReleaseCfg(tsCfg)) != 0) return code;
if ((code = taosSetTfsCfg(tsCfg)) != 0) return code;
if ((code = taosSetS3Cfg(tsCfg)) != 0) return code;
}
taosSetSystemCfg(tsCfg);
if (taosSetFileHandlesLimit() != 0) return -1;
if ((code = taosSetFileHandlesLimit()) != 0) return code;
taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
cfgDumpCfg(tsCfg, tsc, false);
if (taosCheckGlobalCfg() != 0) {
return -1;
if ((code = taosCheckGlobalCfg()) != 0) {
return code;
}
return 0;

2
source/common/src/tmsg.c
View File

@ -10828,7 +10828,9 @@ int32_t tDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) {
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) {
pRsp->batchMetaReq = taosArrayInit(size, POINTER_BYTES);
if (!pRsp->batchMetaReq) return -1;
pRsp->batchMetaLen = taosArrayInit(size, sizeof(int32_t));
if (!pRsp->batchMetaLen) return -1;
for (int32_t i = 0; i < size; i++) {
void *pCreate = NULL;
uint64_t len = 0;

77
source/dnode/mgmt/exe/dmMain.c
View File

@ -167,23 +167,23 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
return -1;
return TSDB_CODE_INVALID_CFG;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-a") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("apollo url overflow");
return -1;
return TSDB_CODE_INVALID_CFG;
}
tstrncpy(global.apolloUrl, argv[i], PATH_MAX);
} else {
printf("'-a' requires a parameter\n");
return -1;
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-s") == 0) {
global.dumpSdb = true;
@ -191,31 +191,31 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("env file path overflow");
return -1;
return TSDB_CODE_INVALID_CFG;
}
tstrncpy(global.envFile, argv[i], PATH_MAX);
} else {
printf("'-E' requires a parameter\n");
return -1;
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-y") == 0) {
global.generateCode = true;
if(i < argc - 1) {
if (i < argc - 1) {
int32_t len = strlen(argv[++i]);
if (len < ENCRYPT_KEY_LEN_MIN) {
printf("ERROR: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN);
return -1;
return TSDB_CODE_INVALID_CFG;
}
if (len > ENCRYPT_KEY_LEN) {
printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN);
return -1;
return TSDB_CODE_INVALID_CFG;
}
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
} else {
printf("'-y' requires a parameter\n");
return -1;
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true;
@ -310,6 +310,7 @@ static void taosCleanupArgs() {
}
int main(int argc, char const *argv[]) {
int32_t code = 0;
#ifdef TD_JEMALLOC_ENABLED
bool jeBackgroundThread = true;
mallctl("background_thread", NULL, NULL, &jeBackgroundThread, sizeof(bool));
@ -319,10 +320,10 @@ int main(int argc, char const *argv[]) {
return -1;
}
if (dmParseArgs(argc, argv) != 0) {
//printf("failed to start since parse args error\n");
if ((code = dmParseArgs(argc, argv)) != 0) {
// printf("failed to start since parse args error\n");
taosCleanupArgs();
return -1;
return code;
}
#ifdef WINDOWS
@ -335,6 +336,7 @@ int main(int argc, char const *argv[]) {
return 0;
}
int mainWindows(int argc, char **argv) {
int32_t code = 0;
#endif
if (global.generateGrant) {
@ -357,7 +359,7 @@ int mainWindows(int argc, char **argv) {
#if defined(LINUX)
if (global.memDbg) {
int32_t code = taosMemoryDbgInit();
code = taosMemoryDbgInit();
if (code) {
printf("failed to init memory dbg, error:%s\n", tstrerror(code));
return code;
@ -366,16 +368,18 @@ int mainWindows(int argc, char **argv) {
printf("memory dbg enabled\n");
}
#endif
if(global.generateCode) {
if (global.generateCode) {
bool toLogFile = false;
if(taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs) != 0){
encryptError("failed to generate encrypt code since dataDir can not be set from cfg file");
return -1;
if ((code = taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs)) != 0) {
encryptError("failed to generate encrypt code since dataDir can not be set from cfg file,reason:%s",
tstrerror(code));
return code;
};
if(dmCheckRunning(tsDataDir) == NULL) {
encryptError("failed to generate encrypt code since taosd is running, please stop it first");
return -1;
TdFilePtr pFile;
if ((code = dmCheckRunning(tsDataDir, &pFile)) != 0) {
encryptError("failed to generate encrypt code since taosd is running, please stop it first, reason:%s",
tstrerror(code));
return code;
}
int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile);
taosCloseLog();
@ -383,30 +387,30 @@ int mainWindows(int argc, char **argv) {
return ret;
}
if (dmInitLog() != 0) {
if ((code = dmInitLog()) != 0) {
printf("failed to start since init log error\n");
taosCleanupArgs();
return -1;
return code;
}
dmPrintArgs(argc, argv);
if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
if ((code = taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0)) != 0) {
dError("failed to start since read config error");
taosCloseLog();
taosCleanupArgs();
return -1;
return code;
}
if (taosConvInit() != 0) {
if ((code = taosConvInit()) != 0) {
dError("failed to init conv");
taosCloseLog();
taosCleanupArgs();
return -1;
return code;
}
if (global.checkS3) {
int32_t code = dmCheckS3();
code = dmCheckS3();
taosCleanupCfg();
taosCloseLog();
taosCleanupArgs();
@ -435,31 +439,32 @@ int mainWindows(int argc, char **argv) {
osSetProcPath(argc, (char **)argv);
taosCleanupArgs();
if(dmGetEncryptKey() != 0){
if ((code = dmGetEncryptKey()) != 0) {
dError("failed to start since failed to get encrypt key");
taosCloseLog();
taosCleanupArgs();
return -1;
return code;
};
if (dmInit() != 0) {
if (terrno == TSDB_CODE_NOT_FOUND) {
if ((code = dmInit()) != 0) {
if (code == TSDB_CODE_NOT_FOUND) {
dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support");
} else {
dError("failed to init dnode since %s", terrstr());
dError("failed to init dnode since %s", tstrerror(code));
}
taosCleanupCfg();
taosCloseLog();
taosConvDestroy();
return -1;
return code;
}
dInfo("start to init service");
dmSetSignalHandle();
tsDndStart = taosGetTimestampMs();
tsDndStartOsUptime = taosGetOsUptime();
int32_t code = dmRun();
code = dmRun();
dInfo("shutting down the service");
dmCleanup();

127
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -45,7 +45,7 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
SRetrieveIpWhiteReq req = {.ipWhiteVer = oldVer};
int32_t contLen = tSerializeRetrieveIpWhite(NULL, 0, &req);
void * pHead = rpcMallocCont(contLen);
void *pHead = rpcMallocCont(contLen);
tSerializeRetrieveIpWhite(pHead, contLen, &req);
SRpcMsg rpcMsg = {.pCont = pHead,
@ -116,7 +116,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.clusterCfg.ttlChangeOnWrite = tsTtlChangeOnWrite;
req.clusterCfg.enableWhiteList = tsEnableWhiteList ? 1 : 0;
req.clusterCfg.encryptionKeyStat = tsEncryptionKeyStat;
req.clusterCfg.encryptionKeyChksum = tsEncryptionKeyChksum;
req.clusterCfg.encryptionKeyChksum = tsEncryptionKeyChksum;
req.clusterCfg.monitorParas.tsEnableMonitor = tsEnableMonitor;
req.clusterCfg.monitorParas.tsMonitorInterval = tsMonitorInterval;
req.clusterCfg.monitorParas.tsSlowLogScope = tsSlowLogScope;
@ -146,7 +146,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.ipWhiteVer = pMgmt->pData->ipWhiteVer;
int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
void * pHead = rpcMallocCont(contLen);
void *pHead = rpcMallocCont(contLen);
tSerializeSStatusReq(pHead, contLen, &req);
tFreeSStatusReq(&req);
@ -207,18 +207,26 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SDCfgDnodeReq cfgReq = {0};
if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
SConfig *pCfg = taosGetCfg();
cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true);
taosCfgDynamicOptions(pCfg, cfgReq.config, true);
return 0;
code = cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true);
if (code != 0) {
if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) {
code = 0;
} else {
return code;
}
}
return taosCfgDynamicOptions(pCfg, cfgReq.config, true);
}
int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
@ -251,7 +259,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK;
pStatus->details[0] = 0;
SMonMloadInfo minfo = {0};
SMonMloadInfo minfo = {0};
(*pMgmt->getMnodeLoadsFp)(&minfo);
if (minfo.isMnode &&
(minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) {
@ -276,32 +284,49 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
}
int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
dDebug("server run status req is received");
SServerStatusRsp statusRsp = {0};
dmGetServerRunStatus(pMgmt, &statusRsp);
pMsg->info.rsp = NULL;
pMsg->info.rspLen = 0;
SRpcMsg rspMsg = {.info = pMsg->info};
int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp);
if (rspLen < 0) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
// rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
// return rspMsg.code;
}
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
// rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
// return rspMsg.code;
}
rspLen = tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp);
if (rspLen < 0) {
return TSDB_CODE_INVALID_MSG;
}
tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp);
pMsg->info.rsp = pRsp;
pMsg->info.rspLen = rspLen;
return 0;
}
SSDataBlock *dmBuildVariablesBlock(void) {
SSDataBlock * pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
size_t size = 0;
int32_t dmBuildVariablesBlock(SSDataBlock **ppBlock) {
int32_t code = 0;
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pBlock == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
size_t size = 0;
const SSysTableMeta *pMeta = NULL;
getInfosDbMeta(&pMeta, &size);
@ -314,52 +339,74 @@ SSDataBlock *dmBuildVariablesBlock(void) {
}
pBlock->pDataBlock = taosArrayInit(pMeta[index].colNum, sizeof(SColumnInfoData));
if (pBlock->pDataBlock == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
for (int32_t i = 0; i < pMeta[index].colNum; ++i) {
SColumnInfoData colInfoData = {0};
colInfoData.info.colId = i + 1;
colInfoData.info.type = pMeta[index].schema[i].type;
colInfoData.info.bytes = pMeta[index].schema[i].bytes;
taosArrayPush(pBlock->pDataBlock, &colInfoData);
if (taosArrayPush(pBlock->pDataBlock, &colInfoData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
pBlock->info.hasVarCol = true;
return pBlock;
_exit:
if (code != 0) {
blockDataDestroy(pBlock);
} else {
*ppBlock = pBlock;
}
return code;
}
int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) {
/*int32_t code = */dumpConfToDataBlock(pBlock, 1);
int32_t code = dumpConfToDataBlock(pBlock, 1);
if (code != 0) {
return code;
}
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0);
colDataSetNItems(pColInfo, 0, (const char *)&dnodeId, pBlock->info.rows, false);
if (pColInfo == NULL) {
return TSDB_CODE_OUT_OF_RANGE;
}
return TSDB_CODE_SUCCESS;
return colDataSetNItems(pColInfo, 0, (const char *)&dnodeId, pBlock->info.rows, false);
}
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t size = 0;
int32_t rowsRead = 0;
int32_t size = 0;
int32_t rowsRead = 0;
int32_t code = 0;
SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
#if 0
if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
terrno = TSDB_CODE_MND_NO_RIGHTS;
return -1;
code = TSDB_CODE_MND_NO_RIGHTS;
return code;
}
#endif
if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
SSDataBlock *pBlock = dmBuildVariablesBlock();
SSDataBlock *pBlock = NULL;
if ((code = dmBuildVariablesBlock(&pBlock)) != 0) {
return code;
}
dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
code = dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
if (code != 0) {
blockDataDestroy(pBlock);
return code;
}
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols +
@ -367,10 +414,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to retrieve data since %s", terrstr());
code = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to retrieve data since %s", tstrerror(code));
blockDataDestroy(pBlock);
return -1;
return code;
}
char *pStart = pRsp->data;
@ -404,7 +451,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SArray *dmGetMsgHandles() {
int32_t code = -1;
SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle));
if (pArray == NULL) goto _OVER;
if (pArray == NULL) {
return NULL;
}
// Requests handled by DNODE
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
@ -416,7 +465,7 @@ SArray *dmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_ENCRYPT_KEY, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
// Requests handled by MNODE

32
source/dnode/mgmt/mgmt_dnode/src/dmInt.c
View File

@ -18,22 +18,23 @@
#include "libs/function/tudf.h"
static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
if (dmStartStatusThread(pMgmt) != 0) {
return -1;
int32_t code = 0;
if ((code = dmStartStatusThread(pMgmt)) != 0) {
return code;
}
#if defined(TD_ENTERPRISE)
if (dmStartNotifyThread(pMgmt) != 0) {
return -1;
if ((code = dmStartNotifyThread(pMgmt)) != 0) {
return code;
}
#endif
if (dmStartMonitorThread(pMgmt) != 0) {
return -1;
if ((code = dmStartMonitorThread(pMgmt)) != 0) {
return code;
}
if (dmStartAuditThread(pMgmt) != 0) {
return -1;
if ((code = dmStartAuditThread(pMgmt)) != 0) {
return code;
}
if (dmStartCrashReportThread(pMgmt) != 0) {
return -1;
if ((code = dmStartCrashReportThread(pMgmt)) != 0) {
return code;
}
return 0;
}
@ -50,10 +51,10 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) {
}
static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt));
if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
pMgmt->pData = pInput->pData;
@ -70,12 +71,11 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;
pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp;
// pMgmt->pData->ipWhiteVer = 0;
if (dmStartWorker(pMgmt) != 0) {
return -1;
if ((code = dmStartWorker(pMgmt)) != 0) {
return code;
}
if (udfStartUdfd(pMgmt->pData->dnodeId) != 0) {
if ((code = udfStartUdfd(pMgmt->pData->dnodeId)) != 0) {
dError("failed to start udfd");
}

34
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -264,12 +264,14 @@ static void *dmCrashReportThreadFp(void *param) {
}
int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) {
dError("failed to create status thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create status thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -285,12 +287,14 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) {
}
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->notifyThread, &thAttr, dmNotifyThreadFp, pMgmt) != 0) {
dError("failed to create notify thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create notify thread since %s", strerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -308,12 +312,14 @@ void dmStopNotifyThread(SDnodeMgmt *pMgmt) {
}
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->monitorThread, &thAttr, dmMonitorThreadFp, pMgmt) != 0) {
dError("failed to create monitor thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create monitor thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -322,12 +328,14 @@ int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
}
int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) {
dError("failed to create audit thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create audit thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -350,6 +358,7 @@ void dmStopAuditThread(SDnodeMgmt *pMgmt) {
}
int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
if (!tsEnableCrashReport) {
return 0;
}
@ -358,8 +367,9 @@ int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->crashReportThread, &thAttr, dmCrashReportThreadFp, pMgmt) != 0) {
dError("failed to create crashReport thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create crashReport thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -431,8 +441,8 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
code = dmProcessCreateEncryptKeyReq(pMgmt, pMsg);
break;
default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in mgmt queue", pMsg);
code = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in mgmt queue, reason:%s", pMsg, tstrerror(code));
break;
}

135
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -21,11 +21,11 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
int32_t code = 0;
tjsonGetInt32ValueFromDouble(pJson, "deployed", pOption->deploy, code);
if (code < 0) return -1;
if (code < 0) return code;
tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code);
if (code < 0) return 0;
if (code < 0) return code;
tjsonGetInt32ValueFromDouble(pJson, "lastIndex", pOption->lastIndex, code);
if (code < 0) return 0;
if (code < 0) return code;
SJson *replicas = tjsonGetObjectItem(pJson, "replicas");
if (replicas == NULL) return 0;
@ -35,17 +35,17 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) {
SJson *replica = tjsonGetArrayItem(replicas, i);
if (replica == NULL) return -1;
if (replica == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
SReplica *pReplica = pOption->replicas + i;
tjsonGetInt32ValueFromDouble(replica, "id", pReplica->id, code);
if (code < 0) return -1;
if (code < 0) return code;
code = tjsonGetStringValue(replica, "fqdn", pReplica->fqdn);
if (code < 0) return -1;
if (code < 0) return code;
tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code);
if (code < 0) return -1;
if (code < 0) return code;
tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code);
if (code < 0) return -1;
if (code < 0) return code;
if (pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER) {
pOption->numOfReplicas++;
}
@ -63,36 +63,41 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
char *pData = NULL;
SJson *pJson = NULL;
char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
int32_t nBytes = snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("mnode file:%s not exist", file);
dInfo("mnode file:%s not exist, reason:%s", file, tstrerror(TAOS_SYSTEM_ERROR(errno)));
return 0;
}
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open mnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open mnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
pData = taosMemoryMalloc(size + 1);
if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read mnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read mnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
@ -100,12 +105,11 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
pJson = tjsonParse(pData);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (mmDecodeOption(pJson, pOption) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
if ((code = mmDecodeOption(pJson, pOption)) < 0) {
goto _OVER;
}
@ -118,37 +122,42 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to read mnode file:%s since %s", file, terrstr());
dError("failed to read mnode file:%s since %s", file, tstrerror(code));
}
return code;
}
static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) {
int32_t code = 0;
if (pOption->deploy && pOption->numOfTotalReplicas > 0) {
if (tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex) < 0) return -1;
if ((code = tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex)) < 0) return code;
SJson *replicas = tjsonCreateArray();
if (replicas == NULL) return -1;
if (tjsonAddItemToObject(pJson, "replicas", replicas) < 0) return -1;
if (replicas == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if ((code = tjsonAddItemToObject(pJson, "replicas", replicas)) < 0) return code;
for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) {
SJson *replica = tjsonCreateObject();
if (replica == NULL) return -1;
if (replica == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
const SReplica *pReplica = pOption->replicas + i;
if (tjsonAddDoubleToObject(replica, "id", pReplica->id) < 0) return -1;
if (tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn) < 0) return -1;
if (tjsonAddDoubleToObject(replica, "port", pReplica->port) < 0) return -1;
if (tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i]) < 0) return -1;
if (tjsonAddItemToArray(replicas, replica) < 0) return -1;
if ((code = tjsonAddDoubleToObject(replica, "id", pReplica->id)) < 0) return code;
if ((code = tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn)) < 0) return code;
if ((code = tjsonAddDoubleToObject(replica, "port", pReplica->port)) < 0) return code;
if ((code = tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i])) < 0) return code;
if ((code = tjsonAddItemToArray(replicas, replica)) < 0) return code;
}
}
if (tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex) < 0) return -1;
if ((code = tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex)) < 0) return code;
if (tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy) < 0) return -1;
if ((code = tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy)) < 0) return code;
return 0;
return code;
}
int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
@ -158,28 +167,59 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t nBytes = snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(realfile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
// terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (mmEncodeOption(pJson, pOption) != 0) goto _OVER;
if (pJson == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
TAOS_CHECK_GOTO(mmEncodeOption(pJson, pOption), NULL, _OVER);
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
terrno = 0;
if (buffer == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, buffer, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosCloseFile(&pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0;
dInfo("succeed to write mnode file:%s, deloyed:%d", realfile, pOption->deploy);
_OVER:
@ -188,8 +228,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write mnode file:%s since %s, deloyed:%d", realfile, terrstr(), pOption->deploy);
dError("failed to write mnode file:%s since %s, deloyed:%d", realfile, tstrerror(code), pOption->deploy);
}
return code;
}

37
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -26,11 +26,12 @@ void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) {
}
int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId;
SDCreateMnodeReq createReq = {0};
if (tDeserializeSDCreateMnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
SMnodeOpt option = {.deploy = true,
@ -56,43 +57,45 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
}
if (option.selfIndex == -1) {
terrno = TSDB_CODE_INVALID_OPTION;
dGError("failed to create mnode since %s, selfIndex is -1", terrstr());
return -1;
code = TSDB_CODE_INVALID_OPTION;
dGError("failed to create mnode since %s, selfIndex is -1", tstrerror(code));
return code;
}
if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
return -1;
if ((code = mmWriteFile(pInput->path, &option)) != 0) {
dGError("failed to write mnode file since %s", tstrerror(code));
return code;
}
return 0;
}
int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId;
SDDropMnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dGError("failed to drop mnode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dGError("failed to drop mnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
SMnodeOpt option = {.deploy = false};
if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
if ((code = mmWriteFile(pInput->path, &option)) != 0) {
dGError("failed to write mnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&dropReq);
return 0;
return code;
}
SArray *mmGetMsgHandles() {

68
source/dnode/mgmt/mgmt_mnode/src/mmInt.c
View File

@ -25,9 +25,10 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) {
}
static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
int32_t code = 0;
SMnodeOpt option = {0};
if (mmReadFile(pInput->path, &option) != 0) {
return -1;
if ((code = mmReadFile(pInput->path, &option)) != 0) {
return code;
}
if (!option.deploy) {
@ -41,7 +42,7 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
dInfo("deploy mnode required. option deploy:%d", option.deploy);
}
return 0;
return code;
}
static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) {
@ -73,22 +74,31 @@ static void mmClose(SMnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt);
}
static int32_t mndOpenWrapper(const char *path, SMnodeOpt *opt, SMnode **pMnode) {
int32_t code = 0;
*pMnode = mndOpen(path, opt);
if (*pMnode == NULL) {
code = terrno;
}
///*pMnode = pNode;
return code;
}
static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
if (walInit() != 0) {
dError("failed to init wal since %s", terrstr());
return -1;
int32_t code = 0;
if ((code = walInit()) != 0) {
dError("failed to init wal since %s", tstrerror(code));
return code;
}
if (syncInit() != 0) {
dError("failed to init sync since %s", terrstr());
return -1;
if ((code = syncInit()) != 0) {
dError("failed to init sync since %s", tstrerror(code));
return code;
}
SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt));
if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pMgmt->pData = pInput->pData;
@ -100,10 +110,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
taosThreadRwlockInit(&pMgmt->lock, NULL);
SMnodeOpt option = {0};
if (mmReadFile(pMgmt->path, &option) != 0) {
dError("failed to read file since %s", terrstr());
if ((code = mmReadFile(pMgmt->path, &option)) != 0) {
dError("failed to read file since %s", tstrerror(code));
mmClose(pMgmt);
return -1;
return code;
}
if (!option.deploy) {
@ -115,18 +125,18 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
mmBuildOptionForOpen(pMgmt, &option);
}
pMgmt->pMnode = mndOpen(pMgmt->path, &option);
if (pMgmt->pMnode == NULL) {
dError("failed to open mnode since %s", terrstr());
code = mndOpenWrapper(pMgmt->path, &option, &pMgmt->pMnode);
if (code != 0) {
dError("failed to open mnode since %s", tstrerror(code));
mmClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("mnode-impl", "initialized");
if (mmStartWorker(pMgmt) != 0) {
dError("failed to start mnode worker since %s", terrstr());
if ((code = mmStartWorker(pMgmt)) != 0) {
dError("failed to start mnode worker since %s", tstrerror(code));
mmClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("mnode-worker", "initialized");
@ -134,9 +144,9 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
option.deploy = true;
option.numOfReplicas = 0;
option.numOfTotalReplicas = 0;
if (mmWriteFile(pMgmt->path, &option) != 0) {
dError("failed to write mnode file since %s", terrstr());
return -1;
if ((code = mmWriteFile(pMgmt->path, &option)) != 0) {
dError("failed to write mnode file since %s", tstrerror(code));
return code;
}
}
@ -160,13 +170,9 @@ static void mmStop(SMnodeMgmt *pMgmt) {
mndStop(pMgmt->pMnode);
}
static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) {
return mndIsCatchUp(pMgmt->pMnode);
}
static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { return mndIsCatchUp(pMgmt->pMnode); }
static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) {
return mndGetRole(pMgmt->pMnode);
}
static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) { return mndGetRole(pMgmt->pMnode); }
SMgmtFunc mmGetMgmtFunc() {
SMgmtFunc mgmtFunc = {0};

51
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -179,13 +179,15 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pWorker = &pMgmt->syncRdWorker;
break;
default:
terrno = TSDB_CODE_INVALID_PARA;
code = TSDB_CODE_INVALID_PARA;
}
if (pWorker == NULL) return -1;
if (pWorker == NULL) return code;
SRpcMsg *pMsg;
code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg);
if (code) return code;
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL;
@ -201,6 +203,7 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
}
int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
int32_t code = 0;
SSingleWorkerCfg qCfg = {
.min = tsNumOfMnodeQueryThreads,
.max = tsNumOfMnodeQueryThreads,
@ -209,9 +212,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.param = pMgmt,
.poolType = QUERY_AUTO_QWORKER_POOL,
};
if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) {
dError("failed to start mnode-query worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &qCfg)) != 0) {
dError("failed to start mnode-query worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg fCfg = {
@ -221,9 +224,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->fetchWorker, &fCfg) != 0) {
dError("failed to start mnode-fetch worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fCfg)) != 0) {
dError("failed to start mnode-fetch worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg rCfg = {
@ -233,9 +236,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) {
dError("failed to start mnode-read worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->readWorker, &rCfg)) != 0) {
dError("failed to start mnode-read worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg wCfg = {
@ -245,9 +248,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) {
dError("failed to start mnode-write worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->writeWorker, &wCfg)) != 0) {
dError("failed to start mnode-write worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg sCfg = {
@ -257,9 +260,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessSyncMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) {
dError("failed to start mnode mnode-sync worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->syncWorker, &sCfg)) != 0) {
dError("failed to start mnode mnode-sync worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg scCfg = {
@ -269,9 +272,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessSyncMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg) != 0) {
dError("failed to start mnode mnode-sync-rd worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg)) != 0) {
dError("failed to start mnode mnode-sync-rd worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg arbCfg = {
@ -281,13 +284,13 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->arbWorker, &arbCfg) != 0) {
dError("failed to start mnode mnode-arb worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->arbWorker, &arbCfg)) != 0) {
dError("failed to start mnode mnode-arb worker since %s", tstrerror(code));
return code;
}
dDebug("mnode workers are initialized");
return 0;
return code;
}
void mmStopWorker(SMnodeMgmt *pMgmt) {

34
source/dnode/mgmt/mgmt_qnode/src/qmHandle.c
View File

@ -30,24 +30,25 @@ void qmGetQnodeLoads(SQnodeMgmt *pMgmt, SQnodeLoad *pInfo) {
}
int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDCreateQnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to create qnode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dError("failed to create qnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq);
return -1;
return code;
}
bool deployed = true;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) {
dError("failed to write qnode file since %s", terrstr());
if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write qnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&createReq);
@ -55,24 +56,25 @@ int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
}
int32_t qmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to drop qnode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dError("failed to drop qnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
bool deployed = false;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) {
dError("failed to write qnode file since %s", terrstr());
if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write qnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&dropReq);

30
source/dnode/mgmt/mgmt_qnode/src/qmInt.c
View File

@ -33,11 +33,18 @@ static void qmClose(SQnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt);
}
static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) {
*pQnode = qndOpen(pOption);
if (*pQnode == NULL) {
return terrno;
}
return 0;
}
static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt));
if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
pMgmt->pData = pInput->pData;
@ -50,29 +57,30 @@ static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
SQnodeOpt option = {0};
qmInitOption(pMgmt, &option);
pMgmt->pQnode = qndOpen(&option);
if (pMgmt->pQnode == NULL) {
dError("failed to open qnode since %s", terrstr());
code = qndOpenWrapper(&option, &pMgmt->pQnode);
if (code != 0) {
dError("failed to open qnode since %s", tstrerror(code));
qmClose(pMgmt);
return -1;
}
tmsgReportStartup("qnode-impl", "initialized");
if (udfcOpen() != 0) {
if ((code = udfcOpen()) != 0) {
dError("qnode can not open udfc");
qmClose(pMgmt);
return -1;
return code;
}
if (qmStartWorker(pMgmt) != 0) {
dError("failed to start qnode worker since %s", terrstr());
if ((code = qmStartWorker(pMgmt)) != 0) {
dError("failed to start qnode worker since %s", tstrerror(code));
qmClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("qnode-worker", "initialized");
pOutput->pMgmt = pMgmt;
return 0;
return code;
}
SMgmtFunc qmGetMgmtFunc() {

16
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -102,6 +102,8 @@ int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
}
int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
int32_t code = 0;
SSingleWorkerCfg queryCfg = {
.min = tsNumOfVnodeQueryThreads,
.max = tsNumOfVnodeQueryThreads,
@ -111,9 +113,9 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
.poolType = QUERY_AUTO_QWORKER_POOL,
};
if (tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg) != 0) {
dError("failed to start qnode-query worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg)) != 0) {
dError("failed to start qnode-query worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg fetchCfg = {
@ -124,13 +126,13 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg) != 0) {
dError("failed to start qnode-fetch worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg)) != 0) {
dError("failed to start qnode-fetch worker since %s", tstrerror(code));
return code;
}
dDebug("qnode workers are initialized");
return 0;
return code;
}
void qmStopWorker(SQnodeMgmt *pMgmt) {

35
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -19,24 +19,25 @@
void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {}
int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDCreateSnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to create snode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dError("failed to create snode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq);
return -1;
return code;
}
bool deployed = true;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) {
dError("failed to write snode file since %s", terrstr());
if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write snode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&createReq);
@ -44,24 +45,26 @@ int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
}
int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to drop snode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dError("failed to drop snode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
bool deployed = false;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) {
dError("failed to write snode file since %s", terrstr());
if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write snode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&dropReq);

38
source/dnode/mgmt/mgmt_snode/src/smInt.c
View File

@ -33,12 +33,19 @@ static void smClose(SSnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt);
}
int32_t sndOpenWrapper(const char *path, SSnodeOpt *pOption, SSnode **pNode) {
*pNode = sndOpen(path, pOption);
if (*pNode == NULL) {
return terrno;
}
return 0;
}
int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt));
if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pMgmt->pData = pInput->pData;
@ -50,35 +57,34 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
SSnodeOpt option = {0};
smInitOption(pMgmt, &option);
pMgmt->pSnode = sndOpen(pMgmt->path, &option);
if (pMgmt->pSnode == NULL) {
dError("failed to open snode since %s", terrstr());
code = sndOpenWrapper(pMgmt->path, &option, &pMgmt->pSnode);
if (code != 0) {
dError("failed to open snode since %s", tstrerror(code));
smClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("snode-impl", "initialized");
if (smStartWorker(pMgmt) != 0) {
dError("failed to start snode worker since %s", terrstr());
if ((code = smStartWorker(pMgmt)) != 0) {
dError("failed to start snode worker since %s", tstrerror(code));
smClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("snode-worker", "initialized");
if (udfcOpen() != 0) {
dError("failed to open udfc in snode");
if ((code = udfcOpen()) != 0) {
dError("failed to open udfc in snode since:%s", tstrerror(code));
smClose(pMgmt);
return -1;
return code;
}
pOutput->pMgmt = pMgmt;
return 0;
}
static int32_t smStartSnodes(SSnodeMgmt *pMgmt) {
return sndInit(pMgmt->pSnode);
}
static int32_t smStartSnodes(SSnodeMgmt *pMgmt) { return sndInit(pMgmt->pSnode); }
SMgmtFunc smGetMgmtFunc() {
SMgmtFunc mgmtFunc = {0};

60
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -68,17 +68,18 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
}
int32_t smStartWorker(SSnodeMgmt *pMgmt) {
int32_t code = 0;
pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *));
if (pMgmt->writeWroker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) {
SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker));
if (pWriteWorker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
SMultiWorkerCfg cfg = {
@ -87,13 +88,13 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
.fp = smProcessWriteQueue,
.param = pMgmt,
};
if (tMultiWorkerInit(pWriteWorker, &cfg) != 0) {
dError("failed to start snode-unique worker since %s", terrstr());
return -1;
if ((code = tMultiWorkerInit(pWriteWorker, &cfg)) != 0) {
dError("failed to start snode-unique worker since %s", tstrerror(code));
return code;
}
if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
@ -105,13 +106,13 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) {
dError("failed to start snode shared-worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) != 0) {
dError("failed to start snode shared-worker since %s", tstrerror(code));
return code;
}
dDebug("snode workers are initialized");
return 0;
return code;
}
void smStopWorker(SSnodeMgmt *pMgmt) {
@ -133,17 +134,18 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
if (code) {
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
return code = terrno;
}
SSnode *pSnode = pMgmt->pSnode;
if (pSnode == NULL) {
dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d len:%d", pMsg, terrstr(),
code = terrno;
dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d len:%d", pMsg, tstrerror(code),
TMSG_INFO(pMsg->msgType), qtype, pRpc->contLen);
taosFreeQitem(pMsg);
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
return code;
}
SMsgHead *pHead = pRpc->pCont;
@ -154,48 +156,44 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
switch (qtype) {
case STREAM_QUEUE:
smPutNodeMsgToStreamQueue(pMgmt, pMsg);
code = smPutNodeMsgToStreamQueue(pMgmt, pMsg);
break;
case WRITE_QUEUE:
smPutNodeMsgToWriteQueue(pMgmt, pMsg);
code = smPutNodeMsgToWriteQueue(pMgmt, pMsg);
break;
default:
terrno = TSDB_CODE_INVALID_PARA;
code = TSDB_CODE_INVALID_PARA;
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
return -1;
return code;
}
return 0;
return code;
}
int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg);
return 0;
return taosWriteQitem(pWorker->queue, pMsg);
}
int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg);
return 0;
return taosWriteQitem(pWorker->queue, pMsg);
}
int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->streamWorker;
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg);
return 0;
return taosWriteQitem(pWorker->queue, pMsg);
}

42
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -54,24 +54,24 @@ typedef struct {
} SWrapperCfg;
typedef struct {
int32_t vgId;
int32_t vgVersion;
int32_t refCount;
int8_t dropped;
int8_t failed;
int8_t disable;
int32_t diskPrimary;
int32_t toVgId;
char *path;
SVnode *pImpl;
SMultiWorker pWriteW;
SMultiWorker pSyncW;
SMultiWorker pSyncRdW;
SMultiWorker pApplyW;
STaosQueue *pQueryQ;
STaosQueue *pStreamQ;
STaosQueue *pFetchQ;
STaosQueue *pMultiMgmQ;
int32_t vgId;
int32_t vgVersion;
int32_t refCount;
int8_t dropped;
int8_t failed;
int8_t disable;
int32_t diskPrimary;
int32_t toVgId;
char *path;
SVnode *pImpl;
SMultiWorker pWriteW;
SMultiWorker pSyncW;
SMultiWorker pSyncRdW;
SMultiWorker pApplyW;
STaosQueue *pQueryQ;
STaosQueue *pStreamQ;
STaosQueue *pFetchQ;
STaosQueue *pMultiMgmQ;
} SVnodeObj;
typedef struct {
@ -107,9 +107,9 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
// vmFile.c
int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt);
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes);
int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt);
int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
// vmWorker.c
int32_t vmStartWorker(SVnodeMgmt *pMgmt);

143
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -19,12 +19,16 @@
#define MAX_CONTENT_LEN 2 * 1024 * 1024
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
taosThreadRwlockRdlock(&pMgmt->lock);
int32_t num = 0;
int32_t size = taosHashGetSize(pMgmt->hash);
SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *));
if (pVnodes == NULL) {
taosThreadRwlockUnlock(&pMgmt->lock);
return TSDB_CODE_OUT_OF_MEMORY;
}
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
@ -42,8 +46,9 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
taosThreadRwlockUnlock(&pMgmt->lock);
*numOfVnodes = num;
*ppVnodes = pVnodes;
return pVnodes;
return 0;
}
static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) {
@ -52,29 +57,32 @@ static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **
*ppCfgs = NULL;
SJson *vnodes = tjsonGetObjectItem(pJson, "vnodes");
if (vnodes == NULL) return -1;
if (vnodes == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t vnodesNum = cJSON_GetArraySize(vnodes);
if (vnodesNum > 0) {
pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg));
if (pCfgs == NULL) return -1;
if (pCfgs == NULL) return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < vnodesNum; ++i) {
SJson *vnode = tjsonGetArrayItem(vnodes, i);
if (vnode == NULL) goto _OVER;
if (vnode == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
SWrapperCfg *pCfg = &pCfgs[i];
tjsonGetInt32ValueFromDouble(vnode, "vgId", pCfg->vgId, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "dropped", pCfg->dropped, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "diskPrimary", pCfg->diskPrimary, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId);
}
@ -98,33 +106,35 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("vnode file:%s not exist", file);
return 0;
code = TAOS_SYSTEM_ERROR(errno);
dInfo("vnode file:%s not exist, reason:%s", file, tstrerror(code));
code = 0;
return code;
}
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open vnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open vnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
pData = taosMemoryMalloc(size + 1);
if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read vnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read vnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
@ -132,12 +142,12 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
pJson = tjsonParse(pData);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (vmDecodeVnodeList(pJson, pMgmt, ppCfgs, numOfVnodes) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
@ -150,28 +160,36 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to read vnode file:%s since %s", file, terrstr());
dError("failed to read vnode file:%s since %s", file, tstrerror(code));
}
return code;
}
static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t numOfVnodes) {
SJson *vnodes = tjsonCreateArray();
if (vnodes == NULL) return -1;
if (tjsonAddItemToObject(pJson, "vnodes", vnodes) < 0) return -1;
int32_t code = 0;
SJson *vnodes = tjsonCreateArray();
if (vnodes == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if ((code = tjsonAddItemToObject(pJson, "vnodes", vnodes)) < 0) {
tjsonDelete(vnodes);
return code;
};
for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = ppVnodes[i];
if (pVnode == NULL) continue;
SJson *vnode = tjsonCreateObject();
if (vnode == NULL) return -1;
if (tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId) < 0) return -1;
if (tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped) < 0) return -1;
if (tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion) < 0) return -1;
if (tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary) < 0) return -1;
if (pVnode->toVgId && tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId) < 0) return -1;
if (tjsonAddItemToArray(vnodes, vnode) < 0) return -1;
if (vnode == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if ((code = tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId)) < 0) return code;
if ((code = tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped)) < 0) return code;
if ((code = tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion)) < 0) return code;
if ((code = tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary)) < 0) return code;
if (pVnode->toVgId) {
if ((code = tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId)) < 0) return code;
}
if ((code = tjsonAddItemToArray(vnodes, vnode)) < 0) return code;
}
return 0;
@ -185,30 +203,60 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
SVnodeObj **ppVnodes = NULL;
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
int32_t nBytes = snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
return TSDB_CODE_OUT_OF_RANGE;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(realfile)) {
return TSDB_CODE_OUT_OF_RANGE;
}
int32_t numOfVnodes = 0;
ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
if (ppVnodes == NULL) goto _OVER;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code) goto _OVER;
terrno = TSDB_CODE_OUT_OF_MEMORY;
// terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes) != 0) goto _OVER;
if (pJson == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if ((code = vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes)) != 0) goto _OVER;
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
terrno = 0;
if (buffer == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, buffer, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
code = taosCloseFile(&pFile);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0;
dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes);
@ -228,8 +276,7 @@ _OVER:
}
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, terrstr(), numOfVnodes);
dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, tstrerror(code), numOfVnodes);
}
return code;
}

25
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -256,8 +256,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
char path[TSDB_FILENAME_LEN] = {0};
if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
if (req.learnerReplica == 0) {
@ -298,25 +297,24 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort ||
strcmp(pReplica->fqdn, tsLocalFqdn) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", req.vgId, pReplica->id, pReplica->fqdn,
pReplica->port);
return -1;
code = TSDB_CODE_INVALID_MSG;
dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode, reason:%s", req.vgId, pReplica->id,
pReplica->fqdn, pReplica->port, tstrerror(code));
return code;
}
if (req.encryptAlgorithm == DND_CA_SM4) {
if (strlen(tsEncryptKey) == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId);
return -1;
code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty, reason:%s", req.vgId, tstrerror(code));
return code;
}
}
vmGenerateVnodeCfg(&req, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, terrstr());
code = terrno != 0 ? terrno : -1;
if ((code = vmTsmaAdjustDays(&vnodeCfg, &req)) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, tstrerror(code));
goto _OVER;
}
@ -327,8 +325,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dError("vgId:%d, already exist", req.vgId);
tFreeSCreateVnodeReq(&req);
vmReleaseVnode(pMgmt, pVnode);
terrno = TSDB_CODE_VND_ALREADY_EXIST;
code = terrno;
code = TSDB_CODE_VND_ALREADY_EXIST;
return 0;
}

101
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -33,6 +33,7 @@ int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
}
int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
int32_t code = 0;
STfs *pTfs = pMgmt->pTfs;
int32_t diskId = 0;
if (!pTfs) {
@ -59,7 +60,12 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
// alloc
int32_t disks[TFS_MAX_DISKS_PER_TIER] = {0};
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
return code;
}
for (int32_t v = 0; v < numOfVnodes; v++) {
SVnodeObj *pVnode = ppVnodes[v];
disks[pVnode->diskPrimary] += 1;
@ -436,6 +442,7 @@ static void *vmCloseVnodeInThread(void *param) {
}
static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
int32_t code = 0;
dInfo("start to close all vnodes");
tSingleWorkerCleanup(&pMgmt->mgmtWorker);
dInfo("vnodes mgmt worker is stopped");
@ -443,7 +450,12 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
dInfo("vnodes multiple mgmt worker is stopped");
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return;
}
int32_t threadNum = tsNumOfCores / 2;
if (threadNum < 1) threadNum = 1;
@ -513,8 +525,14 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
}
static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) {
int32_t code = 0;
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return;
}
if (ppVnodes != NULL) {
for (int32_t i = 0; i < numOfVnodes; ++i) {
@ -549,12 +567,14 @@ static void *vmThreadFp(void *param) {
}
static int32_t vmInitTimer(SVnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->thread, &thAttr, vmThreadFp, pMgmt) != 0) {
dError("failed to create vnode timer thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create vnode timer thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -573,7 +593,10 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = -1;
SVnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodeMgmt));
if (pMgmt == NULL) goto _OVER;
if (pMgmt == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
pMgmt->pData = pInput->pData;
pMgmt->path = pInput->path;
@ -582,8 +605,18 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->msgCb.putToQueueFp = (PutToQueueFp)vmPutRpcMsgToQueue;
pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize;
pMgmt->msgCb.mgmt = pMgmt;
taosThreadRwlockInit(&pMgmt->lock, NULL);
taosThreadMutexInit(&pMgmt->createLock, NULL);
code = taosThreadRwlockInit(&pMgmt->lock, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = taosThreadMutexInit(&pMgmt->createLock, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
pMgmt->pTfs = pInput->pTfs;
if (pMgmt->pTfs == NULL) {
@ -592,38 +625,39 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
}
tmsgReportStartup("vnode-tfs", "initialized");
if (walInit() != 0) {
dError("failed to init wal since %s", terrstr());
if ((code = walInit()) != 0) {
dError("failed to init wal since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-wal", "initialized");
if (syncInit() != 0) {
dError("failed to open sync since %s", terrstr());
if ((code = syncInit()) != 0) {
dError("failed to open sync since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-sync", "initialized");
if (vnodeInit(tsNumOfCommitThreads) != 0) {
dError("failed to init vnode since %s", terrstr());
if ((code = vnodeInit(tsNumOfCommitThreads)) != 0) {
dError("failed to init vnode since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-commit", "initialized");
if (vmStartWorker(pMgmt) != 0) {
dError("failed to init workers since %s", terrstr());
if ((code = vmStartWorker(pMgmt)) != 0) {
dError("failed to init workers since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-worker", "initialized");
if (vmOpenVnodes(pMgmt) != 0) {
dError("failed to open all vnodes since %s", terrstr());
if ((code = vmOpenVnodes(pMgmt)) != 0) {
dError("failed to open all vnodes since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-vnodes", "initialized");
if (udfcOpen() != 0) {
dError("failed to open udfc in vnode");
if ((code = udfcOpen()) != 0) {
dError("failed to open udfc in vnode since %s", tstrerror(code));
goto _OVER;
}
@ -633,7 +667,7 @@ _OVER:
if (code == 0) {
pOutput->pMgmt = pMgmt;
} else {
dError("failed to init vnodes-mgmt since %s", terrstr());
dError("failed to init vnodes-mgmt since %s", tstrerror(code));
vmCleanup(pMgmt);
}
@ -683,18 +717,32 @@ static void *vmRestoreVnodeInThread(void *param) {
}
static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
int32_t code = 0;
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return code;
}
int32_t threadNum = tsNumOfCores / 2;
if (threadNum < 1) threadNum = 1;
int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread));
if (threads == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t t = 0; t < threadNum; ++t) {
threads[t].threadIndex = t;
threads[t].pMgmt = pMgmt;
threads[t].ppVnodes = taosMemoryCalloc(vnodesPerThread, sizeof(SVnode *));
if (threads[t].ppVnodes == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
}
for (int32_t v = 0; v < numOfVnodes; ++v) {
@ -717,6 +765,7 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) {
dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno));
ASSERT(errno == 0);
}
taosThreadAttrDestroy(&thAttr);
@ -742,6 +791,14 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
}
return vmInitTimer(pMgmt);
_exit:
for (int32_t t = 0; t < threadNum; ++t) {
SVnodeThread *pThread = &threads[t];
taosMemoryFree(pThread->ppVnodes);
}
taosMemoryFree(threads);
return code;
}
static void vmStop(SVnodeMgmt *pMgmt) { vmCleanupTimer(pMgmt); }

70
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -200,26 +200,33 @@ static bool vmDataSpaceSufficient(SVnodeObj *pVnode) {
}
}
static int32_t vmAcquireVnodeWrapper(SVnodeMgmt *pMgt, int32_t vgId, SVnodeObj **pNode) {
*pNode = vmAcquireVnode(pMgt, vgId);
if (*pNode == NULL) {
return terrno;
}
return 0;
}
static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId;
if (pMsg->contLen < sizeof(SMsgHead)) {
dGError("invalid rpc msg with no msg head at pCont. pMsg:%p, type:%s, contLen:%d", pMsg, TMSG_INFO(pMsg->msgType),
pMsg->contLen);
return -1;
return TSDB_CODE_INVALID_MSG;
}
SMsgHead *pHead = pMsg->pCont;
int32_t code = 0;
pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) {
SVnodeObj *pVnode = NULL;
code = vmAcquireVnodeWrapper(pMgmt, pHead->vgId, &pVnode);
if (code != 0) {
dGDebug("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
terrno = (terrno != 0) ? terrno : -1;
return terrno;
tstrerror(code), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
return code;
}
switch (qtype) {
@ -234,49 +241,45 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
break;
case STREAM_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pStreamQ, pMsg);
code = taosWriteQitem(pVnode->pStreamQ, pMsg);
break;
case FETCH_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pFetchQ, pMsg);
code = taosWriteQitem(pVnode->pFetchQ, pMsg);
break;
case WRITE_QUEUE:
if (!vmDataSpaceSufficient(pVnode)) {
terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE;
code = terrno;
code = TSDB_CODE_NO_ENOUGH_DISKSPACE;
dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break;
}
if (pMsg->msgType == TDMT_VND_SUBMIT && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) {
terrno = TSDB_CODE_VND_NO_WRITE_AUTH;
code = terrno;
code = TSDB_CODE_VND_NO_WRITE_AUTH;
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break;
}
if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) {
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since its disable", pVnode->vgId, pMsg);
terrno = TSDB_CODE_VND_STOPPED;
code = terrno;
code = TSDB_CODE_VND_STOPPED;
break;
}
dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pWriteW.queue, pMsg);
code = taosWriteQitem(pVnode->pWriteW.queue, pMsg);
break;
case SYNC_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pSyncW.queue, pMsg);
code = taosWriteQitem(pVnode->pSyncW.queue, pMsg);
break;
case SYNC_RD_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pSyncRdW.queue, pMsg);
code = taosWriteQitem(pVnode->pSyncRdW.queue, pMsg);
break;
case APPLY_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pApplyW.queue, pMsg);
code = taosWriteQitem(pVnode->pApplyW.queue, pMsg);
break;
default:
code = -1;
terrno = TSDB_CODE_INVALID_PARA;
code = TSDB_CODE_INVALID_MSG;
break;
}
@ -299,15 +302,13 @@ int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMs
int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-multi-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg);
return 0;
return taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg);
}
int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg);
return 0;
return taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg);
}
int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
@ -317,7 +318,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pRpc->contLen);
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
return TSDB_CODE_INVALID_MSG;
}
EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM;
@ -326,7 +327,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
if (code) {
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
return code;
}
SMsgHead *pHead = pRpc->pCont;
@ -383,6 +384,7 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
}
int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
int32_t code = 0;
SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl};
SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
@ -398,8 +400,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL ||
pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
@ -428,26 +429,27 @@ void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
}
int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
int32_t code = 0;
SQueryAutoQWorkerPool *pQPool = &pMgmt->queryPool;
pQPool->name = "vnode-query";
pQPool->min = tsNumOfVnodeQueryThreads;
pQPool->max = tsNumOfVnodeQueryThreads;
if (tQueryAutoQWorkerInit(pQPool) != 0) return -1;
if ((code = tQueryAutoQWorkerInit(pQPool)) != 0) return code;
SAutoQWorkerPool *pStreamPool = &pMgmt->streamPool;
pStreamPool->name = "vnode-stream";
pStreamPool->ratio = tsRatioOfVnodeStreamThreads;
if (tAutoQWorkerInit(pStreamPool) != 0) return -1;
if ((code = tAutoQWorkerInit(pStreamPool)) != 0) return code;
SWWorkerPool *pFPool = &pMgmt->fetchPool;
pFPool->name = "vnode-fetch";
pFPool->max = tsNumOfVnodeFetchThreads;
if (tWWorkerInit(pFPool) != 0) return -1;
if ((code = tWWorkerInit(pFPool)) != 0) return code;
SSingleWorkerCfg mgmtCfg = {
.min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) return -1;
if ((code = tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg)) != 0) return code;
int32_t threadNum = 0;
if (tsNumOfCores == 1) {
@ -461,7 +463,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
.fp = (FItem)vmProcessMultiMgmtQueue,
.param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg) != 0) return -1;
if ((code = tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg)) != 0) return code;
dDebug("vnode workers are initialized");
return 0;

1
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -98,6 +98,7 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType nType);
int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode);
int32_t dmInitVarsWrapper(SDnode *pDnode);
void dmClearVars(SDnode *pDnode);
int32_t dmInitModule(SDnode *pDnode);
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);

160
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -14,20 +14,22 @@
*/
#define _DEFAULT_SOURCE
// clang-format off
#include "dmMgmt.h"
#include "audit.h"
#include "libs/function/tudf.h"
#include "tgrant.h"
#include "tcompare.h"
// clang-format on
#define DM_INIT_AUDIT() \
do { \
auditCfg.port = tsMonitorPort; \
auditCfg.server = tsMonitorFqdn; \
auditCfg.comp = tsMonitorComp; \
if (auditInit(&auditCfg) != 0) { \
return -1; \
} \
#define DM_INIT_AUDIT() \
do { \
auditCfg.port = tsMonitorPort; \
auditCfg.server = tsMonitorFqdn; \
auditCfg.comp = tsMonitorComp; \
if ((code = auditInit(&auditCfg)) != 0) { \
return code; \
} \
} while (0)
static SDnode globalDnode = {0};
@ -35,10 +37,11 @@ static SDnode globalDnode = {0};
SDnode *dmInstance() { return &globalDnode; }
static int32_t dmCheckRepeatInit(SDnode *pDnode) {
int32_t code = 0;
if (atomic_val_compare_exchange_8(&pDnode->once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
dError("env is already initialized");
terrno = TSDB_CODE_REPEAT_INIT;
return -1;
code = TSDB_CODE_REPEAT_INIT;
return code;
}
return 0;
}
@ -58,18 +61,15 @@ static int32_t dmInitMonitor() {
monCfg.port = tsMonitorPort;
monCfg.server = tsMonitorFqdn;
monCfg.comp = tsMonitorComp;
if (monInit(&monCfg) != 0) {
if (terrno != 0) code = terrno;
goto _exit;
if ((code = monInit(&monCfg)) != 0) {
dError("failed to init monitor since %s", tstrerror(code));
}
_exit:
if (code) terrno = code;
return code;
}
static int32_t dmInitAudit() {
SAuditCfg auditCfg = {0};
int32_t code = 0;
DM_INIT_AUDIT();
@ -88,27 +88,34 @@ static bool dmDataSpaceAvailable() {
return true;
}
static bool dmCheckDiskSpace() {
static int32_t dmCheckDiskSpace() {
osUpdate();
// availability
bool ret = true;
int32_t code = 0;
if (!dmDataSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE;
ret = false;
code = TSDB_CODE_NO_DISKSPACE;
return code;
}
if (!osLogSpaceAvailable()) {
dError("log disk space unavailable, i.e. %s", tsLogDir);
terrno = TSDB_CODE_NO_DISKSPACE;
ret = false;
code = TSDB_CODE_NO_DISKSPACE;
return code;
}
if (!osTempSpaceAvailable()) {
dError("temp disk space unavailable, i.e. %s", tsTempDir);
terrno = TSDB_CODE_NO_DISKSPACE;
ret = false;
code = TSDB_CODE_NO_DISKSPACE;
return code;
}
return ret;
return code;
}
int32_t tfsOpenWrapper(SDiskCfg *pCfg, int32_t ndisk, STfs **tfs) {
*tfs = tfsOpen(pCfg, ndisk);
if (*tfs == NULL) {
return terrno;
}
return 0;
}
int32_t dmDiskInit() {
SDnode *pDnode = dmInstance();
SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0};
@ -120,10 +127,10 @@ int32_t dmDiskInit() {
numOfDisks = 1;
}
pDnode->pTfs = tfsOpen(pDisks, numOfDisks);
if (pDnode->pTfs == NULL) {
dError("failed to init tfs since %s", terrstr());
return -1;
int32_t code = tfsOpenWrapper(pDisks, numOfDisks, &pDnode->pTfs);
if (code != 0) {
dError("failed to init tfs since %s", tstrerror(code));
return code;
}
return 0;
}
@ -146,6 +153,12 @@ static bool dmCheckDataDirVersion() {
return true;
}
static int32_t dmCheckDataDirVersionWrapper() {
if (!dmCheckDataDirVersion()) {
return TSDB_CODE_INVALID_DATA_FMT;
}
return 0;
}
#if defined(USE_S3)
extern int32_t s3Begin();
@ -156,17 +169,21 @@ extern int8_t tsS3Enabled;
int32_t dmInit() {
dInfo("start to init dnode env");
if (dmDiskInit() != 0) return -1;
if (!dmCheckDataDirVersion()) return -1;
if (!dmCheckDiskSpace()) return -1;
if (dmCheckRepeatInit(dmInstance()) != 0) return -1;
if (dmInitSystem() != 0) return -1;
if (dmInitMonitor() != 0) return -1;
if (dmInitAudit() != 0) return -1;
if (dmInitDnode(dmInstance()) != 0) return -1;
if (InitRegexCache() != 0) return -1;
int32_t code = 0;
if ((code = dmDiskInit()) != 0) return code;
if (!dmCheckDataDirVersion()) {
code = TSDB_CODE_INVALID_DATA_FMT;
return code;
}
if ((code = dmCheckDiskSpace()) != 0) return code;
if ((code = dmCheckRepeatInit(dmInstance())) != 0) return code;
if ((code = dmInitSystem()) != 0) return code;
if ((code = dmInitMonitor()) != 0) return code;
if ((code = dmInitAudit()) != 0) return code;
if ((code = dmInitDnode(dmInstance())) != 0) return code;
if ((code = InitRegexCache() != 0)) return code;
#if defined(USE_S3)
if (s3Begin() != 0) return -1;
if ((code = s3Begin()) != 0) return code;
#endif
dInfo("dnode env is initialized");
@ -217,6 +234,7 @@ int32_t dmRun() {
}
static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
@ -224,19 +242,19 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dmReleaseWrapper(pWrapper);
switch (ntype) {
case MNODE:
terrno = TSDB_CODE_MNODE_ALREADY_DEPLOYED;
code = TSDB_CODE_MNODE_ALREADY_DEPLOYED;
break;
case QNODE:
terrno = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
code = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
break;
case SNODE:
terrno = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
code = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
break;
default:
terrno = TSDB_CODE_APP_ERROR;
code = TSDB_CODE_APP_ERROR;
}
dError("failed to create node since %s", terrstr());
return -1;
dError("failed to create node since %s", tstrerror(code));
return code;
}
dInfo("start to process create-node-request");
@ -244,18 +262,18 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) {
dmReleaseWrapper(pWrapper);
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return code;
}
taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to create", pWrapper->name);
int32_t code = (*pWrapper->func.createFp)(&input, pMsg);
code = (*pWrapper->func.createFp)(&input, pMsg);
if (code != 0) {
dError("node:%s, failed to create since %s", pWrapper->name, terrstr());
dError("node:%s, failed to create since %s", pWrapper->name, tstrerror(code));
} else {
dInfo("node:%s, has been created", pWrapper->name);
code = dmOpenNode(pWrapper);
@ -271,12 +289,13 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
}
static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) {
dError("fail to process alter node type since node not exist");
return -1;
return TSDB_CODE_INVALID_MSG;
}
dmReleaseWrapper(pWrapper);
@ -289,16 +308,16 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dInfo("node:%s, checking node role:%d", pWrapper->name, role);
if (role == TAOS_SYNC_ROLE_VOTER) {
dError("node:%s, failed to alter node type since node already is role:%d", pWrapper->name, role);
terrno = TSDB_CODE_MNODE_ALREADY_IS_VOTER;
return -1;
code = TSDB_CODE_MNODE_ALREADY_IS_VOTER;
return code;
}
}
if (pWrapper->func.isCatchUpFp != NULL) {
dInfo("node:%s, checking node catch up", pWrapper->name);
if ((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1) {
terrno = TSDB_CODE_MNODE_NOT_CATCH_UP;
return -1;
code = TSDB_CODE_MNODE_NOT_CATCH_UP;
return code;
}
}
@ -314,17 +333,17 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) {
taosThreadMutexUnlock(&pDnode->mutex);
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return code;
}
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to create", pWrapper->name);
int32_t code = (*pWrapper->func.createFp)(&input, pMsg);
code = (*pWrapper->func.createFp)(&input, pMsg);
if (code != 0) {
dError("node:%s, failed to create since %s", pWrapper->name, terrstr());
dError("node:%s, failed to create since %s", pWrapper->name, tstrerror(code));
} else {
dInfo("node:%s, has been created", pWrapper->name);
code = dmOpenNode(pWrapper);
@ -340,35 +359,36 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
}
static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) {
switch (ntype) {
case MNODE:
terrno = TSDB_CODE_MNODE_NOT_DEPLOYED;
code = TSDB_CODE_MNODE_NOT_DEPLOYED;
break;
case QNODE:
terrno = TSDB_CODE_QNODE_NOT_DEPLOYED;
code = TSDB_CODE_QNODE_NOT_DEPLOYED;
break;
case SNODE:
terrno = TSDB_CODE_SNODE_NOT_DEPLOYED;
code = TSDB_CODE_SNODE_NOT_DEPLOYED;
break;
default:
terrno = TSDB_CODE_APP_ERROR;
code = TSDB_CODE_APP_ERROR;
}
dError("failed to drop node since %s", terrstr());
return -1;
dError("failed to drop node since %s", tstrerror(code));
return terrno = code;
}
taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to drop", pWrapper->name);
int32_t code = (*pWrapper->func.dropFp)(&input, pMsg);
code = (*pWrapper->func.dropFp)(&input, pMsg);
if (code != 0) {
dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
dError("node:%s, failed to drop since %s", pWrapper->name, tstrerror(code));
} else {
dInfo("node:%s, has been dropped", pWrapper->name);
pWrapper->required = false;
@ -416,6 +436,4 @@ void dmReportStartup(const char *pName, const char *pDesc) {
int64_t dmGetClusterId() { return globalDnode.data.clusterId; }
bool dmReadyForTest() {
return dmInstance()->data.dnodeVer > 0;
}
bool dmReadyForTest() { return dmInstance()->data.dnodeVer > 0; }

54
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -42,7 +42,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
int32_t code = -1;
char path[PATH_MAX + 100] = {0};
if (dmInitVars(pDnode) != 0) {
if ((code = dmInitVarsWrapper(pDnode)) != 0) {
goto _OVER;
}
@ -65,26 +65,31 @@ int32_t dmInitDnode(SDnode *pDnode) {
snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name);
pWrapper->path = taosStrdup(path);
if (pWrapper->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
pWrapper->required = dmRequireNode(pDnode, pWrapper);
}
pDnode->lockfile = dmCheckRunning(tsDataDir);
if (pDnode->lockfile == NULL) {
code = dmCheckRunning(tsDataDir, &pDnode->lockfile);
if (code != 0) {
goto _OVER;
}
if (dmInitModule(pDnode) != 0) {
if ((code = dmInitModule(pDnode)) != 0) {
goto _OVER;
}
indexInit(tsNumOfCommitThreads);
streamMetaInit();
dmInitStatusClient(pDnode);
dmInitSyncClient(pDnode);
if ((code = dmInitStatusClient(pDnode)) != 0) {
goto _OVER;
}
if ((code = dmInitSyncClient(pDnode)) != 0) {
goto _OVER;
}
dmReportStartup("dnode-transport", "initialized");
dDebug("dnode is created, ptr:%p", pDnode);
@ -94,7 +99,7 @@ _OVER:
if (code != 0 && pDnode != NULL) {
dmClearVars(pDnode);
pDnode = NULL;
dError("failed to create dnode since %s", terrstr());
dError("failed to create dnode since %s", tstrerror(code));
}
return code;
@ -122,7 +127,15 @@ void dmCleanupDnode(SDnode *pDnode) {
dDebug("dnode is closed, ptr:%p", pDnode);
}
int32_t dmInitVarsWrapper(SDnode *pDnode) {
int32_t code = dmInitVars(pDnode);
if (code == -1) {
return terrno;
}
return 0;
}
int32_t dmInitVars(SDnode *pDnode) {
int32_t code = 0;
SDnodeData *pData = &pDnode->data;
pData->dnodeId = 0;
pData->clusterId = 0;
@ -138,21 +151,21 @@ int32_t dmInitVars(SDnode *pDnode) {
taosMemoryFreeClear(machineId);
} else {
#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
return -1;
code = TSDB_CODE_DNODE_NO_MACHINE_CODE;
return terrno = code;
#endif
}
pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (pData->dnodeHash == NULL) {
dError("failed to init dnode hash");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return terrno = code;
}
if (dmReadEps(pData) != 0) {
dError("failed to read file since %s", terrstr());
return -1;
if ((code = dmReadEps(pData)) != 0) {
dError("failed to read file since %s", tstrerror(code));
return code;
}
#if defined(TD_ENTERPRISE)
@ -274,22 +287,21 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) {
} else {
switch (pWrapper->ntype) {
case MNODE:
terrno = TSDB_CODE_MNODE_NOT_FOUND;
code = TSDB_CODE_MNODE_NOT_FOUND;
break;
case QNODE:
terrno = TSDB_CODE_QNODE_NOT_FOUND;
code = TSDB_CODE_QNODE_NOT_FOUND;
break;
case SNODE:
terrno = TSDB_CODE_SNODE_NOT_FOUND;
code = TSDB_CODE_SNODE_NOT_FOUND;
break;
case VNODE:
terrno = TSDB_CODE_VND_STOPPED;
code = TSDB_CODE_VND_STOPPED;
break;
default:
terrno = TSDB_CODE_APP_IS_STOPPING;
code = TSDB_CODE_APP_IS_STOPPING;
break;
}
code = -1;
}
taosThreadRwlockUnlock(&pWrapper->lock);

47
source/dnode/mgmt/node_mgmt/src/dmNodes.c
View File

@ -17,12 +17,13 @@
#include "dmMgmt.h"
int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
int32_t code = 0;
SDnode *pDnode = pWrapper->pDnode;
if (taosMkDir(pWrapper->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr());
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, tstrerror(code));
return code;
}
SMgmtOutputOpt output = {0};
@ -30,9 +31,9 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
dInfo("node:%s, start to open", pWrapper->name);
tmsgSetDefault(&input.msgCb);
if ((*pWrapper->func.openFp)(&input, &output) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
return -1;
if ((code = (*pWrapper->func.openFp)(&input, &output)) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, tstrerror(code));
return code;
}
dInfo("node:%s, has been opened", pWrapper->name);
pWrapper->deployed = true;
@ -46,11 +47,12 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
}
int32_t dmStartNode(SMgmtWrapper *pWrapper) {
int32_t code = 0;
if (pWrapper->func.startFp != NULL) {
dDebug("node:%s, start to start", pWrapper->name);
if ((*pWrapper->func.startFp)(pWrapper->pMgmt) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
return -1;
if ((code = (*pWrapper->func.startFp)(pWrapper->pMgmt)) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, tstrerror(code));
return code;
}
dDebug("node:%s, has been started", pWrapper->name);
}
@ -86,12 +88,13 @@ void dmCloseNode(SMgmtWrapper *pWrapper) {
}
static int32_t dmOpenNodes(SDnode *pDnode) {
int32_t code = 0;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (!pWrapper->required) continue;
if (dmOpenNode(pWrapper) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
return -1;
if ((code = dmOpenNode(pWrapper)) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, tstrerror(code));
return code;
}
}
@ -100,12 +103,13 @@ static int32_t dmOpenNodes(SDnode *pDnode) {
}
static int32_t dmStartNodes(SDnode *pDnode) {
int32_t code = 0;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (!pWrapper->required) continue;
if (dmStartNode(pWrapper) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
return -1;
if ((code = dmStartNode(pWrapper)) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, tstrerror(code));
return code;
}
}
@ -129,19 +133,20 @@ static void dmCloseNodes(SDnode *pDnode) {
}
int32_t dmRunDnode(SDnode *pDnode) {
int32_t code = 0;
int32_t count = 0;
if (dmOpenNodes(pDnode) != 0) {
dError("failed to open nodes since %s", terrstr());
if ((code = dmOpenNodes(pDnode)) != 0) {
dError("failed to open nodes since %s", tstrerror(code));
dmCloseNodes(pDnode);
return -1;
return code;
}
if (dmStartNodes(pDnode) != 0) {
dError("failed to start nodes since %s", terrstr());
if ((code = dmStartNodes(pDnode)) != 0) {
dError("failed to start nodes since %s", tstrerror(code));
dmSetStatus(pDnode, DND_STAT_STOPPED);
dmStopNodes(pDnode);
dmCloseNodes(pDnode);
return -1;
return code;
}
while (1) {

68
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -39,9 +39,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)];
if (msgFp == NULL) {
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
// terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed since no handler, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
return -1;
return TSDB_CODE_MSG_NOT_PROCESSED;
}
dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name);
@ -54,14 +54,15 @@ static bool dmFailFastFp(tmsg_t msgType) {
return msgType == TDMT_SYNC_HEARTBEAT || msgType == TDMT_SYNC_APPEND_ENTRIES;
}
static void dmConvertErrCode(tmsg_t msgType) {
if (terrno != TSDB_CODE_APP_IS_STOPPING) {
return;
static int32_t dmConvertErrCode(tmsg_t msgType, int32_t code) {
if (code != TSDB_CODE_APP_IS_STOPPING) {
return code;
}
if ((msgType > TDMT_VND_MSG_MIN && msgType < TDMT_VND_MSG_MAX) ||
(msgType > TDMT_SCH_MSG_MIN && msgType < TDMT_SCH_MSG_MAX)) {
terrno = TSDB_CODE_VND_STOPPED;
code = TSDB_CODE_VND_STOPPED;
}
return code;
}
static void dmUpdateRpcIpWhite(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) {
SUpdateIpWhite ipWhite = {0}; // aosMemoryCalloc(1, sizeof(SUpdateIpWhite));
@ -99,14 +100,14 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
int32_t svrVer = 0;
taosVersionStrToInt(version, &svrVer);
if (0 != taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) {
if ((code = taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) != 0) {
dError("Version not compatible, cli ver: %d, svr ver: %d", pRpc->info.cliVer, svrVer);
goto _OVER;
}
bool isForbidden = dmIsForbiddenIp(pRpc->info.forbiddenIp, pRpc->info.conn.user, pRpc->info.conn.clientIp);
if (isForbidden) {
terrno = TSDB_CODE_IP_NOT_IN_WHITE_LIST;
code = TSDB_CODE_IP_NOT_IN_WHITE_LIST;
goto _OVER;
}
@ -119,7 +120,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
case TDMT_SCH_FETCH_RSP:
case TDMT_SCH_MERGE_FETCH_RSP:
case TDMT_VND_SUBMIT_RSP:
qWorkerProcessRspMsg(NULL, NULL, pRpc, 0);
code = qWorkerProcessRspMsg(NULL, NULL, pRpc, 0);
return;
case TDMT_MND_STATUS_RSP:
if (pEpSet != NULL) {
@ -148,32 +149,32 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
return;
} else {
if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING;
code = TSDB_CODE_APP_IS_STARTING;
} else {
terrno = TSDB_CODE_APP_IS_STOPPING;
code = TSDB_CODE_APP_IS_STOPPING;
}
goto _OVER;
}
}
} else {
terrno = TSDB_CODE_APP_IS_STARTING;
code = TSDB_CODE_APP_IS_STARTING;
goto _OVER;
}
if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) {
dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN;
code = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER;
} else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) &&
(!IsReq(pRpc)) && (pRpc->pCont == NULL)) {
dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code));
terrno = pRpc->code;
code = pRpc->code;
goto _OVER;
}
if (pHandle->defaultNtype == NODE_END) {
dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
code = TSDB_CODE_MSG_NOT_PROCESSED;
goto _OVER;
}
@ -197,12 +198,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
}
} else {
dGError("msg:%p, type:%s contLen is 0", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN;
code = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER;
}
}
if (dmMarkWrapper(pWrapper) != 0) {
if ((code = dmMarkWrapper(pWrapper)) != 0) {
pWrapper = NULL;
goto _OVER;
}
@ -221,12 +222,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
_OVER:
if (code != 0) {
dmConvertErrCode(pRpc->msgType);
if (terrno != 0) code = terrno;
code = dmConvertErrCode(pRpc->msgType, code);
if (pMsg) {
dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr());
dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code));
} else {
dGTrace("msg:%p, failed to process empty msg since %s", pMsg, terrstr());
dGTrace("msg:%p, failed to process empty msg since %s", pMsg, tstrerror(code));
}
if (IsReq(pRpc)) {
@ -280,17 +280,19 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
}
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING;
code = TSDB_CODE_APP_IS_STARTING;
} else {
terrno = TSDB_CODE_APP_IS_STOPPING;
code = TSDB_CODE_APP_IS_STOPPING;
}
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle);
return -1;
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), tstrerror(code),
pMsg->info.handle);
return code;
} else {
pMsg->info.handle = 0;
rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL);
@ -298,17 +300,19 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
}
}
static inline int32_t dmSendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING;
code = TSDB_CODE_APP_IS_STARTING;
} else {
terrno = TSDB_CODE_APP_IS_STOPPING;
code = TSDB_CODE_APP_IS_STOPPING;
}
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle);
return -1;
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), tstrerror(code),
pMsg->info.handle);
return code;
} else {
rpcSendRequest(pDnode->trans.syncRpc, pEpSet, pMsg, NULL);
return 0;
@ -426,7 +430,7 @@ int32_t dmInitStatusClient(SDnode *pDnode) {
pTrans->statusRpc = rpcOpen(&rpcInit);
if (pTrans->statusRpc == NULL) {
dError("failed to init dnode rpc status client");
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
dDebug("dnode rpc status client is initialized");
@ -471,7 +475,7 @@ int32_t dmInitSyncClient(SDnode *pDnode) {
pTrans->syncRpc = rpcOpen(&rpcInit);
if (pTrans->syncRpc == NULL) {
dError("failed to init dnode rpc sync client");
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
dDebug("dnode rpc sync client is initialized");
@ -521,7 +525,7 @@ int32_t dmInitServer(SDnode *pDnode) {
pTrans->serverRpc = rpcOpen(&rpcInit);
if (pTrans->serverRpc == NULL) {
dError("failed to init dnode rpc server");
return -1;
return terrno;
}
dDebug("dnode rpc server is initialized");

33
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -145,20 +145,20 @@ typedef struct {
} SDnodeData;
typedef struct {
const char *path;
const char *name;
STfs *pTfs;
SDnodeData *pData;
SMsgCb msgCb;
ProcessCreateNodeFp processCreateNodeFp;
const char *path;
const char *name;
STfs *pTfs;
SDnodeData *pData;
SMsgCb msgCb;
ProcessCreateNodeFp processCreateNodeFp;
ProcessAlterNodeTypeFp processAlterNodeTypeFp;
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordFp;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;
GetQnodeLoadsFp getQnodeLoadsFp;
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordFp;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;
GetQnodeLoadsFp getQnodeLoadsFp;
} SMgmtInputOpt;
typedef struct {
@ -203,9 +203,10 @@ void *dmSetMgmtHandle(SArray *pArray, tmsg_t msgType, void *nodeMsgFp, boo
void dmGetMonitorSystemInfo(SMonSysInfo *pInfo);
// dmFile.c
int32_t dmReadFile(const char *path, const char *name, bool *pDeployed);
int32_t dmWriteFile(const char *path, const char *name, bool deployed);
TdFilePtr dmCheckRunning(const char *dataDir);
int32_t dmReadFile(const char *path, const char *name, bool *pDeployed);
int32_t dmWriteFile(const char *path, const char *name, bool deployed);
int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile);
//int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile);
// dmodule.c
int32_t dmInitDndInfo(SDnodeData *pData);

122
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -95,18 +95,18 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) {
}
int dmOccurrences(char *str, char *toSearch) {
int count = 0;
char *ptr = str;
while ((ptr = strstr(ptr, toSearch)) != NULL) {
count++;
ptr++;
}
return count;
int count = 0;
char *ptr = str;
while ((ptr = strstr(ptr, toSearch)) != NULL) {
count++;
ptr++;
}
return count;
}
void dmSplitStr(char** arr, char* str, const char* del) {
void dmSplitStr(char **arr, char *str, const char *del) {
char *lasts;
char* s = strsep(&str, del);
char *s = strsep(&str, del);
while (s != NULL) {
*arr++ = s;
s = strsep(&str, del);
@ -132,11 +132,10 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("dnode file:%s not exist", file);
#ifdef TD_ENTERPRISE
if(strlen(tsEncryptAlgorithm) > 0){
if(strcmp(tsEncryptAlgorithm, "sm4") == 0) {
if (strlen(tsEncryptAlgorithm) > 0) {
if (strcmp(tsEncryptAlgorithm, "sm4") == 0) {
pData->encryptAlgorigthm = DND_CA_SM4;
}
else{
} else {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm);
goto _OVER;
@ -144,45 +143,45 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("start to parse encryptScope:%s", tsEncryptScope);
int32_t scopeLen = strlen(tsEncryptScope);
if(scopeLen == 0){
if (scopeLen == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER;
}
char* tmp = taosMemoryMalloc(scopeLen + 1);
char *tmp = taosMemoryMalloc(scopeLen + 1);
memset(tmp, 0, scopeLen + 1);
memcpy(tmp, tsEncryptScope, scopeLen);
int32_t count = dmOccurrences(tmp, ",");
char** array = taosMemoryMalloc(sizeof(char*) * (count + 1));
memset(array, 0, sizeof(char*) * (count + 1));
char **array = taosMemoryMalloc(sizeof(char *) * (count + 1));
memset(array, 0, sizeof(char *) * (count + 1));
dmSplitStr(array, tmp, ",");
for(int32_t i = 0; i < count + 1; i++){
char* str = *(array + i);
for (int32_t i = 0; i < count + 1; i++) {
char *str = *(array + i);
bool success = false;
if(strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0){
if (strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_TSDB;
success = true;
}
if(strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0){
if (strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_VNODE_WAL;
success = true;
}
if(strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0){
if (strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_SDB;
success = true;
}
if(strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0){
if (strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_MNODE_WAL;
success = true;
}
if(!success){
if (!success) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
taosMemoryFree(tmp);
taosMemoryFree(array);
@ -196,7 +195,7 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("set tsCryptAlgorithm:%s, tsCryptScope:%s from cfg", tsEncryptAlgorithm, tsEncryptScope);
}
#endif
code = 0;
goto _OVER;
@ -204,26 +203,26 @@ int32_t dmReadEps(SDnodeData *pData) {
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER;
}
@ -232,12 +231,11 @@ int32_t dmReadEps(SDnodeData *pData) {
pJson = tjsonParse(content);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (dmDecodeEps(pJson, pData) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
if ((code = dmDecodeEps(pJson, pData)) < 0) {
goto _OVER;
}
@ -251,7 +249,7 @@ _OVER:
if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr());
return code;
return terrno = code;
}
if (taosArrayGetSize(pData->dnodeEps) == 0) {
@ -261,8 +259,8 @@ _OVER:
taosArrayPush(pData->dnodeEps, &dnodeEp);
}
if (dmReadDnodePairs(pData) != 0) {
return -1;
if ((code = dmReadDnodePairs(pData)) != 0) {
return terrno = code;
}
dDebug("reset dnode list on startup");
@ -270,8 +268,8 @@ _OVER:
if (pData->oldDnodeEps == NULL && dmIsEpChanged(pData, pData->dnodeId, tsLocalEp)) {
dError("localEp %s different with %s and need to be reconfigured", tsLocalEp, file);
terrno = TSDB_CODE_INVALID_CFG;
return -1;
code = TSDB_CODE_INVALID_CFG;
return terrno = code;
}
return code;
@ -308,7 +306,7 @@ static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) {
}
int32_t dmWriteEps(SDnodeData *pData) {
int32_t code = -1;
int32_t code = 0;
char *buffer = NULL;
SJson *pJson = NULL;
TdFilePtr pFile = NULL;
@ -317,26 +315,30 @@ int32_t dmWriteEps(SDnodeData *pData) {
snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
terrno = TSDB_CODE_OUT_OF_MEMORY;
// if ((code == dmInitDndInfo(pData)) != 0) goto _OVER;
TAOS_CHECK_GOTO(dmInitDndInfo(pData), NULL, _OVER);
if ((code == dmInitDndInfo(pData)) != 0) goto _OVER;
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (pJson == NULL) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER);
pData->engineVer = tsVersion;
if (dmEncodeEps(pJson, pData) != 0) goto _OVER;
TAOS_CHECK_GOTO(dmEncodeEps(pJson, pData), NULL, _OVER); // dmEncodeEps(pJson, pData) != 0) goto _OVER;
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
terrno = 0;
if (buffer == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER);
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosRenameFile(file, realfile) != 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
code = 0;
pData->updateTime = taosGetTimestampMs();
@ -349,8 +351,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, terrstr(), pData->dnodeVer);
dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, tstrerror(code), pData->dnodeVer);
}
return code;
}
@ -592,28 +593,29 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dDebug("dnode file:%s not exist", file);
code = TAOS_SYSTEM_ERROR(errno);
dDebug("dnode file:%s not exist, reason:%s", file, tstrerror(code));
code = 0;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -627,12 +629,13 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
pJson = tjsonParse(content);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair));
if (pData->oldDnodeEps == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to calloc dnodeEp array since %s", strerror(errno));
goto _OVER;
}
@ -640,7 +643,8 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
if (dmDecodeEpPairs(pJson, pData) < 0) {
taosArrayDestroy(pData->oldDnodeEps);
pData->oldDnodeEps = NULL;
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
@ -653,7 +657,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr());
dError("failed to read dnode file:%s since %s", file, tstrerror(code));
return code;
}
@ -680,8 +684,8 @@ _OVER:
pair->oldPort, pair->newFqdn, pair->newPort, pDnodeEp->id);
taosArrayDestroy(pData->oldDnodeEps);
pData->oldDnodeEps = NULL;
terrno = TSDB_CODE_INVALID_CFG;
return -1;
code = TSDB_CODE_INVALID_CFG;
return code;
}
}
}

346
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -14,23 +14,23 @@
*/
#define _DEFAULT_SOURCE
#include "dmUtil.h"
#include "tjson.h"
#include "tgrant.h"
#include "crypt.h"
#include "dmUtil.h"
#include "tchecksum.h"
#include "tgrant.h"
#include "tjson.h"
#define MAXLEN 1024
#define DM_KEY_INDICATOR "this indicator!"
#define DM_ENCRYPT_CODE_FILE "encryptCode.cfg"
#define DM_CHECK_CODE_FILE "checkCode.bin"
#define MAXLEN 1024
#define DM_KEY_INDICATOR "this indicator!"
#define DM_ENCRYPT_CODE_FILE "encryptCode.cfg"
#define DM_CHECK_CODE_FILE "checkCode.bin"
static int32_t dmDecodeFile(SJson *pJson, bool *deployed) {
int32_t code = 0;
int32_t value = 0;
tjsonGetInt32ValueFromDouble(pJson, "deployed", value, code);
if (code < 0) return -1;
if (code < 0) return code;
*deployed = (value != 0);
return code;
@ -42,7 +42,11 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
char *content = NULL;
SJson *pJson = NULL;
char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
int32_t nBytes = snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("file:%s not exist", file);
@ -52,27 +56,27 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", file, tstrerror(code));
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat file:%s since %s", file, tstrerror(code));
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read file:%s since %s", file, tstrerror(code));
goto _OVER;
}
@ -80,12 +84,12 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
pJson = tjsonParse(content);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (dmDecodeFile(pJson, pDeployed) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
@ -98,13 +102,13 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr());
dError("failed to read dnode file:%s since %s", file, tstrerror(code));
}
return code;
}
static int32_t dmEncodeFile(SJson *pJson, bool deployed) {
if (tjsonAddDoubleToObject(pJson, "deployed", deployed) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "deployed", deployed) < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
return 0;
}
@ -115,26 +119,57 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) {
TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name);
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t nBytes = snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (dmEncodeFile(pJson, deployed) != 0) goto _OVER;
if (pJson == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if ((code = dmEncodeFile(pJson, deployed)) != 0) goto _OVER;
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
terrno = 0;
if (buffer == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, buffer, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosCloseFile(&pFile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0;
dInfo("succeed to write file:%s, deloyed:%d", realfile, deployed);
@ -145,54 +180,58 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write file:%s since %s, deloyed:%d", realfile, terrstr(), deployed);
dError("failed to write file:%s since %s, deloyed:%d", realfile, tstrerror(code), deployed);
}
return code;
}
TdFilePtr dmCheckRunning(const char *dataDir) {
char filepath[PATH_MAX] = {0};
int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile) {
int32_t code = 0;
char filepath[PATH_MAX] = {0};
snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP);
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", filepath, terrstr());
return NULL;
*pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC);
if (*pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", filepath, tstrerror(code));
return code;
}
int32_t retryTimes = 0;
int32_t ret = 0;
do {
ret = taosLockFile(pFile);
ret = taosLockFile(*pFile);
if (ret == 0) break;
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
taosMsleep(1000);
retryTimes++;
dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes);
dError("failed to lock file:%s since %s, retryTimes:%d", filepath, tstrerror(code), retryTimes);
} while (retryTimes < 12);
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile);
return NULL;
code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(pFile);
*pFile = NULL;
return code;
}
terrno = 0;
dDebug("lock file:%s to prevent repeated starts", filepath);
return pFile;
return code;
}
extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode);
static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool toLogFile){
static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool toLogFile) {
TdFilePtr pFile = NULL;
char *result = NULL;
int32_t code = -1;
int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR));
result = taosMemoryMalloc(len);
if (result == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SCryptOpts opts;
strncpy(opts.key, key, ENCRYPT_KEY_LEN);
@ -203,48 +242,81 @@ static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool
CBC_Encrypt(&opts);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosWriteFile(pFile, opts.result, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, opts.result, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosCloseFile(&pFile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
encryptDebug("succeed to write checkCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
if(result != NULL) taosMemoryFree(result);
if (pFile != NULL) taosCloseFile(&pFile);
if (result != NULL) taosMemoryFree(result);
return code;
}
static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode, bool toLogFile){
static int32_t dmWriteEncryptCodeFile(char *file, char *realfile, char *encryptCode, bool toLogFile) {
TdFilePtr pFile = NULL;
int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(encryptCode);
if (taosWriteFile(pFile, encryptCode, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, encryptCode, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosCloseFile(&pFile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
encryptDebug("succeed to write encryptCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
if (pFile != NULL) taosCloseFile(&pFile);
return code;
}
static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) {
char *content = NULL;
int64_t size = 0;
TdFilePtr pFile = NULL;
@ -253,31 +325,31 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to open dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to open dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to fstat dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to fstat dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
content = taosMemoryMalloc(size);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to read dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to read dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
encryptDebug("succeed to read checkCode file:%s", file);
int len = ENCRYPTED_LEN(size);
result = taosMemoryMalloc(len);
@ -289,8 +361,8 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
opts.unitLen = 16;
CBC_Decrypt(&opts);
if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
if (strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
code = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
encryptError("failed to compare decrypted result");
goto _OVER;
}
@ -298,9 +370,9 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
encryptDebug("succeed to compare checkCode file:%s", file);
code = 0;
_OVER:
if(result != NULL) taosMemoryFree(result);
if(content != NULL) taosMemoryFree(content);
if(pFile != NULL) taosCloseFile(&pFile);
if (result != NULL) taosMemoryFree(result);
if (content != NULL) taosMemoryFree(content);
if (pFile != NULL) taosCloseFile(&pFile);
return code;
}
@ -319,40 +391,57 @@ int32_t dmUpdateEncryptKey(char *key, bool toLogFile) {
char checkFile[PATH_MAX] = {0};
char realCheckFile[PATH_MAX] = {0};
snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP);
snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
int32_t nBytes = snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
terrno = 0;
nBytes = snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
nBytes = snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
nBytes = snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
if (taosMkDir(folder) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to create dir:%s since %s", folder, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to create dir:%s since %s", folder, tstrerror(code));
goto _OVER;
}
if(taosCheckExistFile(realCheckFile)){
if(dmCompareEncryptKey(realCheckFile, key, toLogFile) != 0){
if (taosCheckExistFile(realCheckFile)) {
if ((code = dmCompareEncryptKey(realCheckFile, key, toLogFile)) != 0) {
goto _OVER;
}
}
if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (generateEncryptCode(key, machineId, &encryptCode) != 0) {
if ((code = generateEncryptCode(key, machineId, &encryptCode)) != 0) {
goto _OVER;
}
if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile) != 0){
if ((code = dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile)) != 0) {
goto _OVER;
}
if(dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile) != 0){
if ((code = dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile)) != 0) {
goto _OVER;
}
@ -363,8 +452,7 @@ _OVER:
taosMemoryFree(encryptCode);
taosMemoryFree(machineId);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to update encrypt key since %s", terrstr());
encryptError("failed to update encrypt key since %s", tstrerror(code));
}
return code;
#else
@ -374,89 +462,102 @@ _OVER:
extern int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);
static int32_t dmReadEncryptCodeFile(char* file, char** output){
static int32_t dmReadEncryptCodeFile(char *file, char **output) {
TdFilePtr pFile = NULL;
int32_t code = -1;
char *content = NULL;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
content[size] = '\0';
*output = content;
content = NULL;
dInfo("succeed to read encryptCode file:%s", file);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
if (pFile != NULL) taosCloseFile(&pFile);
taosMemoryFree(content);
return code;
}
int32_t dmGetEncryptKey(){
int32_t dmGetEncryptKey() {
#ifdef TD_ENTERPRISE
int32_t code = -1;
char encryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
char *machineId = NULL;
char *encryptKey = NULL;
char *content = NULL;
int32_t code = -1;
char encryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
char *machineId = NULL;
char *encryptKey = NULL;
char *content = NULL;
snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE);
int32_t nBytes = snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP,
DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= sizeof(encryptFile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
return code;
}
if(!taosCheckExistFile(encryptFile)){
nBytes = snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= sizeof(checkFile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
return code;
}
if (!taosCheckExistFile(encryptFile)) {
code = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dInfo("no exist, checkCode file:%s", encryptFile);
return 0;
}
if(dmReadEncryptCodeFile(encryptFile, &content) != 0){
if ((code = dmReadEncryptCodeFile(encryptFile, &content)) != 0) {
goto _OVER;
}
if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if(checkAndGetCryptKey(content, machineId, &encryptKey) != 0){
if ((code = checkAndGetCryptKey(content, machineId, &encryptKey)) != 0) {
goto _OVER;
}
taosMemoryFreeClear(machineId);
taosMemoryFreeClear(content);
if(encryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("failed to read key since %s", terrstr());
if (encryptKey[0] == '\0') {
code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("failed to read key since %s", tstrerror(code));
goto _OVER;
}
if(dmCompareEncryptKey(checkFile, encryptKey, true) != 0){
if ((code = dmCompareEncryptKey(checkFile, encryptKey, true)) != 0) {
goto _OVER;
}
@ -471,8 +572,7 @@ _OVER:
if (encryptKey != NULL) taosMemoryFree(encryptKey);
if (machineId != NULL) taosMemoryFree(machineId);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to get encrypt key since %s", terrstr());
dError("failed to get encrypt key since %s", tstrerror(code));
}
return code;
#else

690
source/dnode/mnode/impl/src/mndDb.c
View File

File diff suppressed because it is too large Load Diff

267
source/dnode/mnode/impl/src/mndIndex.c
View File

@ -93,6 +93,7 @@ static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName,
}
int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
@ -110,18 +111,20 @@ int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStb
if (pReq == NULL) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_VND_CREATE_INDEX;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
return -1;
TAOS_RETURN(code);
}
sdbRelease(pSdb, pVgroup);
}
@ -164,6 +167,7 @@ _err:
return NULL;
}
int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
@ -182,6 +186,8 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
if (pReq == NULL) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
STransAction action = {0};
@ -189,7 +195,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
action.pCont = pReq;
action.contLen = len;
action.msgType = TDMT_VND_DROP_INDEX;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
@ -198,7 +204,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
sdbRelease(pSdb, pVgroup);
}
return 0;
TAOS_RETURN(code);
}
void mndCleanupIdx(SMnode *pMnode) {
@ -332,61 +338,91 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) {
}
int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
TAOS_CHECK_RETURN(mndTransAppendPrepareLog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
return 0;
TAOS_RETURN(code);
}
int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0;
TAOS_RETURN(code);
}
int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) {
sdbFreeRaw(pRedoRaw);
return -1;
}
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY));
return 0;
TAOS_RETURN(code);
}
int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
sdbFreeRaw(pCommitRaw);
return -1;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
sdbFreeRaw(pCommitRaw);
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0;
TAOS_RETURN(code);
}
static int32_t mndSetCreateIdxVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) {
int32_t code = 0;
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
if (pVgRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pVgRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
}
static int32_t mndSetCreateIdxVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) {
int32_t code = 0;
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pVgRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pVgRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
// static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
@ -427,10 +463,7 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
SDbObj *pDb = NULL;
SCreateTagIndexReq createReq = {0};
if (tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("idx:%s start to create", createReq.idxName);
// if (mndCheckCreateIdxReq(&createReq) != 0) {
@ -439,29 +472,30 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
pDb = mndAcquireDbByStb(pMnode, createReq.stbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
code = TSDB_CODE_MND_DB_NOT_EXIST;
if (terrno != 0) code = terrno;
goto _OVER;
}
pStb = mndAcquireStb(pMnode, createReq.stbName);
if (pStb == NULL) {
mError("idx:%s, failed to create since stb:%s not exist", createReq.idxName, createReq.stbName);
code = TSDB_CODE_MND_DB_NOT_EXIST;
if (terrno != 0) code = terrno;
goto _OVER;
}
SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx) == 0) {
if ((code = mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx)) == 0) {
pIdx = idx.pIdx;
} else {
goto _OVER;
}
if (pIdx != NULL) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
goto _OVER;
}
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER);
code = mndAddIndex(pMnode, pReq, &createReq, pDb, pStb);
if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) {
@ -472,32 +506,42 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("stb:%s, failed to create since %s", createReq.idxName, terrstr());
mError("stb:%s, failed to create since %s", createReq.idxName, tstrerror(code));
}
mndReleaseStb(pMnode, pStb);
mndReleaseIdx(pMnode, pIdx);
mndReleaseDb(pMnode, pDb);
return code;
TAOS_RETURN(code);
}
int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendPrepareLog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
return 0;
TAOS_RETURN(code);
}
int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
return 0;
TAOS_RETURN(code);
}
static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq) {
@ -592,6 +636,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) {
static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pOld, SStbObj *pNew, char *tagName,
int on) {
int32_t code = 0;
taosRLockLatch(&pOld->lock);
memcpy(pNew, pOld, sizeof(SStbObj));
taosRUnLockLatch(&pOld->lock);
@ -604,28 +649,24 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pOld, tagName, &hasIdx);
if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_NOT_EXIST;
TAOS_RETURN(code);
}
col_id_t colId = pOld->pTags[tag].colId;
if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) {
return -1;
}
if (mndAllocStbSchemas(pOld, pNew) != 0) {
return -1;
}
TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId));
TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew));
SSchema *pTag = pNew->pTags + tag;
if (on == 1) {
if (hasIdx && tag != 0) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
TAOS_RETURN(code);
} else {
SSCHMEA_SET_IDX_ON(pTag);
}
} else {
if (hasIdx == 0) {
terrno = TSDB_CODE_MND_SMA_NOT_EXIST;
code = TSDB_CODE_MND_SMA_NOT_EXIST;
} else {
SSCHMEA_SET_IDX_OFF(pTag);
pTag->flags = 0;
@ -634,11 +675,15 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
pNew->tagVer++;
SSdbRaw *pCommitRaw = mndStbActionEncode(pNew);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0;
TAOS_RETURN(code);
}
int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
// impl later
@ -649,17 +694,17 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt
// mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name);
mndTransSetDbName(pTrans, pDb->name, pStb->name);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
mndTransSetSerial(pTrans);
if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER;
if (mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
@ -671,7 +716,7 @@ _OVER:
taosMemoryFree(newStb.pCmpr);
}
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
int8_t mndCheckIndexNameByTagName(SMnode *pMnode, SIdxObj *pIdxObj) {
// build index on first tag, and no index name;
@ -725,25 +770,23 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pStb, req->colName, &hasIdx);
if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_NOT_EXIST;
TAOS_RETURN(code);
}
int8_t exist = 0;
if (tag == 0 && hasIdx == 1) {
exist = mndCheckIndexNameByTagName(pMnode, &idxObj);
if (exist) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
TAOS_RETURN(code);
}
} else if (hasIdx == 1) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
TAOS_RETURN(code);
}
col_id_t colId = pStb->pTags[tag].colId;
if (mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId) != 0) {
return -1;
}
TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId));
// SSchema *pTag = pStb->pTags + tag;
// if (IS_IDX_ON(pTag)) {
@ -752,7 +795,7 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
// }
code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj);
return code;
TAOS_RETURN(code);
}
static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *pIdx) {
@ -763,22 +806,30 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p
SStbObj newObj = {0};
pStb = mndAcquireStb(pMnode, pIdx->stb);
if (pStb == NULL) goto _OVER;
if (pStb == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-index");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name);
mndTransSetDbName(pTrans, pDb->name, NULL);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
mndTransSetSerial(pTrans);
if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER;
if (mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
@ -789,7 +840,7 @@ _OVER:
mndTransDrop(pTrans);
mndReleaseStb(pMnode, pStb);
return code;
TAOS_RETURN(code);
}
int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
@ -798,13 +849,10 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
SIdxObj *pIdx = NULL;
SDropTagIndexReq req = {0};
if (tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req), NULL, _OVER);
mInfo("idx:%s, start to drop", req.name);
SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx) == 0) {
if ((code = mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx)) == 0) {
pIdx = idx.pIdx;
} else {
goto _OVER;
@ -815,7 +863,7 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
code = 0;
goto _OVER;
} else {
terrno = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST;
code = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST;
goto _OVER;
}
}
@ -823,23 +871,22 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
pDb = mndAcquireDbByIdx(pMnode, req.name);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
if (terrno != 0) code = terrno;
goto _OVER;
}
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER);
code = mndDropIdx(pMnode, pReq, pDb, pIdx);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("idx:%s, failed to drop since %s", req.name, terrstr());
mError("idx:%s, failed to drop since %s", req.name, tstrerror(code));
}
mndReleaseIdx(pMnode, pIdx);
mndReleaseDb(pMnode, pDb);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) {
// do nothing
@ -847,6 +894,7 @@ static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) {
}
int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
@ -856,17 +904,17 @@ int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
if (pIter == NULL) break;
if (pIdx->stbUid == pStb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) {
if ((code = mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx)) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pIdx);
return -1;
TAOS_RETURN(code);
}
}
sdbRelease(pSdb, pIdx);
}
return 0;
TAOS_RETURN(code);
}
int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *idx) {
@ -891,6 +939,7 @@ int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxOb
return -1;
}
int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
@ -900,15 +949,15 @@ int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
if (pIter == NULL) break;
if (pIdx->dbUid == pDb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) {
if ((code = mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx)) != 0) {
sdbRelease(pSdb, pIdx);
sdbCancelFetch(pSdb, pIter);
return -1;
TAOS_RETURN(code);
}
}
sdbRelease(pSdb, pIdx);
}
return 0;
TAOS_RETURN(code);
}

12
source/dnode/mnode/impl/src/mndIndexCom.c
View File

@ -30,7 +30,7 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
SSmaObj *pSma = mndGetIdx(pMnode, name, SDB_SMA);
SIdxObj *pIdx = mndGetIdx(pMnode, name, SDB_IDX);
terrno = 0;
int32_t code = 0;
if (pSma == NULL && pIdx == NULL) return 0;
@ -40,8 +40,8 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
idx->pIdx = pSma;
} else {
mndReleaseSma(pMnode, pSma);
terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_SMA_ALREADY_EXIST;
TAOS_RETURN(code);
}
} else {
if (type == SDB_IDX) {
@ -49,9 +49,9 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
idx->pIdx = pIdx;
} else {
mndReleaseIdx(pMnode, pIdx);
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
TAOS_RETURN(code);
}
}
return 0;
TAOS_RETURN(code);
}

55
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -18,10 +18,11 @@
#include "systable.h"
static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
int32_t code = 0;
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
for (int32_t i = 0; i < colNum; ++i) {
@ -35,10 +36,11 @@ static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t co
}
*pDst = schema;
return 0;
TAOS_RETURN(code);
}
static int32_t mndInsInitMeta(SHashObj *hash) {
int32_t code = 0;
STableMetaRsp meta = {0};
tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName));
@ -55,24 +57,23 @@ static int32_t mndInsInitMeta(SHashObj *hash) {
meta.numOfColumns = pInfosTableMeta[i].colNum;
meta.sysInfo = pInfosTableMeta[i].sysInfo;
if (mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas)) {
return -1;
}
TAOS_CHECK_RETURN(mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas));
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
}
return 0;
TAOS_RETURN(code);
}
int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, bool sysinfo,
STableMetaRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->infosMeta) {
terrno = TSDB_CODE_APP_ERROR;
return -1;
code = TSDB_CODE_APP_ERROR;
TAOS_RETURN(code);
}
STableMetaRsp *pMeta = NULL;
@ -84,40 +85,41 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
TAOS_RETURN(code);
}
if (!sysinfo && pMeta->sysInfo) {
mError("no permission to get schema of table name:%s", tbName);
terrno = TSDB_CODE_PAR_PERMISSION_DENIED;
return -1;
code = TSDB_CODE_PAR_PERMISSION_DENIED;
TAOS_RETURN(code);
}
*pRsp = *pMeta;
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL;
return -1;
TAOS_RETURN(code);
}
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
return 0;
TAOS_RETURN(code);
}
int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->infosMeta) {
terrno = TSDB_CODE_APP_ERROR;
return -1;
code = TSDB_CODE_APP_ERROR;
TAOS_RETURN(code);
}
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
TAOS_RETURN(code);
}
strcpy(pRsp->tbName, pMeta->tbName);
@ -129,22 +131,21 @@ int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbN
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL;
return -1;
TAOS_RETURN(code);
}
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
pRsp->pSchemaExt = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchemaExt));
return 0;
TAOS_RETURN(code);
}
int32_t mndInitInfos(SMnode *pMnode) {
pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pMnode->infosMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
return mndInsInitMeta(pMnode->infosMeta);

212
source/dnode/mnode/impl/src/mndMain.c
View File

@ -50,8 +50,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0;
taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) {
terrno = TSDB_CODE_APP_IS_STOPPING;
code = -1;
code = TSDB_CODE_APP_IS_STOPPING;
} else if (!mndIsLeader(pMnode)) {
code = -1;
} else {
@ -63,7 +62,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
#endif
}
taosThreadRwlockUnlock(&pMnode->lock);
return code;
TAOS_RETURN(code);
}
static inline void mndReleaseRpc(SMnode *pMnode) {
@ -78,6 +77,7 @@ static inline void mndReleaseRpc(SMnode *pMnode) {
}
static void *mndBuildTimerMsg(int32_t *pContLen) {
terrno = 0;
SMTimerReq timerReq = {0};
int32_t contLen = tSerializeSMTimerMsg(NULL, 0, &timerReq);
@ -329,6 +329,7 @@ static int32_t minCronTime() {
return min <= 1 ? 2 : min;
}
void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
int32_t code = 0;
if (sec % tsTtlPushIntervalSec == 0) {
mndPullupTtl(pMnode);
}
@ -378,14 +379,14 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
}
if (sec % (tsArbHeartBeatIntervalSec) == 0) {
if (mndPullupArbHeartbeat(pMnode) != 0) {
mError("failed to pullup arb heartbeat, since:%s", terrstr());
if ((code = mndPullupArbHeartbeat(pMnode)) != 0) {
mError("failed to pullup arb heartbeat, since:%s", tstrerror(code));
}
}
if (sec % (tsArbCheckSyncIntervalSec) == 0) {
if (mndPullupArbCheckSync(pMnode) != 0) {
mError("failed to pullup arb check sync, since:%s", terrstr());
if ((code = mndPullupArbCheckSync(pMnode)) != 0) {
mError("failed to pullup arb check sync, since:%s", tstrerror(code));
}
}
}
@ -426,17 +427,18 @@ static void *mndThreadFp(void *param) {
}
static int32_t mndInitTimer(SMnode *pMnode) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode) != 0) {
if ((code = taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode)) != 0) {
mError("failed to create timer thread since %s", strerror(errno));
return -1;
TAOS_RETURN(code);
}
taosThreadAttrDestroy(&thAttr);
tmsgReportStartup("mnode-timer", "initialized");
return 0;
TAOS_RETURN(code);
}
static void mndCleanupTimer(SMnode *pMnode) {
@ -447,41 +449,41 @@ static void mndCleanupTimer(SMnode *pMnode) {
}
static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
int32_t code = 0;
pMnode->path = taosStrdup(path);
if (pMnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
if (taosMkDir(pMnode->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndInitWal(SMnode *pMnode) {
int32_t code = 0;
char path[PATH_MAX + 20] = {0};
snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
SWalCfg cfg = {
.vgId = 1,
.fsyncPeriod = 0,
.rollPeriod = -1,
.segSize = -1,
.retentionPeriod = 0,
.retentionSize = 0,
.level = TAOS_WAL_FSYNC,
.encryptAlgorithm = 0,
.encryptKey = {0}
};
SWalCfg cfg = {.vgId = 1,
.fsyncPeriod = 0,
.rollPeriod = -1,
.segSize = -1,
.retentionPeriod = 0,
.retentionSize = 0,
.level = TAOS_WAL_FSYNC,
.encryptAlgorithm = 0,
.encryptKey = {0}};
#if defined(TD_ENTERPRISE)
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0;
if(tsEncryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
TAOS_RETURN(code);
}
else{
strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
@ -491,11 +493,13 @@ static int32_t mndInitWal(SMnode *pMnode) {
pMnode->pWal = walOpen(path, &cfg);
if (pMnode->pWal == NULL) {
mError("failed to open wal since %s. wal:%s", terrstr(), path);
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mError("failed to open wal since %s. wal:%s", tstrerror(code), path);
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static void mndCloseWal(SMnode *pMnode) {
@ -506,6 +510,7 @@ static void mndCloseWal(SMnode *pMnode) {
}
static int32_t mndInitSdb(SMnode *pMnode) {
int32_t code = 0;
SSdbOpt opt = {0};
opt.path = pMnode->path;
opt.pMnode = pMnode;
@ -513,10 +518,12 @@ static int32_t mndInitSdb(SMnode *pMnode) {
pMnode->pSdb = sdbInit(&opt);
if (pMnode->pSdb == NULL) {
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndOpenSdb(SMnode *pMnode) {
@ -542,48 +549,47 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle
step.initFp = initFp;
step.cleanupFp = cleanupFp;
if (taosArrayPush(pMnode->pSteps, &step) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
return 0;
TAOS_RETURN(0);
}
static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem) != 0) return -1;
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem));
return 0;
}
@ -609,17 +615,16 @@ static void mndCleanupSteps(SMnode *pMnode, int32_t pos) {
}
static int32_t mndExecSteps(SMnode *pMnode) {
int32_t code = 0;
int32_t size = taosArrayGetSize(pMnode->pSteps);
for (int32_t pos = 0; pos < size; pos++) {
SMnodeStep *pStep = taosArrayGet(pMnode->pSteps, pos);
if (pStep->initFp == NULL) continue;
if ((*pStep->initFp)(pMnode) != 0) {
int32_t code = terrno;
mError("%s exec failed since %s, start to cleanup", pStep->name, terrstr());
if ((code = (*pStep->initFp)(pMnode)) != 0) {
mError("%s exec failed since %s, start to cleanup", pStep->name, tstrerror(code));
mndCleanupSteps(pMnode, pos);
terrno = code;
return -1;
TAOS_RETURN(code);
} else {
mInfo("%s is initialized", pStep->name);
tmsgReportStartup(pStep->name, "initialized");
@ -627,7 +632,7 @@ static int32_t mndExecSteps(SMnode *pMnode) {
}
pMnode->clusterId = mndGetClusterId(pMnode);
return 0;
TAOS_RETURN(0);
}
static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
@ -642,6 +647,7 @@ static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
}
SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
terrno = 0;
mInfo("start to open mnode in %s", path);
SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode));
@ -668,7 +674,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
int32_t code = mndCreateDir(pMnode, path);
if (code != 0) {
code = terrno;
mError("failed to open mnode since %s", terrstr());
mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode);
terrno = code;
return NULL;
@ -677,7 +683,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
code = mndInitSteps(pMnode);
if (code != 0) {
code = terrno;
mError("failed to open mnode since %s", terrstr());
mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode);
terrno = code;
return NULL;
@ -686,7 +692,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
code = mndExecSteps(pMnode);
if (code != 0) {
code = terrno;
mError("failed to open mnode since %s", terrstr());
mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode);
terrno = code;
return NULL;
@ -762,45 +768,47 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
int32_t code = syncProcessMsg(pMgmt->sync, pMsg);
if (code != 0) {
mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType),
terrstr(), code);
tstrerror(code), code);
}
return code;
}
static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0;
int32_t code = 0;
if (!IsReq(pMsg)) TAOS_RETURN(code);
if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY ||
pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT ||
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK ||
pMsg->msgType == TDMT_SCH_TASK_NOTIFY) {
return 0;
TAOS_RETURN(code);
}
SMnode *pMnode = pMsg->info.node;
taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) {
taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_APP_IS_STOPPING;
return -1;
code = TSDB_CODE_APP_IS_STOPPING;
TAOS_RETURN(code);
}
terrno = 0;
SSyncState state = syncGetState(pMnode->syncMgmt.sync);
if (terrno != 0) {
taosThreadRwlockUnlock(&pMnode->lock);
return -1;
code = terrno;
TAOS_RETURN(code);
}
if (state.state != TAOS_SYNC_STATE_LEADER) {
taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_SYN_NOT_LEADER;
code = TSDB_CODE_SYN_NOT_LEADER;
goto _OVER;
}
if (!state.restored || !pMnode->restored) {
taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_SYN_RESTORING;
code = TSDB_CODE_SYN_RESTORING;
goto _OVER;
}
@ -812,7 +820,7 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
#endif
taosThreadRwlockUnlock(&pMnode->lock);
return 0;
TAOS_RETURN(code);
_OVER:
if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
@ -824,19 +832,17 @@ _OVER:
pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state));
return -1;
TAOS_RETURN(code);
}
const STraceId *trace = &pMsg->info.traceId;
SEpSet epSet = {0};
int32_t tmpCode = terrno;
mndGetMnodeEpSet(pMnode, &epSet);
terrno = tmpCode;
mGDebug(
"msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d "
"role:%s, redirect numOfEps:%d inUse:%d, type:%s",
pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored,
pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code), pMnode->restored, pMnode->stopped, state.restored,
syncStr(state.state), epSet.numOfEps, epSet.inUse, TMSG_INFO(pMsg->msgType));
if (epSet.numOfEps <= 0) return -1;
@ -853,10 +859,10 @@ _OVER:
pMsg->info.rspLen = contLen;
}
return -1;
TAOS_RETURN(code);
}
int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) {
int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
SMnode *pMnode = pMsg->info.node;
const STraceId *trace = &pMsg->info.traceId;
int32_t code = TSDB_CODE_SUCCESS;
@ -867,12 +873,12 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) {
fpExt = pMnode->msgFpExt[TMSG_INDEX(pMsg->msgType)];
if (fpExt == NULL) {
mGError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
return -1;
code = TSDB_CODE_MSG_NOT_PROCESSED;
TAOS_RETURN(code);
}
}
if (mndCheckMnodeState(pMsg) != 0) return -1;
TAOS_CHECK_RETURN(mndCheckMnodeState(pMsg));
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
if (fp)
@ -886,6 +892,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) {
} else if (code == 0) {
mGTrace("msg:%p, successfully processed", pMsg);
} else {
//TODO removve this wrong set code
if (code == -1) {
code = terrno;
}
@ -893,7 +900,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) {
TMSG_INFO(pMsg->msgType));
}
return code;
TAOS_RETURN(code);
}
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
@ -925,7 +932,8 @@ int64_t mndGenerateUid(const char *name, int32_t len) {
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
if (mndAcquireRpc(pMnode) != 0) return -1;
int32_t code = 0;
TAOS_CHECK_RETURN(mndAcquireRpc(pMnode));
SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs();
@ -937,7 +945,9 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
pStbInfo->stbs == NULL) {
mndReleaseRpc(pMnode);
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
// cluster info
@ -1063,7 +1073,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
}
mndReleaseRpc(pMnode);
return 0;
TAOS_RETURN(code);
}
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {

256
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -69,6 +69,7 @@ int32_t mndInitMnode(SMnode *pMnode) {
void mndCleanupMnode(SMnode *pMnode) {}
SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) {
terrno = 0;
SMnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_MNODE, &mnodeId);
if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_MNODE_NOT_EXIST;
@ -82,13 +83,18 @@ void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj) {
}
static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
int32_t code = 0;
SMnodeObj mnodeObj = {0};
mnodeObj.id = 1;
mnodeObj.createdTime = taosGetTimestampMs();
mnodeObj.updateTime = mnodeObj.createdTime;
SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj);
if (pRaw == NULL) return -1;
if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw);
@ -97,25 +103,27 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
if (pTrans == NULL) {
sdbFreeRaw(pRaw);
mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr());
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
mInfo("trans:%d, used to create mnode:%d", pTrans->id, mnodeObj.id);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
TAOS_RETURN(code);
}
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) {
@ -188,16 +196,19 @@ _OVER:
}
static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
int32_t code = 0;
mTrace("mnode:%d, perform insert action, row:%p", pObj->id, pObj);
pObj->pDnode = sdbAcquireNotReadyObj(pSdb, SDB_DNODE, &pObj->id);
if (pObj->pDnode == NULL) {
mError("mnode:%d, failed to perform insert action since %s", pObj->id, terrstr());
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
int32_t code = 0;
}
pObj->syncState = TAOS_SYNC_STATE_OFFLINE;
mndReloadSyncConfig(pSdb->pMnode);
return 0;
TAOS_RETURN(code);
}
static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) {
@ -271,38 +282,59 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
}
static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
}
int32_t mndSetRestoreCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
static int32_t mndSetCreateMnodeUndoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndMnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq);
@ -315,15 +347,16 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p
.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeTypeReq *pAlterMnodeTypeReq,
SEpSet *pAlterMnodeTypeEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq);
@ -337,14 +370,15 @@ static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeType
.acceptableCode = TSDB_CODE_MNODE_ALREADY_IS_VOTER,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq);
@ -357,15 +391,16 @@ static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pA
.acceptableCode = 0,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq);
@ -378,11 +413,11 @@ static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDrop
.acceptableCode = TSDB_CODE_MNODE_NOT_DEPLOYED,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -426,9 +461,9 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset));
return 0;
TAOS_RETURN(0);
}
int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -474,9 +509,9 @@ int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset));
return 0;
TAOS_RETURN(0);
}
static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -517,9 +552,9 @@ static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset));
return 0;
TAOS_RETURN(0);
}
int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -565,19 +600,23 @@ int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset));
return 0;
TAOS_RETURN(0);
}
static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
SMnodeObj mnodeObj = {0};
mnodeObj.id = pDnode->id;
@ -586,8 +625,8 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mnodeObj.role = TAOS_SYNC_ROLE_LEARNER;
mnodeObj.lastIndex = pMnode->applied;
if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj), NULL, _OVER);
SMnodeObj mnodeLeaderObj = {0};
mnodeLeaderObj.id = pDnode->id;
@ -596,15 +635,15 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mnodeLeaderObj.role = TAOS_SYNC_ROLE_VOTER;
mnodeLeaderObj.lastIndex = pMnode->applied + 1;
if (mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj) != 0) goto _OVER;
if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
@ -614,19 +653,14 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL;
SMCreateMnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("mnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE), NULL, _OVER);
pObj = mndAcquireMnode(pMnode, createReq.dnodeId);
if (pObj != NULL) {
terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST;
code = TSDB_CODE_MND_MNODE_ALREADY_EXIST;
goto _OVER;
} else if (terrno != TSDB_CODE_MND_MNODE_NOT_EXIST) {
goto _OVER;
@ -634,17 +668,17 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER;
}
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) {
terrno = TSDB_CODE_MND_TOO_MANY_MNODES;
code = TSDB_CODE_MND_TOO_MANY_MNODES;
goto _OVER;
}
if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
terrno = TSDB_CODE_DNODE_OFFLINE;
code = TSDB_CODE_DNODE_OFFLINE;
goto _OVER;
}
@ -665,27 +699,38 @@ _OVER:
mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndSetDropMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
}
static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj,
bool force) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SDDropMnodeReq dropReq = {0};
@ -700,32 +745,32 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
if (totalMnodes == 2) {
if (force) {
mError("cant't force drop dnode, since a mnode on it and replica is 2");
terrno = TSDB_CODE_MNODE_ONLY_TWO_MNODE;
return -1;
code = TSDB_CODE_MNODE_ONLY_TWO_MNODE;
TAOS_RETURN(code);
}
mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj));
if (!force) {
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet));
}
} else if (totalMnodes == 3) {
mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes);
if (!force) {
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet));
}
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj));
} else {
return -1;
TAOS_RETURN(-1);
}
return 0;
TAOS_RETURN(code);
}
int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) {
if (pObj == NULL) return 0;
pObj->lastIndex = pMnode->applied;
if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force) != 0) return -1;
if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force));
TAOS_CHECK_RETURN(mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj));
return 0;
}
@ -734,19 +779,23 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) {
STrans *pTrans = NULL;
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-mnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
@ -755,38 +804,35 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
SMnodeObj *pObj = NULL;
SMDropMnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
mInfo("mnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE), NULL, _OVER);
if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG;
code = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
pObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
if (pMnode->selfDnodeId == dropReq.dnodeId) {
terrno = TSDB_CODE_MND_CANT_DROP_LEADER;
code = TSDB_CODE_MND_CANT_DROP_LEADER;
goto _OVER;
}
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
terrno = TSDB_CODE_MND_TOO_FEW_MNODES;
code = TSDB_CODE_MND_TOO_FEW_MNODES;
goto _OVER;
}
if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) {
terrno = TSDB_CODE_DNODE_OFFLINE;
code = TSDB_CODE_DNODE_OFFLINE;
goto _OVER;
}
@ -805,7 +851,7 @@ _OVER:
mndReleaseMnode(pMnode, pObj);
tFreeSMCreateQnodeReq(&dropReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
@ -892,13 +938,11 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
#if 1
return 0;
#else
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SDAlterMnodeReq alterReq = {0};
if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
TAOS_CHECK_RETURN(tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq));
SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1};
memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas));
@ -913,9 +957,9 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
return 0;
}
if (mndWriteFile(pMnode->path, &option) != 0) {
if ((code = mndWriteFile(pMnode->path, &option)) != 0) {
mError("failed to write mnode file since %s", terrstr());
return -1;
TAOS_RETURN(code);
}
SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1};
@ -939,14 +983,14 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
}
}
int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
if (code != 0) {
mError("failed to sync reconfig since %s", terrstr());
} else {
mInfo("alter mnode sync success");
}
return code;
TAOS_RETURN(code);
#endif
}

53
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -19,10 +19,11 @@
// connection/application/
int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
int32_t code = 0;
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
for (int32_t i = 0; i < colNum; ++i) {
@ -34,10 +35,11 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S
}
*pDst = schema;
return TSDB_CODE_SUCCESS;
TAOS_RETURN(code);
}
int32_t mndPerfsInitMeta(SHashObj *hash) {
int32_t code = 0;
STableMetaRsp meta = {0};
tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName));
@ -53,56 +55,56 @@ int32_t mndPerfsInitMeta(SHashObj *hash) {
tstrncpy(meta.tbName, pSysDbTableMeta[i].name, sizeof(meta.tbName));
meta.numOfColumns = pSysDbTableMeta[i].colNum;
if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) {
return -1;
}
TAOS_CHECK_RETURN(mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas));
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
}
return TSDB_CODE_SUCCESS;
TAOS_RETURN(code);
}
int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->perfsMeta) {
terrno = TSDB_CODE_APP_ERROR;
return -1;
code = TSDB_CODE_APP_ERROR;
TAOS_RETURN(code);
}
STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == meta) {
mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
TAOS_RETURN(code);
}
*pRsp = *meta;
pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL;
return -1;
TAOS_RETURN(code);
}
memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema));
return 0;
TAOS_RETURN(code);
}
int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->perfsMeta) {
terrno = TSDB_CODE_APP_ERROR;
return -1;
code = TSDB_CODE_APP_ERROR;
TAOS_RETURN(code);
}
STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == pMeta) {
mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
TAOS_RETURN(code);
}
strcpy(pRsp->tbName, pMeta->tbName);
@ -114,20 +116,21 @@ int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *t
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL;
return -1;
TAOS_RETURN(code);
}
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
return 0;
TAOS_RETURN(code);
}
int32_t mndInitPerfs(SMnode *pMnode) {
int32_t code = 0;
pMnode->perfsMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pMnode->perfsMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
return mndPerfsInitMeta(pMnode->perfsMeta);

77
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -85,22 +85,23 @@ static void mndCancelGetNextApp(SMnode *pMnode, void *pIter);
static int32_t mndProcessSvrVerReq(SRpcMsg *pReq);
int32_t mndInitProfile(SMnode *pMnode) {
int32_t code = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
// in ms
int32_t checkTime = tsShellActivityTimer * 2 * 1000;
pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn");
if (pMgmt->connCache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc profile cache since %s", terrstr());
return -1;
TAOS_RETURN(code);
}
pMgmt->appCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, checkTime, true, (__cache_free_fn_t)mndFreeApp, "app");
if (pMgmt->appCache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc profile cache since %s", terrstr());
return -1;
TAOS_RETURN(code);
}
mndSetMsgHandle(pMnode, TDMT_MND_HEARTBEAT, mndProcessHeartBeatReq);
@ -116,7 +117,7 @@ int32_t mndInitProfile(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndRetrieveApps);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndCancelGetNextApp);
return 0;
TAOS_RETURN(code);
}
void mndCleanupProfile(SMnode *pMnode) {
@ -384,6 +385,7 @@ static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq *pReq)
static void mndFreeApp(SAppObj *pApp) { mTrace("app %" PRIx64 " is destroyed", pApp->appId); }
static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) {
terrno = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SAppObj *pApp = taosCacheAcquireByKey(pMgmt->appCache, &appId, sizeof(appId));
@ -431,13 +433,16 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) {
}
static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) {
int32_t code = 0;
SAppHbReq *pReq = &pHbReq->app;
SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId);
if (pApp == NULL) {
pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq);
if (pApp == NULL) {
mError("failed to create new app %" PRIx64 " since %s", pReq->appId, terrstr());
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} else {
mDebug("a new app %" PRIx64 " is created", pReq->appId);
mndReleaseApp(pMnode, pApp);
@ -475,6 +480,7 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) {
static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq,
SClientHbBatchRsp *pBatchRsp, SConnPreparedObj *pObj) {
int32_t code = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
SRpcConnInfo connInfo = pMsg->info.conn;
@ -492,7 +498,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
pHbReq->app.pid, pHbReq->app.name, 0);
if (pConn == NULL) {
mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr());
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} else {
mDebug("user:%s, conn:%u is freed, will create a new conn:%u", connInfo.user, pBasic->connId, pConn->id);
}
@ -501,9 +509,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic));
if (rspBasic == NULL) {
mndReleaseConn(pMnode, pConn, true);
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr());
return -1;
TAOS_RETURN(code);
}
mndSaveQueryList(pConn, pBasic);
@ -539,9 +547,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
hbRsp.info = taosArrayInit(kvNum, sizeof(SKv));
if (NULL == hbRsp.info) {
mError("taosArrayInit %d rsp kv failed", kvNum);
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
tFreeClientHbRsp(&hbRsp);
return -1;
TAOS_RETURN(code);
}
#ifdef TD_ENTERPRISE
@ -554,8 +562,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer);
SDynViewVersion *pRspVer = NULL;
if (0 != mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer)) {
return -1;
if (0 != (code = mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer))) {
TAOS_RETURN(code);
}
if (needCheck) {
@ -647,13 +655,14 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
}
static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SClientHbBatchReq batchReq = {0};
if (tDeserializeSClientHbBatchReq(pReq->pCont, pReq->contLen, &batchReq) != 0) {
taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
TAOS_RETURN(code);
}
SConnPreparedObj obj = {0};
@ -699,31 +708,27 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
taosArrayDestroy(obj.pQnodeList);
return 0;
TAOS_RETURN(code);
}
static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SKillQueryReq killReq = {0};
if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
TAOS_CHECK_RETURN(tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq));
mInfo("kill query msg is received, queryId:%s", killReq.queryStrId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) {
return -1;
}
TAOS_CHECK_RETURN(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY));
int32_t connId = 0;
uint64_t queryId = 0;
char *p = strchr(killReq.queryStrId, ':');
if (NULL == p) {
mError("invalid query id %s", killReq.queryStrId);
terrno = TSDB_CODE_MND_INVALID_QUERY_ID;
return -1;
code = TSDB_CODE_MND_INVALID_QUERY_ID;
TAOS_RETURN(code);
}
*p = 0;
connId = taosStr2Int32(killReq.queryStrId, NULL, 16);
@ -732,40 +737,36 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &connId, sizeof(int32_t));
if (pConn == NULL) {
mError("connId:%x, failed to kill queryId:%" PRIx64 ", conn not exist", connId, queryId);
terrno = TSDB_CODE_MND_INVALID_CONN_ID;
return -1;
code = TSDB_CODE_MND_INVALID_CONN_ID;
TAOS_RETURN(code);
} else {
mInfo("connId:%x, queryId:%" PRIx64 " is killed by user:%s", connId, queryId, pReq->info.conn.user);
pConn->killId = queryId;
taosCacheRelease(pMgmt->connCache, (void **)&pConn, false);
return 0;
TAOS_RETURN(code);
}
}
static int32_t mndProcessKillConnReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SKillConnReq killReq = {0};
if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
TAOS_CHECK_RETURN(tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq));
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) {
return -1;
}
TAOS_CHECK_RETURN(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN));
SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t));
if (pConn == NULL) {
mError("connId:%u, failed to kill connection, conn not exist", killReq.connId);
terrno = TSDB_CODE_MND_INVALID_CONN_ID;
return -1;
code = TSDB_CODE_MND_INVALID_CONN_ID;
TAOS_RETURN(code);
} else {
mInfo("connId:%u, is killed by user:%s", killReq.connId, pReq->info.conn.user);
pConn->killed = 1;
taosCacheRelease(pMgmt->connCache, (void **)&pConn, false);
return TSDB_CODE_SUCCESS;
TAOS_RETURN(code);
}
}

190
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -138,15 +138,16 @@ _OVER:
}
static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj) {
int32_t code = 0;
mTrace("qnode:%d, perform insert action, row:%p", pObj->id, pObj);
pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id);
if (pObj->pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
mError("qnode:%d, failed to perform insert action since %s", pObj->id, terrstr());
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) {
@ -166,27 +167,42 @@ static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew
}
static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
}
static int32_t mndSetCreateQnodeUndoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndQnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
@ -194,6 +210,7 @@ bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
}
int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDCreateQnodeReq createReq = {0};
createReq.dnodeId = pDnode->id;
@ -212,23 +229,24 @@ int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeOb
action.msgType = TDMT_DND_CREATE_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -239,12 +257,12 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_DROP_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) {
@ -256,22 +274,26 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
qnodeObj.updateTime = qnodeObj.createdTime;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER;
if (mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj) != 0) goto _OVER;
if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) goto _OVER;
if (mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj) != 0) goto _OVER;
if (mndSetCreateQnodeUndoActions(pTrans, pDnode, &qnodeObj) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateQnodeUndoActions(pTrans, pDnode, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
@ -281,19 +303,14 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL;
SMCreateQnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("qnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE), NULL, _OVER);
pObj = mndAcquireQnode(pMnode, createReq.dnodeId);
if (pObj != NULL) {
terrno = TSDB_CODE_MND_QNODE_ALREADY_EXIST;
code = TSDB_CODE_MND_QNODE_ALREADY_EXIST;
goto _OVER;
} else if (terrno != TSDB_CODE_MND_QNODE_NOT_EXIST) {
goto _OVER;
@ -301,7 +318,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER;
}
@ -320,34 +337,47 @@ _OVER:
mndReleaseQnode(pMnode, pObj);
mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
}
static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if (mndTransAppendCommitlog(pTrans, pCommitRaw))
;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED))
;
TAOS_RETURN(code);
}
static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -358,20 +388,20 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn
action.msgType = TDMT_DND_DROP_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj, bool force) {
if (pObj == NULL) return 0;
if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) return -1;
if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropQnodeRedoLogs(pTrans, pObj));
TAOS_CHECK_RETURN(mndSetDropQnodeCommitLogs(pTrans, pObj));
if (!force) {
if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj));
}
return 0;
}
@ -380,18 +410,22 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
@ -400,23 +434,20 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
SQnodeObj *pObj = NULL;
SMDropQnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
mInfo("qnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE), NULL, _OVER);
if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG;
code = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
pObj = mndAcquireQnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
@ -435,10 +466,11 @@ _OVER:
mndReleaseQnode(pMnode, pObj);
tFreeSDDropQnodeReq(&dropReq);
return code;
TAOS_RETURN(code);
}
int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SQnodeObj *pObj = NULL;
@ -447,8 +479,8 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad));
if (NULL == qnodeList) {
mError("failed to alloc epSet while process qnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
while (1) {
@ -484,20 +516,14 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
SQnodeListReq qlistReq = {0};
SQnodeListRsp qlistRsp = {0};
if (tDeserializeSQnodeListReq(pReq->pCont, pReq->contLen, &qlistReq) != 0) {
mError("failed to parse qnode list req");
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSQnodeListReq(pReq->pCont, pReq->contLen, &qlistReq), NULL, _OVER);
if (mndCreateQnodeList(pMnode, &qlistRsp.qnodeList, qlistReq.rowNum) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCreateQnodeList(pMnode, &qlistRsp.qnodeList, qlistReq.rowNum), NULL, _OVER);
int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -509,7 +535,7 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
_OVER:
tFreeSQnodeListRsp(&qlistRsp);
return code;
TAOS_RETURN(code);
}
static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

6
source/dnode/mnode/impl/src/mndQuery.c
View File

@ -88,7 +88,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
void *pRsp = NULL;
SMnode *pMnode = pMsg->info.node;
if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) {
if ((code = tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
mError("tDeserializeSBatchReq failed");
goto _exit;
@ -119,7 +119,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(req->msgType)];
if (fp == NULL) {
mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
code = TSDB_CODE_MSG_NOT_PROCESSED;
taosArrayDestroy(batchRsp.pRsps);
return -1;
}
@ -164,7 +164,7 @@ _exit:
taosArrayDestroyEx(batchReq.pMsgs, tFreeSBatchReqMsg);
taosArrayDestroyEx(batchRsp.pRsps, mnodeFreeSBatchRspMsg);
return code;
TAOS_RETURN(code);
}
int32_t mndInitQuery(SMnode *pMnode) {

82
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -50,17 +50,17 @@ static bool isCountWindowStreamTask(SSubplan* pPlan) {
int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
int64_t watermark, int64_t deleteMark) {
int32_t code = 0;
SNode* pAst = NULL;
SQueryPlan* pPlan = NULL;
terrno = TSDB_CODE_SUCCESS;
if (nodesStringToNode(ast, &pAst) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
if (qSetSTableIdForRsma(pAst, uid) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
@ -75,33 +75,33 @@ int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64
};
if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
int32_t opNum = LIST_LENGTH(inner->pNodeList);
if (opNum != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
if (qSubPlanToString(plan, pDst, pDstLen) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
END:
if (pAst) nodesDestroyNode(pAst);
if (pPlan) nodesDestroyNode((SNode*)pPlan);
return terrno;
TAOS_RETURN(code);
}
int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
@ -127,6 +127,7 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList,
SStreamTask* pTask) {
int32_t code = 0;
bool isShuffle = false;
if (pStream->fixedSinkVgId == 0) {
@ -135,9 +136,7 @@ int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SAr
isShuffle = true;
pTask->outputInfo.type = TASK_OUTPUT__SHUFFLE_DISPATCH;
pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH;
if (mndExtractDbInfo(pMnode, pDb, &pTask->outputInfo.shuffleDispatcher.dbInfo, NULL) < 0) {
return -1;
}
TAOS_CHECK_RETURN(mndExtractDbInfo(pMnode, pDb, &pTask->outputInfo.shuffleDispatcher.dbInfo, NULL));
}
sdbRelease(pMnode->pSdb, pDb);
@ -166,7 +165,7 @@ int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SAr
streamTaskSetFixedDownstreamInfo(pTask, pOneSinkTask);
}
return 0;
TAOS_RETURN(code);
}
int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) {
@ -639,14 +638,15 @@ static void bindTwoLevel(SArray* tasks, int32_t begin, int32_t end) {
static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SEpSet* pEpset, int64_t skey,
SArray* pVerList) {
int32_t code = 0;
SSdb* pSdb = pMnode->pSdb;
int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans);
bool hasExtraSink = false;
bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb);
if (pDbObj == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
}
bool multiTarget = (pDbObj->cfg.numOfVgroups > 1);
@ -670,9 +670,11 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
SSubplan* plan = getScanSubPlan(pPlan); // source plan
if (plan == NULL) {
return terrno;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
int32_t code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, numOfPlanLevel == 1);
code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, numOfPlanLevel == 1);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -688,7 +690,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (numOfPlanLevel == 3) {
plan = getAggSubPlan(pPlan, 1); // middle agg plan
if (plan == NULL) {
return terrno;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
do {
SArray** list = taosArrayGetLast(pStream->tasks);
@ -715,7 +719,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
plan = getAggSubPlan(pPlan, 0);
if (plan == NULL) {
return terrno;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
mDebug("doScheduleStream add final agg");
@ -724,7 +730,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
addNewTaskList(pStream);
code = addAggTask(pStream, pMnode, plan, pEpset, true);
if (code != TSDB_CODE_SUCCESS) {
return code;
TAOS_RETURN(code);
}
bindTwoLevel(pStream->tasks, 0, size);
if (pStream->conf.fillHistory) {
@ -735,26 +741,28 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (pStream->conf.fillHistory) {
bindAggSink(pStream, pMnode, pStream->pHTasksList);
}
return TDB_CODE_SUCCESS;
TAOS_RETURN(code);
}
int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SArray* pVgVerList) {
int32_t code = 0;
SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
}
SEpSet mnodeEpset = {0};
mndGetMnodeEpSet(pMnode, &mnodeEpset);
int32_t code = doScheduleStream(pStream, pMnode, pPlan, &mnodeEpset, skey, pVgVerList);
code = doScheduleStream(pStream, pMnode, pPlan, &mnodeEpset, skey, pVgVerList);
qDestroyQueryPlan(pPlan);
return code;
TAOS_RETURN(code);
}
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
int32_t code = 0;
SSdb* pSdb = pMnode->pSdb;
SVgObj* pVgroup = NULL;
SQueryPlan* pPlan = NULL;
@ -763,21 +771,21 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) {
pPlan = qStringToQueryPlan(pTopic->physicalPlan);
if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
}
} else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) {
SNode* pAst = NULL;
if (nodesStringToNode(pTopic->ast, &pAst) != 0) {
if ((code = nodesStringToNode(pTopic->ast, &pAst)) != 0) {
mError("topic:%s, failed to create since %s", pTopic->name, terrstr());
return -1;
TAOS_RETURN(code);
}
SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) {
if ((code = qCreateQueryPlan(&cxt, &pPlan, NULL)) != 0) {
mError("failed to create topic:%s since %s", pTopic->name, terrstr());
nodesDestroyNode(pAst);
return -1;
TAOS_RETURN(code);
}
nodesDestroyNode(pAst);
}
@ -786,8 +794,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) {
qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1;
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
TAOS_RETURN(code);
}
SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
@ -795,8 +803,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList);
if (opNum != 1) {
qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1;
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
TAOS_RETURN(code);
}
pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0);
@ -831,13 +839,13 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) {
qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
}
} else {
pSub->qmsg = taosStrdup("");
}
qDestroyQueryPlan(pPlan);
return 0;
TAOS_RETURN(code);
}

12
source/dnode/snode/src/snode.c
View File

@ -61,6 +61,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
}
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
int32_t code = 0;
SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode));
if (pSnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -68,12 +69,17 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
}
stopRsync();
startRsync();
code = startRsync();
if (code != 0) {
terrno = code;
goto FAIL;
}
pSnode->msgCb = pOption->msgCb;
int32_t code = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE,
taosGetTimestampMs(), tqStartTaskCompleteCallback, &pSnode->pMeta);
code = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE,
taosGetTimestampMs(), tqStartTaskCompleteCallback, &pSnode->pMeta);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto FAIL;
}

51
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -103,31 +103,31 @@ int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TQ_ERR_RETURN(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_RETURN(tdbTbUpsert(ttb, key, kLen, value, vLen, txn));
TQ_ERR_RETURN(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_RETURN(tdbPostCommit(pTq->pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbUpsert(ttb, key, kLen, value, vLen, txn));
TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn));
return 0;
END:
tdbAbort(pTq->pMetaDB, txn);
return code;
}
int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) {
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TQ_ERR_RETURN(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_RETURN(tdbTbDelete(ttb, key, kLen, txn));
TQ_ERR_RETURN(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_RETURN(tdbPostCommit(pTq->pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbDelete(ttb, key, kLen, txn));
TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn));
return 0;
}
static int32_t tqMetaTransformOffsetInfo(STQ* pTq, char* path) {
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_RETURN(tqOffsetRestoreFromFile(pTq, path));
END:
tdbAbort(pTq->pMetaDB, txn);
return code;
}
@ -408,7 +408,8 @@ END:
}
int32_t tqMetaOpen(STQ* pTq) {
char* maindb = NULL;
char* maindb = NULL;
char* offsetNew = NULL;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME));
if(!taosCheckExistFile(maindb)){
@ -416,12 +417,20 @@ int32_t tqMetaOpen(STQ* pTq) {
TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
}else{
TQ_ERR_GO_TO_END(tqMetaTransform(pTq));
taosRemoveFile(maindb);
(void)taosRemoveFile(maindb);
}
TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
if(taosCheckExistFile(offsetNew)){
TQ_ERR_GO_TO_END(tqOffsetRestoreFromFile(pTq, offsetNew));
(void)taosRemoveFile(offsetNew);
}
TQ_ERR_GO_TO_END(tqMetaRestoreCheckInfo(pTq));
END:
taosMemoryFree(maindb);
taosMemoryFree(offsetNew);
return code;
}
@ -445,14 +454,14 @@ int32_t tqMetaTransform(STQ* pTq) {
TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore));
TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
if(taosCheckExistFile(offset) && taosCopyFile(offset, offsetNew) < 0){
tqError("copy offset file error");
if(taosCheckExistFile(offset)) {
if (taosCopyFile(offset, offsetNew) < 0) {
tqError("copy offset file error");
} else {
(void)taosRemoveFile(offset);
}
}
TQ_ERR_GO_TO_END(tqMetaTransformOffsetInfo(pTq, offsetNew));
(void)taosRemoveFile(offset);
(void)taosRemoveFile(offsetNew);
END:
taosMemoryFree(offset);
taosMemoryFree(offsetNew);

1
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -73,6 +73,7 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) {
}
TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset.subKey, strlen(offset.subKey), pMemBuf, size));
tqInfo("tq: offset restore from file to tdb, subkey:%s", offset.subKey);
taosMemoryFree(pMemBuf);
pMemBuf = NULL;
}

256
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -88,33 +88,33 @@ const ESyncRole vnodeStrToRole(char *str) {
int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
const SVnodeCfg *pCfg = (SVnodeCfg *)pObj;
if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1;
if (tjsonAddStringToObject(pJson, "dbname", pCfg->dbname) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "cacheLast", pCfg->cacheLast) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "cacheLastSize", pCfg->cacheLastSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1;
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId));
TAOS_CHECK_RETURN(tjsonAddStringToObject(pJson, "dbname", pCfg->dbname));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "cacheLast", pCfg->cacheLast));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "cacheLastSize", pCfg->cacheLastSize));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize));
if (pCfg->tsdbCfg.retentions[0].keep > 0) {
int32_t nRetention = 1;
if (pCfg->tsdbCfg.retentions[1].freq > 0) {
@ -124,61 +124,67 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
}
}
SJson *pNodeRetentions = tjsonCreateArray();
if (pNodeRetentions == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tjsonAddItemToObject(pJson, "retentions", pNodeRetentions);
for (int32_t i = 0; i < nRetention; ++i) {
SJson *pNodeRetention = tjsonCreateObject();
const SRetention *pRetention = pCfg->tsdbCfg.retentions + i;
tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq);
tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit);
tjsonAddIntegerToObject(pNodeRetention, "keep", pRetention->keep);
tjsonAddIntegerToObject(pNodeRetention, "keepUnit", pRetention->keepUnit);
tjsonAddItemToArray(pNodeRetentions, pNodeRetention);
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "keep", pRetention->keep));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "keepUnit", pRetention->keepUnit));
TAOS_CHECK_RETURN(tjsonAddItemToArray(pNodeRetentions, pNodeRetention));
}
}
if (tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashChange", pCfg->hashChange) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries) < 0) return -1;
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashChange", pCfg->hashChange));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries));
SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1;
if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1;
if (nodeInfo == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TAOS_CHECK_RETURN(tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo));
vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId,
pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) {
SJson *info = tjsonCreateObject();
SJson *info = tjsonCreateObject();
if (info == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1;
if (tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort) < 0) return -1;
if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1;
if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1;
if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1;
if (tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)) < 0) return -1;
if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1;
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort));
TAOS_CHECK_RETURN(tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId));
TAOS_CHECK_RETURN(tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)));
TAOS_CHECK_RETURN(tjsonAddItemToArray(nodeInfo, info));
vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId);
}
@ -191,50 +197,50 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
int32_t code;
tjsonGetNumberValue(pJson, "vgId", pCfg->vgId, code);
if (code < 0) return -1;
if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1;
if (code) return code;
if ((code = tjsonGetStringValue(pJson, "dbname", pCfg->dbname))) return code;
tjsonGetNumberValue(pJson, "dbId", pCfg->dbId, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "szPage", pCfg->szPage, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "cacheLast", pCfg->cacheLast, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "cacheLastSize", pCfg->cacheLastSize, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset, code);
if (code < 0) return -1;
if (code) return code;
SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
if (nRetention > TSDB_RETENTION_MAX) {
@ -244,52 +250,54 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i);
ASSERT(pNodeRetention != NULL);
tjsonGetNumberValue(pNodeRetention, "freq", (pCfg->tsdbCfg.retentions)[i].freq, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "freqUnit", (pCfg->tsdbCfg.retentions)[i].freqUnit, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code);
if (code) return code;
}
tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
if (code < 0) return -1;
if (code) return code;
#if defined(TD_ENTERPRISE)
if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
} else {
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
if (code < 0) return -1;
if (code) return code;
#if defined(TD_ENTERPRISE)
if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
} else {
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
if (code < 0) return -1;
if (code) return code;
#if defined(TD_ENTERPRISE)
if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
@ -303,35 +311,35 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER;
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "hashChange", pCfg->hashChange, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code);
if (code < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code);
if (code < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries, code);
if (code < 0) return -1;
if (code) return code;
SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
int arraySize = tjsonGetArraySize(nodeInfo);
@ -344,15 +352,15 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1;
tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn);
tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code);
if (code < 0) return -1;
if (code) return code;
char role[10] = {0};
code = tjsonGetStringValue(info, "isReplica", role);
if (code < 0) return -1;
if (code) return code;
if (strlen(role) != 0) {
pNode->nodeRole = vnodeStrToRole(role);
} else {

200
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -125,20 +125,16 @@ int vnodeBegin(SVnode *pVnode) {
TSDB_CHECK_CODE(code, lino, _exit);
// begin meta
if (metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL) < 0) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL);
TSDB_CHECK_CODE(code, lino, _exit);
// begin tsdb
if (tsdbBegin(pVnode->pTsdb) < 0) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbBegin(pVnode->pTsdb);
TSDB_CHECK_CODE(code, lino, _exit);
// begin sma
if (VND_IS_RSMA(pVnode) && smaBegin(pVnode->pSma) < 0) {
code = terrno;
if (VND_IS_RSMA(pVnode)) {
code = smaBegin(pVnode->pSma);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -171,54 +167,41 @@ int vnodeShouldCommit(SVnode *pVnode, bool atExit) {
}
int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile;
char *data;
TdFilePtr pFile = NULL;
char *data = NULL;
snprintf(fname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP);
// encode info
data = NULL;
if (vnodeEncodeInfo(pInfo, &data) < 0) {
vError("failed to encode json info.");
return -1;
}
code = vnodeEncodeInfo(pInfo, &data);
TSDB_CHECK_CODE(code, lino, _exit);
// save info to a vnode_tmp.json
pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) {
vError("failed to open info file:%s for write:%s", fname, terrstr());
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
if (taosWriteFile(pFile, data, strlen(data)) < 0) {
vError("failed to write info file:%s error:%s", fname, terrstr());
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
if (taosFsyncFile(pFile) < 0) {
vError("failed to fsync info file:%s error:%s", fname, terrstr());
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
taosCloseFile(&pFile);
// free info binary
taosMemoryFree(data);
vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", pInfo->config.vgId, fname,
pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, pInfo->config.syncCfg.changeVersion);
return 0;
_err:
_exit:
if (code) {
vError("vgId:%d %s failed at %s:%d since %s", pInfo->config.vgId, __func__, __FILE__, lino, tstrerror(code));
} else {
vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", pInfo->config.vgId, fname,
pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, pInfo->config.syncCfg.changeVersion);
}
taosCloseFile(&pFile);
taosMemoryFree(data);
return -1;
return code;
}
int vnodeCommitInfo(const char *dir) {
@ -229,8 +212,7 @@ int vnodeCommitInfo(const char *dir) {
snprintf(tfname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP);
if (taosRenameFile(tfname, fname) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
vInfo("vnode info is committed, dir:%s", dir);
@ -238,6 +220,8 @@ int vnodeCommitInfo(const char *dir) {
}
int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) {
int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile = NULL;
char *pData = NULL;
@ -248,44 +232,35 @@ int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) {
// read info
pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
pData = taosMemoryMalloc(size + 1);
if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
if (taosReadFile(pFile, pData, size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
pData[size] = '\0';
taosCloseFile(&pFile);
// decode info
if (vnodeDecodeInfo(pData, pInfo) < 0) {
taosMemoryFree(pData);
return -1;
code = vnodeDecodeInfo(pData, pInfo);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
vError("vgId:%d %s failed at %s:%d since %s", pInfo->config.vgId, __func__, __FILE__, lino, tstrerror(code));
}
taosMemoryFree(pData);
return 0;
_err:
taosCloseFile(&pFile);
taosMemoryFree(pData);
return -1;
return code;
}
static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
@ -297,7 +272,8 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
// wait last commit task
vnodeAWait(&pVnode->commitTask);
if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit;
code = syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg);
TSDB_CHECK_CODE(code, lino, _exit);
pVnode->state.commitTerm = pVnode->state.applyTerm;
@ -312,17 +288,16 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN);
vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode));
if (vnodeSaveInfo(dir, &pInfo->info) < 0) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = vnodeSaveInfo(dir, &pInfo->info);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbPreCommit(pVnode->pTsdb);
metaPrepareAsyncCommit(pVnode->pMeta);
code = metaPrepareAsyncCommit(pVnode->pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
code = smaPrepareAsyncCommit(pVnode->pSma);
if (code) goto _exit;
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&pVnode->mutex);
ASSERT(pVnode->onCommit == NULL);
@ -459,10 +434,8 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) {
}
// commit info
if (vnodeCommitInfo(dir) < 0) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = vnodeCommitInfo(dir);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCommitCommit(pVnode->pTsdb);
TSDB_CHECK_CODE(code, lino, _exit);
@ -472,10 +445,8 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) {
TSDB_CHECK_CODE(code, lino, _exit);
}
if (metaFinishCommit(pVnode->pMeta, pInfo->txn) < 0) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = metaFinishCommit(pVnode->pMeta, pInfo->txn);
TSDB_CHECK_CODE(code, lino, _exit);
pVnode->state.committed = pInfo->info.state.committed;
@ -492,7 +463,7 @@ _exit:
} else {
vInfo("vgId:%d, commit end", TD_VID(pVnode));
}
return 0;
return code;
}
bool vnodeShouldRollback(SVnode *pVnode) {
@ -514,15 +485,15 @@ void vnodeRollback(SVnode *pVnode) {
offset = strlen(tFName);
snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP);
(void)taosRemoveFile(tFName);
TAOS_UNUSED(taosRemoveFile(tFName));
}
static int vnodeEncodeState(const void *pObj, SJson *pJson) {
const SVState *pState = (SVState *)pObj;
if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm) < 0) return -1;
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit version", pState->committed));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm));
return 0;
}
@ -532,70 +503,67 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
int32_t code;
tjsonGetNumberValue(pJson, "commit version", pState->committed, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code);
if (code < 0) return -1;
if (code) return code;
return 0;
}
static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) {
SJson *pJson;
char *pData;
*ppData = NULL;
int32_t code = 0;
int32_t lino;
SJson *pJson = NULL;
char *pData = NULL;
pJson = tjsonCreateObject();
if (pJson == NULL) {
return -1;
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
if (tjsonAddObject(pJson, "config", vnodeEncodeConfig, (void *)&pInfo->config) < 0) {
goto _err;
}
code = tjsonAddObject(pJson, "config", vnodeEncodeConfig, (void *)&pInfo->config);
TSDB_CHECK_CODE(code, lino, _exit);
if (tjsonAddObject(pJson, "state", vnodeEncodeState, (void *)&pInfo->state) < 0) {
goto _err;
}
code = tjsonAddObject(pJson, "state", vnodeEncodeState, (void *)&pInfo->state);
TSDB_CHECK_CODE(code, lino, _exit);
pData = tjsonToString(pJson);
if (pData == NULL) {
goto _err;
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
tjsonDelete(pJson);
*ppData = pData;
return 0;
_err:
tjsonDelete(pJson);
return -1;
_exit:
if (code) {
tjsonDelete(pJson);
*ppData = NULL;
} else {
*ppData = pData;
}
return code;
}
int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) {
SJson *pJson = NULL;
int32_t code = 0;
int32_t lino;
SJson *pJson = NULL;
pJson = tjsonParse(pData);
if (pJson == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
return -1;
}
if (tjsonToObject(pJson, "config", vnodeDecodeConfig, (void *)&pInfo->config) < 0) {
goto _err;
}
code = tjsonToObject(pJson, "config", vnodeDecodeConfig, (void *)&pInfo->config);
TSDB_CHECK_CODE(code, lino, _exit);
if (tjsonToObject(pJson, "state", vnodeDecodeState, (void *)&pInfo->state) < 0) {
goto _err;
}
code = tjsonToObject(pJson, "state", vnodeDecodeState, (void *)&pInfo->state);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
tjsonDelete(pJson);
return 0;
_err:
tjsonDelete(pJson);
return -1;
return code;
}

431
source/dnode/vnode/src/vnd/vnodeSnapshot.c
View File

@ -17,20 +17,19 @@
#include "vnd.h"
static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) {
int32_t code = -1;
int32_t code = 0;
STsdbFSetPartList *pList = tsdbFSetPartListCreate();
if (pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _out;
}
if (tDeserializeTsdbFSetPartList(buf, bufLen, pList) < 0) {
terrno = TSDB_CODE_INVALID_DATA_FMT;
goto _out;
}
if (tsdbFSetPartListToRangeDiff(pList, ppRanges) < 0) {
goto _out;
}
code = 0;
code = tDeserializeTsdbFSetPartList(buf, bufLen, pList);
if (code) goto _out;
code = tsdbFSetPartListToRangeDiff(pList, ppRanges);
if (code) goto _out;
_out:
tsdbFSetPartListDestroy(&pList);
return code;
@ -48,29 +47,29 @@ struct SVSnapReader {
int8_t metaDone;
SMetaSnapReader *pMetaReader;
// tsdb
int8_t tsdbDone;
int8_t tsdbDone;
TFileSetRangeArray *pRanges;
STsdbSnapReader *pTsdbReader;
STsdbSnapReader *pTsdbReader;
// tsdb raw
int8_t tsdbRAWDone;
STsdbSnapRAWReader *pTsdbRAWReader;
// tq
int8_t tqHandleDone;
STqSnapReader *pTqSnapReader;
int8_t tqOffsetDone;
STqSnapReader *pTqOffsetReader;
int8_t tqCheckInfoDone;
STqSnapReader *pTqCheckInfoReader;
int8_t tqHandleDone;
STqSnapReader *pTqSnapReader;
int8_t tqOffsetDone;
STqSnapReader *pTqOffsetReader;
int8_t tqCheckInfoDone;
STqSnapReader *pTqCheckInfoReader;
// stream
int8_t streamTaskDone;
SStreamTaskReader *pStreamTaskReader;
int8_t streamStateDone;
SStreamStateReader *pStreamStateReader;
// rsma
int8_t rsmaDone;
int8_t rsmaDone;
TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
SRSmaSnapReader *pRsmaReader;
SRSmaSnapReader *pRsmaReader;
};
static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) {
@ -88,14 +87,15 @@ static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader,
}
static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) {
int32_t code = 0;
SVnode *pVnode = pReader->pVnode;
int32_t code = -1;
if (pParam->data) {
// decode
SSyncTLV *datHead = (void *)pParam->data;
if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
terrno = TSDB_CODE_INVALID_DATA_FMT;
code = TSDB_CODE_INVALID_DATA_FMT;
terrno = code;
goto _out;
}
@ -116,21 +116,25 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP
ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ);
if (ppRanges == NULL) {
vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
code = TSDB_CODE_INVALID_DATA_FMT;
goto _out;
}
if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
code = vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges);
if (code) {
vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
goto _out;
}
} break;
case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) {
code = tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts);
if (code) {
vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out;
}
} break;
default:
vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
code = TSDB_CODE_INVALID_DATA_FMT;
goto _out;
}
}
@ -147,7 +151,7 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP
vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode),
(pReader->tsdbDone ? "raw" : "normal"));
}
code = 0;
_out:
return code;
}
@ -160,42 +164,43 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader
pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
pReader->pVnode = pVnode;
pReader->sver = sver;
pReader->ever = ever;
// snapshot info
if (vnodeSnapReaderDealWithSnapInfo(pReader, pParam) < 0) {
goto _err;
}
code = vnodeSnapReaderDealWithSnapInfo(pReader, pParam);
if (code) goto _exit;
// open tsdb snapshot raw reader
if (!pReader->tsdbRAWDone) {
ASSERT(pReader->sver == 0);
code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
if (code) goto _err;
if (code) goto _exit;
}
// check snapshot ever
SSnapshot snapshot = {0};
vnodeGetSnapshot(pVnode, &snapshot);
code = vnodeGetSnapshot(pVnode, &snapshot);
if (code) goto _exit;
if (ever != snapshot.lastApplyIndex) {
vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64,
TD_VID(pVnode), ever, snapshot.lastApplyIndex);
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _err;
goto _exit;
}
vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
*ppReader = pReader;
return code;
_err:
vError("vgId:%d, vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code));
*ppReader = NULL;
_exit:
if (code) {
vError("vgId:%d, vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code));
*ppReader = NULL;
} else {
vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
*ppReader = pReader;
}
return code;
}
@ -245,6 +250,7 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) {
int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
int32_t code = 0;
int32_t lino;
SVnode *pVnode = pReader->pVnode;
int32_t vgId = TD_VID(pReader->pVnode);
@ -261,31 +267,28 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ);
if (NULL == pFile) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
int64_t size;
if (taosFStatFile(pFile, &size, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1);
if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFile);
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
((SSnapDataHdr *)(*ppData))->type = SNAP_DATA_CFG;
((SSnapDataHdr *)(*ppData))->size = size + 1;
((SSnapDataHdr *)(*ppData))->data[size] = '\0';
if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(*ppData);
taosCloseFile(&pFile);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
taosCloseFile(&pFile);
@ -299,20 +302,18 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
// open reader if not
if (pReader->pMetaReader == NULL) {
code = metaSnapReaderOpen(pReader->pVnode->pMeta, pReader->sver, pReader->ever, &pReader->pMetaReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = metaSnapRead(pReader->pMetaReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->metaDone = 1;
code = metaSnapReaderClose(&pReader->pMetaReader);
if (code) goto _err;
}
pReader->metaDone = 1;
code = metaSnapReaderClose(&pReader->pMetaReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -322,20 +323,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (pReader->pTsdbReader == NULL) {
code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pRanges,
&pReader->pTsdbReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbSnapRead(pReader->pTsdbReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->tsdbDone = 1;
code = tsdbSnapReaderClose(&pReader->pTsdbReader);
if (code) goto _err;
}
pReader->tsdbDone = 1;
code = tsdbSnapReaderClose(&pReader->pTsdbReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -344,20 +342,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (pReader->pTsdbRAWReader == NULL) {
ASSERT(pReader->sver == 0);
code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->tsdbRAWDone = 1;
code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
if (code) goto _err;
}
pReader->tsdbRAWDone = 1;
code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -365,59 +360,53 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
vInfo("vgId:%d tq transform start", vgId);
if (!pReader->tqHandleDone) {
if (pReader->pTqSnapReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE, &pReader->pTqSnapReader);
if (code < 0) goto _err;
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE,
&pReader->pTqSnapReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapRead(pReader->pTqSnapReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->tqHandleDone = 1;
code = tqSnapReaderClose(&pReader->pTqSnapReader);
if (code) goto _err;
}
pReader->tqHandleDone = 1;
code = tqSnapReaderClose(&pReader->pTqSnapReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
if (!pReader->tqCheckInfoDone) {
if (pReader->pTqCheckInfoReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO, &pReader->pTqCheckInfoReader);
if (code < 0) goto _err;
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO,
&pReader->pTqCheckInfoReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapRead(pReader->pTqCheckInfoReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->tqCheckInfoDone = 1;
code = tqSnapReaderClose(&pReader->pTqCheckInfoReader);
if (code) goto _err;
}
pReader->tqCheckInfoDone = 1;
code = tqSnapReaderClose(&pReader->pTqCheckInfoReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
if (!pReader->tqOffsetDone) {
if (pReader->pTqOffsetReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET, &pReader->pTqOffsetReader);
if (code < 0) goto _err;
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET,
&pReader->pTqOffsetReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapRead(pReader->pTqOffsetReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->tqOffsetDone = 1;
code = tqSnapReaderClose(&pReader->pTqOffsetReader);
if (code) goto _err;
}
pReader->tqOffsetDone = 1;
code = tqSnapReaderClose(&pReader->pTqOffsetReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -426,28 +415,19 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (!pReader->streamTaskDone) {
if (pReader->pStreamTaskReader == NULL) {
code = streamTaskSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver, &pReader->pStreamTaskReader);
if (code) {
vError("vgId:%d open streamtask snapshot reader failed, code:%s", vgId, tstrerror(code));
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
code = streamTaskSnapRead(pReader->pStreamTaskReader, ppData);
if (code) {
vError("vgId:%d error happens during read data from streatask snapshot, code:%s", vgId, tstrerror(code));
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
vInfo("vgId:%d no streamTask snapshot", vgId);
goto _exit;
} else {
if (*ppData) {
vInfo("vgId:%d no streamTask snapshot", vgId);
goto _exit;
} else {
pReader->streamTaskDone = 1;
code = streamTaskSnapReaderClose(pReader->pStreamTaskReader);
if (code) {
goto _err;
}
pReader->pStreamTaskReader = NULL;
}
pReader->streamTaskDone = 1;
code = streamTaskSnapReaderClose(pReader->pStreamTaskReader);
TSDB_CHECK_CODE(code, lino, _exit);
pReader->pStreamTaskReader = NULL;
}
}
if (!pReader->streamStateDone) {
@ -457,21 +437,18 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (code) {
pReader->streamStateDone = 1;
pReader->pStreamStateReader = NULL;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = streamStateSnapRead(pReader->pStreamStateReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->streamStateDone = 1;
code = streamStateSnapReaderClose(pReader->pStreamStateReader);
if (code) goto _err;
pReader->pStreamStateReader = NULL;
}
pReader->streamStateDone = 1;
code = streamStateSnapReaderClose(pReader->pStreamStateReader);
TSDB_CHECK_CODE(code, lino, _exit);
pReader->pStreamStateReader = NULL;
}
}
@ -480,20 +457,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
// open if not
if (pReader->pRsmaReader == NULL) {
code = rsmaSnapReaderOpen(pReader->pVnode->pSma, pReader->sver, pReader->ever, &pReader->pRsmaReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = rsmaSnapRead(pReader->pRsmaReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->rsmaDone = 1;
code = rsmaSnapReaderClose(&pReader->pRsmaReader);
if (code) goto _err;
}
pReader->rsmaDone = 1;
code = rsmaSnapReaderClose(&pReader->pRsmaReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -501,21 +475,21 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
*nData = 0;
_exit:
if (*ppData) {
SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData);
pReader->index++;
*nData = sizeof(SSnapDataHdr) + pHdr->size;
pHdr->index = pReader->index;
vDebug("vgId:%d, vnode snapshot read data, index:%" PRId64 " type:%d blockLen:%d ", vgId, pReader->index,
pHdr->type, *nData);
if (code) {
vError("vgId:%d, vnode snapshot read failed at %s:%d since %s", vgId, __FILE__, lino, tstrerror(code));
} else {
vInfo("vgId:%d, vnode snapshot read data end, index:%" PRId64, vgId, pReader->index);
}
return code;
if (*ppData) {
SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData);
_err:
vError("vgId:%d, vnode snapshot read failed since %s", vgId, tstrerror(code));
pReader->index++;
*nData = sizeof(SSnapDataHdr) + pHdr->size;
pHdr->index = pReader->index;
vDebug("vgId:%d, vnode snapshot read data, index:%" PRId64 " type:%d blockLen:%d ", vgId, pReader->index,
pHdr->type, *nData);
} else {
vInfo("vgId:%d, vnode snapshot read data end, index:%" PRId64, vgId, pReader->index);
}
}
return code;
}
@ -532,19 +506,19 @@ struct SVSnapWriter {
SMetaSnapWriter *pMetaSnapWriter;
// tsdb
TFileSetRangeArray *pRanges;
STsdbSnapWriter *pTsdbSnapWriter;
STsdbSnapWriter *pTsdbSnapWriter;
// tsdb raw
STsdbSnapRAWWriter *pTsdbSnapRAWWriter;
// tq
STqSnapWriter *pTqSnapHandleWriter;
STqSnapWriter *pTqSnapOffsetWriter;
STqSnapWriter *pTqSnapCheckInfoWriter;
STqSnapWriter *pTqSnapHandleWriter;
STqSnapWriter *pTqSnapOffsetWriter;
STqSnapWriter *pTqSnapCheckInfoWriter;
// stream
SStreamTaskWriter *pStreamTaskWriter;
SStreamStateWriter *pStreamStateWriter;
// rsma
TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
SRSmaSnapWriter *pRsmaSnapWriter;
SRSmaSnapWriter *pRsmaSnapWriter;
};
TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) {
@ -563,18 +537,18 @@ TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t
static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) {
SVnode *pVnode = pWriter->pVnode;
int32_t code = -1;
int32_t code = 0;
int32_t lino;
if (pParam->data) {
SSyncTLV *datHead = (void *)pParam->data;
if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
terrno = TSDB_CODE_INVALID_DATA_FMT;
goto _out;
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
}
STsdbRepOpts tsdbOpts = {0};
TFileSetRangeArray **ppRanges = NULL;
int32_t offset = 0;
int32_t offset = 0;
while (offset + sizeof(SSyncTLV) < datHead->len) {
SSyncTLV *subField = (void *)(datHead->val + offset);
@ -589,30 +563,30 @@ static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotP
ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ);
if (ppRanges == NULL) {
vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
goto _out;
}
if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
goto _out;
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
code = vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges);
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) {
vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out;
}
code = tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts);
TSDB_CHECK_CODE(code, lino, _exit);
} break;
default:
vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
goto _out;
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
goto _exit;
}
}
vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format);
}
code = 0;
_out:
_exit:
if (code) {
vError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
}
return code;
}
@ -634,6 +608,7 @@ static int32_t vnodeEnableBgTask(SVnode *pVnode) {
int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) {
int32_t code = 0;
int32_t lino;
SVSnapWriter *pWriter = NULL;
int64_t sver = pParam->start;
int64_t ever = pParam->end;
@ -644,8 +619,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
// alloc
pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
pWriter->pVnode = pVnode;
pWriter->sver = sver;
@ -655,19 +629,19 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
pWriter->commitID = ++pVnode->state.commitID;
// snapshot info
if (vnodeSnapWriterDealWithSnapInfo(pWriter, pParam) < 0) {
goto _err;
code = vnodeSnapWriterDealWithSnapInfo(pWriter, pParam);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
vError("vgId:%d, vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
if (pWriter) taosMemoryFreeClear(pWriter);
*ppWriter = NULL;
} else {
vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
sver, ever, pWriter->commitID);
*ppWriter = pWriter;
}
vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
sver, ever, pWriter->commitID);
*ppWriter = pWriter;
return code;
_err:
vError("vgId:%d, vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
if (pWriter) taosMemoryFreeClear(pWriter);
*ppWriter = NULL;
return code;
}
@ -785,14 +759,13 @@ _exit:
static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
int32_t code = 0;
int32_t lino;
SVnode *pVnode = pWriter->pVnode;
SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
// decode info
if (vnodeDecodeInfo(pHdr->data, &pWriter->info) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
code = vnodeDecodeInfo(pHdr->data, &pWriter->info);
TSDB_CHECK_CODE(code, lino, _exit);
// change some value
pWriter->info.state.commitID = pWriter->commitID;
@ -805,10 +778,8 @@ static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_
pWriter->info.config = pVnode->config;
pWriter->info.config.vndStats = vndStats;
vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId);
if (vnodeSaveInfo(dir, &pWriter->info) < 0) {
code = terrno;
goto _exit;
}
code = vnodeSaveInfo(dir, &pWriter->info);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
return code;
@ -816,6 +787,7 @@ _exit:
int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
int32_t code = 0;
int32_t lino;
SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
SVnode *pVnode = pWriter->pVnode;
@ -824,7 +796,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pHdr->index != pWriter->index + 1) {
vError("vgId:%d, unexpected vnode snapshot msg. index:%" PRId64 ", expected index:%" PRId64, TD_VID(pVnode),
pHdr->index, pWriter->index + 1);
return -1;
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_MSG, lino, _exit);
}
pWriter->index = pHdr->index;
@ -835,17 +807,17 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
switch (pHdr->type) {
case SNAP_DATA_CFG: {
code = vnodeSnapWriteInfo(pWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_META: {
// meta
if (pWriter->pMetaSnapWriter == NULL) {
code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_TSDB:
case SNAP_DATA_DEL: {
@ -853,69 +825,69 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pWriter->pTsdbSnapWriter == NULL) {
code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, pWriter->pRanges,
&pWriter->pTsdbSnapWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_RAW: {
// tsdb
if (pWriter->pTsdbSnapRAWWriter == NULL) {
ASSERT(pWriter->sver == 0);
code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_TQ_HANDLE: {
// tq handle
if (pWriter->pTqSnapHandleWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapHandleWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapHandleWrite(pWriter->pTqSnapHandleWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_TQ_CHECKINFO: {
// tq checkinfo
if (pWriter->pTqSnapCheckInfoWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapCheckInfoWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapCheckInfoWrite(pWriter->pTqSnapCheckInfoWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_TQ_OFFSET: {
// tq offset
if (pWriter->pTqSnapOffsetWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapOffsetWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapOffsetWrite(pWriter->pTqSnapOffsetWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_STREAM_TASK:
case SNAP_DATA_STREAM_TASK_CHECKPOINT: {
if (pWriter->pStreamTaskWriter == NULL) {
code = streamTaskSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamTaskWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = streamTaskSnapWrite(pWriter->pStreamTaskWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_STREAM_STATE_BACKEND: {
if (pWriter->pStreamStateWriter == NULL) {
code = streamStateSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamStateWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = streamStateSnapWrite(pWriter->pStreamStateWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_RSMA1:
@ -925,20 +897,19 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pWriter->pRsmaSnapWriter == NULL) {
code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, (void **)pWriter->pRsmaRanges,
&pWriter->pRsmaSnapWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = rsmaSnapWrite(pWriter->pRsmaSnapWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
default:
break;
}
_exit:
return code;
_err:
vError("vgId:%d, vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode),
tstrerror(code), pHdr->index, pHdr->type, nData);
if (code) {
vError("vgId:%d, vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode),
tstrerror(code), pHdr->index, pHdr->type, nData);
}
return code;
}

34
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -387,7 +387,7 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
if (msgcb == NULL || msgcb->putToQueueFp == NULL) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
return -1;
return TSDB_CODE_INVALID_PARA;
}
int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg);
@ -400,13 +400,13 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
if (pMsg == NULL || pMsg->pCont == NULL) {
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (msgcb == NULL || msgcb->putToQueueFp == NULL) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
return -1;
return TSDB_CODE_INVALID_PARA;
}
int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
@ -485,8 +485,7 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta
static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) {
SVnode *pVnode = pFsm->data;
int32_t code = vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader);
return code;
return vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader);
}
static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
@ -496,8 +495,7 @@ static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
SVnode *pVnode = pFsm->data;
int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
return code;
return vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
}
static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) {
@ -514,8 +512,7 @@ static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void
}
} while (true);
int32_t code = vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter);
return code;
return vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter);
}
static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) {
@ -580,7 +577,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
if (pMeta->startInfo.startAllTasks == 1) {
pMeta->startInfo.restartCount += 1;
vDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId,
pMeta->startInfo.restartCount);
pMeta->startInfo.restartCount);
} else {
pMeta->startInfo.startAllTasks = 1;
streamMetaWUnLock(pMeta);
@ -636,7 +633,7 @@ static void vnodeBecomeLeader(const SSyncFSM *pFsm) {
}
}
static void vnodeBecomeAssignedLeader(const SSyncFSM* pFsm) {
static void vnodeBecomeAssignedLeader(const SSyncFSM *pFsm) {
SVnode *pVnode = pFsm->data;
vDebug("vgId:%d, become assigned leader", pVnode->config.vgId);
if (pVnode->pTq) {
@ -662,12 +659,16 @@ static int32_t vnodeApplyQueueItems(const SSyncFSM *pFsm) {
int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE);
return itemSize;
} else {
return -1;
return TSDB_CODE_INVALID_PARA;
}
}
static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
if (pFsm == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pFsm->data = pVnode;
pFsm->FpCommitCb = vnodeSyncCommitMsg;
pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex;
@ -724,7 +725,7 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path, int32_t vnodeVersion) {
pVnode->sync = syncOpen(&syncInfo, vnodeVersion);
if (pVnode->sync <= 0) {
vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr());
return -1;
return terrno;
}
return 0;
@ -732,9 +733,10 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path, int32_t vnodeVersion) {
int32_t vnodeSyncStart(SVnode *pVnode) {
vInfo("vgId:%d, start sync", pVnode->config.vgId);
if (syncStart(pVnode->sync) < 0) {
vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, terrstr());
return -1;
int32_t code = syncStart(pVnode->sync);
if (code) {
vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, tstrerror(code));
return code;
}
return 0;
}

3
source/libs/catalog/src/catalog.c
View File

@ -446,7 +446,8 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName,
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
char* pJson = parseTagDatatoJson(pTag);
char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
STagVal tagVal;
tagVal.cid = 0;
tagVal.type = TSDB_DATA_TYPE_JSON;

3
source/libs/catalog/src/ctgAsync.c
View File

@ -2079,7 +2079,8 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf*
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
char* pJson = parseTagDatatoJson(pTag);
char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
STagVal tagVal;
tagVal.cid = 0;
tagVal.type = TSDB_DATA_TYPE_JSON;

185
source/libs/command/src/command.c
View File

@ -23,6 +23,15 @@
#include "tglobal.h"
#include "tgrant.h"
#define COL_DATA_SET_VAL_AND_CHECK(pCol, rows, buf, isNull) \
do { \
int _code = colDataSetVal(pCol, rows, buf, isNull);\
if (TSDB_CODE_SUCCESS != _code) { \
terrno = _code; \
return _code; \
} \
} while(0)
extern SConfig* tsCfg;
static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) {
@ -100,7 +109,7 @@ static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) {
if (TSDB_CODE_SUCCESS == code) {
*pOutput = pBlock;
} else {
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
}
return code;
}
@ -108,7 +117,7 @@ static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) {
static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta,
int8_t biMode) {
int32_t blockCap = (biMode != 0) ? numOfRows + 1 : numOfRows;
blockDataEnsureCapacity(pBlock, blockCap);
QRY_ERR_RET(blockDataEnsureCapacity(pBlock, blockCap));
pBlock->info.rows = 0;
// field
@ -138,11 +147,12 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
continue;
}
STR_TO_VARSTR(buf, pMeta->schema[i].name);
colDataSetVal(pCol1, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol1, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name);
colDataSetVal(pCol2, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol2, pBlock->info.rows, buf, false);
int32_t bytes = getSchemaBytes(pMeta->schema + i);
colDataSetVal(pCol3, pBlock->info.rows, (const char*)&bytes, false);
COL_DATA_SET_VAL_AND_CHECK(pCol3, pBlock->info.rows, (const char*)&bytes, false);
if (TSDB_VIEW_TABLE != pMeta->tableType) {
if (i >= pMeta->tableInfo.numOfColumns) {
STR_TO_VARSTR(buf, "TAG");
@ -155,22 +165,22 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
} else {
STR_TO_VARSTR(buf, "VIEW COL");
}
colDataSetVal(pCol4, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol4, pBlock->info.rows, buf, false);
if (useCompress(pMeta->tableType) && pMeta->schemaExt) {
if (i < pMeta->tableInfo.numOfColumns) {
STR_TO_VARSTR(buf, columnEncodeStr(COMPRESS_L1_TYPE_U32(pMeta->schemaExt[i].compress)));
colDataSetVal(pCol5, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol5, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, columnCompressStr(COMPRESS_L2_TYPE_U32(pMeta->schemaExt[i].compress)));
colDataSetVal(pCol6, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol6, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pMeta->schemaExt[i].compress)));
colDataSetVal(pCol7, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol7, pBlock->info.rows, buf, false);
} else {
STR_TO_VARSTR(buf, fillTagCol == 0 ? "" : "disabled");
colDataSetVal(pCol5, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol5, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, fillTagCol == 0 ? "" : "disabled");
colDataSetVal(pCol6, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol6, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, fillTagCol == 0 ? "" : "disabled");
colDataSetVal(pCol7, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol7, pBlock->info.rows, buf, false);
}
}
@ -180,13 +190,13 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
}
if (pMeta->tableType == TSDB_SUPER_TABLE && biMode != 0) {
STR_TO_VARSTR(buf, "tbname");
colDataSetVal(pCol1, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol1, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, "VARCHAR");
colDataSetVal(pCol2, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol2, pBlock->info.rows, buf, false);
int32_t bytes = TSDB_TABLE_NAME_LEN - 1;
colDataSetVal(pCol3, pBlock->info.rows, (const char*)&bytes, false);
COL_DATA_SET_VAL_AND_CHECK(pCol3, pBlock->info.rows, (const char*)&bytes, false);
STR_TO_VARSTR(buf, "TAG");
colDataSetVal(pCol4, pBlock->info.rows, buf, false);
COL_DATA_SET_VAL_AND_CHECK(pCol4, pBlock->info.rows, buf, false);
++(pBlock->info.rows);
}
if (pBlock->info.rows <= 0) {
@ -212,7 +222,7 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp**
code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp);
}
}
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
return code;
}
@ -234,7 +244,7 @@ static int32_t buildCreateDBResultDataBlock(SSDataBlock** pOutput) {
if (TSDB_CODE_SUCCESS == code) {
*pOutput = pBlock;
} else {
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
}
return code;
}
@ -264,13 +274,17 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) {
return v;
}
static char* buildRetension(SArray* pRetension) {
static int32_t buildRetension(SArray* pRetension, char **ppRetentions ) {
size_t size = taosArrayGetSize(pRetension);
if (size == 0) {
return NULL;
*ppRetentions = NULL;
return TSDB_CODE_SUCCESS;
}
char* p1 = taosMemoryCalloc(1, 100);
if(NULL == p1) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t len = 0;
for (int32_t i = 0; i < size; ++i) {
@ -288,7 +302,8 @@ static char* buildRetension(SArray* pRetension) {
}
}
return p1;
*ppRetentions = p1;
return TSDB_CODE_SUCCESS;
}
static const char* cacheModelStr(int8_t cacheModel) {
@ -319,14 +334,14 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) {
return TSDB_CACHE_MODEL_NONE_STR;
}
static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) {
blockDataEnsureCapacity(pBlock, 1);
static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) {
QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1));
pBlock->info.rows = 1;
SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
char buf1[SHOW_CREATE_DB_RESULT_FIELD1_LEN] = {0};
STR_TO_VARSTR(buf1, dbName);
colDataSetVal(pCol1, 0, buf1, false);
COL_DATA_SET_VAL_AND_CHECK(pCol1, 0, buf1, false);
SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
char buf2[SHOW_CREATE_DB_RESULT_FIELD2_LEN] = {0};
@ -347,7 +362,8 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
break;
}
char* retentions = buildRetension(pCfg->pRetensions);
char* pRetentions = NULL;
QRY_ERR_RET(buildRetension(pCfg->pRetensions, &pRetentions));
int32_t dbFNameLen = strlen(dbFName);
int32_t hashPrefix = 0;
if (pCfg->hashPrefix > 0) {
@ -374,28 +390,30 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), pCfg->s3ChunkSize,
pCfg->s3KeepLocal, pCfg->s3Compact);
if (retentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);
if (pRetentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", pRetentions);
}
}
taosMemoryFree(retentions);
taosMemoryFree(pRetentions);
(varDataLen(buf2)) = len;
colDataSetVal(pCol2, 0, buf2, false);
COL_DATA_SET_VAL_AND_CHECK(pCol2, 0, buf2, false);
return TSDB_CODE_SUCCESS;
}
static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = NULL;
int32_t code = buildCreateDBResultDataBlock(&pBlock);
if (TSDB_CODE_SUCCESS == code) {
setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->dbFName, pStmt->pCfg);
code = setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->dbFName, pStmt->pCfg);
}
if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp);
}
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
return code;
}
@ -415,7 +433,7 @@ static int32_t buildCreateTbResultDataBlock(SSDataBlock** pOutput) {
if (TSDB_CODE_SUCCESS == code) {
*pOutput = pBlock;
} else {
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
}
return code;
}
@ -436,7 +454,7 @@ static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) {
if (TSDB_CODE_SUCCESS == code) {
*pOutput = pBlock;
} else {
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
}
return code;
}
@ -494,6 +512,7 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
}
int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* pTagVals = NULL;
STag* pTag = (STag*)pCfg->pTags;
@ -503,20 +522,15 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
}
if (tTagIsJson(pTag)) {
char* pJson = parseTagDatatoJson(pTag);
if (pJson) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson);
taosMemoryFree(pJson);
}
char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson);
taosMemoryFree(pJson);
return TSDB_CODE_SUCCESS;
}
int32_t code = tTagToValArray((const STag*)pCfg->pTags, &pTagVals);
if (code) {
return code;
}
QRY_ERR_RET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals));
int16_t valueNum = taosArrayGetSize(pTagVals);
int32_t num = 0;
int32_t j = 0;
@ -534,16 +548,18 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j);
if (pSchema->colId > pTagVal->cid) {
qError("tag value and column mismatch, schemaId:%d, valId:%d", pSchema->colId, pTagVal->cid);
taosArrayDestroy(pTagVals);
return TSDB_CODE_APP_ERROR;
code = TSDB_CODE_APP_ERROR;
TAOS_CHECK_ERRNO(code);
} else if (pSchema->colId == pTagVal->cid) {
char type = pTagVal->type;
int32_t tlen = 0;
if (IS_VAR_DATA_TYPE(type)) {
dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen);
code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen);
TAOS_CHECK_ERRNO(code);
} else {
dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen);
code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen);
TAOS_CHECK_ERRNO(code);
}
*len += tlen;
j++;
@ -580,9 +596,10 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
*/
}
_exit:
taosArrayDestroy(pTagVals);
return TSDB_CODE_SUCCESS;
return code;
}
void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) {
@ -647,20 +664,19 @@ void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg*
}
static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg) {
int32_t code = 0;
blockDataEnsureCapacity(pBlock, 1);
int32_t code = TSDB_CODE_SUCCESS;
QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1));
pBlock->info.rows = 1;
SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
char buf1[SHOW_CREATE_TB_RESULT_FIELD1_LEN] = {0};
STR_TO_VARSTR(buf1, tbName);
colDataSetVal(pCol1, 0, buf1, false);
QRY_ERR_RET(colDataSetVal(pCol1, 0, buf1, false));
SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
char* buf2 = taosMemoryMalloc(SHOW_CREATE_TB_RESULT_FIELD2_LEN);
if (NULL == buf2) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t len = 0;
@ -677,10 +693,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
appendTagNameFields(buf2, &len, pCfg);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS (");
code = appendTagValues(buf2, &len, pCfg);
if (code) {
taosMemoryFree(buf2);
return code;
}
TAOS_CHECK_ERRNO(code);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")");
appendTableOptions(buf2, &len, pDbCfg, pCfg);
} else {
@ -692,29 +705,30 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
varDataLen(buf2) = (len > 65535) ? 65535 : len;
colDataSetVal(pCol2, 0, buf2, false);
code = colDataSetVal(pCol2, 0, buf2, false);
TAOS_CHECK_ERRNO(code);
_exit:
taosMemoryFree(buf2);
return TSDB_CODE_SUCCESS;
return code;
}
static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreateViewStmt* pStmt) {
int32_t code = 0;
blockDataEnsureCapacity(pBlock, 1);
QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1));
pBlock->info.rows = 1;
SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
char buf1[SHOW_CREATE_VIEW_RESULT_FIELD1_LEN + 1] = {0};
snprintf(varDataVal(buf1), TSDB_VIEW_FNAME_LEN + 4, "`%s`.`%s`", pStmt->dbName, pStmt->viewName);
varDataSetLen(buf1, strlen(varDataVal(buf1)));
colDataSetVal(pCol1, 0, buf1, false);
QRY_ERR_RET(colDataSetVal(pCol1, 0, buf1, false));
SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
char* buf2 = taosMemoryMalloc(SHOW_CREATE_VIEW_RESULT_FIELD2_LEN);
if (NULL == buf2) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SViewMeta* pMeta = pStmt->pViewMeta;
@ -723,11 +737,10 @@ static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreate
pStmt->dbName, pStmt->viewName, pMeta->querySql);
int32_t len = strlen(varDataVal(buf2));
varDataLen(buf2) = (len > 65535) ? 65535 : len;
colDataSetVal(pCol2, 0, buf2, false);
code = colDataSetVal(pCol2, 0, buf2, false);
taosMemoryFree(buf2);
return TSDB_CODE_SUCCESS;
return code;
}
static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) {
@ -739,7 +752,7 @@ static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRs
if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp);
}
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
return code;
}
@ -832,23 +845,39 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) {
pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData));
if (NULL == pBlock->pDataBlock) {
taosMemoryFree(pBlock);
return TSDB_CODE_OUT_OF_MEMORY;
}
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
if(taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) {
goto _exit;
}
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
if(taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) {
goto _exit;
}
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
if(taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) {
goto _exit;
}
*pOutput = pBlock;
return TSDB_CODE_SUCCESS;
_exit:
if(terrno != TSDB_CODE_SUCCESS) {
taosMemoryFree(pBlock);
taosArrayDestroy(pBlock->pDataBlock);
}
return terrno;
}
static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) {
@ -860,7 +889,7 @@ static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) {
if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp);
}
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
return code;
}
@ -881,14 +910,14 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p
infoData.info.type = pExpr->resType.type;
infoData.info.bytes = pExpr->resType.bytes;
}
blockDataAppendColInfo(pBlock, &infoData);
QRY_ERR_RET(blockDataAppendColInfo(pBlock, &infoData));
}
*pOutput = pBlock;
return TSDB_CODE_SUCCESS;
}
int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) {
blockDataEnsureCapacity(pBlock, 1);
QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1));
int32_t index = 0;
SNode* pProj = NULL;
@ -897,9 +926,9 @@ int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) {
return TSDB_CODE_PAR_INVALID_SELECTED_EXPR;
} else {
if (((SValueNode*)pProj)->isNull) {
colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, NULL, true);
QRY_ERR_RET(colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, NULL, true));
} else {
colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, nodesGetValueFromNode((SValueNode*)pProj), false);
QRY_ERR_RET(colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, nodesGetValueFromNode((SValueNode*)pProj), false));
}
}
}
@ -917,7 +946,7 @@ static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** p
if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, LIST_LENGTH(pSelect->pProjectionList), pRsp);
}
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
return code;
}
@ -930,7 +959,7 @@ static int32_t execShowCreateView(SShowCreateViewStmt* pStmt, SRetrieveTableRsp*
if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_VIEW_RESULT_COLS, pRsp);
}
blockDataDestroy(pBlock);
(void)blockDataDestroy(pBlock);
return code;
}

46
source/libs/command/src/explain.c
View File

@ -199,7 +199,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S
return TSDB_CODE_APP_ERROR;
}
taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx);
if(taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx) == NULL) return terrno;
} else {
for (int32_t i = 0; i < group->nodeNum; ++i) {
rsp = taosArrayGet(group->nodeExecInfo, i);
@ -208,7 +208,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S
return TSDB_CODE_APP_ERROR;
}
taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx);
if(taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx) == NULL) return terrno;
}
}
@ -1920,6 +1920,7 @@ _return:
}
int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
int32_t code = 0;
SExplainCtx *pCtx = (SExplainCtx *)ctx;
int32_t rowNum = taosArrayGetSize(pCtx->rows);
if (rowNum <= 0) {
@ -1929,14 +1930,14 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
SSDataBlock *pBlock = createDataBlock();
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1);
blockDataAppendColInfo(pBlock, &infoData);
blockDataEnsureCapacity(pBlock, rowNum);
QRY_ERR_JRET(blockDataAppendColInfo(pBlock, &infoData));
QRY_ERR_JRET(blockDataEnsureCapacity(pBlock, rowNum));
SColumnInfoData *pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
for (int32_t i = 0; i < rowNum; ++i) {
SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
colDataSetVal(pInfoData, i, row->buf, false);
QRY_ERR_JRET(colDataSetVal(pInfoData, i, row->buf, false));
}
pBlock->info.rows = rowNum;
@ -1946,8 +1947,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
if (NULL == rsp) {
qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
blockDataDestroy(pBlock);
QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
rsp->completed = 1;
@ -1961,10 +1961,11 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
SET_PAYLOAD_LEN(rsp->data, len, len);
_return:
blockDataDestroy(pBlock);
*pRsp = rsp;
return TSDB_CODE_SUCCESS;
QRY_RET(code);
}
int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) {
@ -2099,32 +2100,30 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t
group->nodeExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainRsp));
if (NULL == group->nodeExecInfo) {
qError("taosArrayInit %d explainExecInfo failed", group->nodeNum);
tFreeSExplainRsp(pRspMsg);
taosWUnLockLatch(&group->lock);
QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_ERRNO(code);
}
group->physiPlanExecNum = pRspMsg->numOfPlans;
} else if (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum) {
qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo),
group->nodeNum);
tFreeSExplainRsp(pRspMsg);
taosWUnLockLatch(&group->lock);
QRY_ERR_RET(TSDB_CODE_APP_ERROR);
code = TSDB_CODE_APP_ERROR;
TAOS_CHECK_ERRNO(code);
}
if (group->physiPlanExecNum != pRspMsg->numOfPlans) {
qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum,
groupId);
tFreeSExplainRsp(pRspMsg);
taosWUnLockLatch(&group->lock);
QRY_ERR_RET(TSDB_CODE_APP_ERROR);
code = TSDB_CODE_APP_ERROR;
TAOS_CHECK_ERRNO(code);
}
taosArrayPush(group->nodeExecInfo, pRspMsg);
if(taosArrayPush(group->nodeExecInfo, pRspMsg) == NULL)
{
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_ERRNO(code);
}
groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum);
@ -2140,6 +2139,11 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t
}
return TSDB_CODE_SUCCESS;
_exit:
tFreeSExplainRsp(pRspMsg);
taosWUnLockLatch(&group->lock);
return code;
}
int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) {

2
source/libs/executor/inc/executil.h
View File

@ -179,7 +179,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore);
void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn);
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
int32_t initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
SColumn extractColumnFromColumnNode(SColumnNode* pColNode);

21
source/libs/executor/inc/executorInt.h
View File

@ -834,9 +834,8 @@ void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbI
/**
* @brief copydata from hash table, instead of copying from SGroupResInfo's pRow
*/
int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold,
bool ignoreGroup);
void doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, bool ignoreGroup);
bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo);
@ -862,10 +861,11 @@ int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* p
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name);
void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId);
void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
int32_t setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId,
const char* name);
int32_t setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId);
int32_t setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer);
int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput,
int32_t* rowEntryInfoOffset);
@ -878,7 +878,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList);
void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol);
int32_t setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol);
int32_t checkForQueryBuf(size_t numOfTables);
@ -897,8 +898,8 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup,
SStateStore* pStore);
void appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp,
void* pTbName);
int32_t appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp,
void* pTbName);
uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);

11
source/libs/executor/inc/mergejoin.h
View File

@ -432,6 +432,15 @@ typedef struct SMJoinOperatorInfo {
} \
} while (0)
#define MJ_RET(c) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
} \
return _code; \
} while (0)
void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin);
@ -459,7 +468,7 @@ int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pT
int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable);
int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable);
int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastBuildGrp);
bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build);
int32_t mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build, bool* cont);
int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pFirst, SMJoinGrpRows* pSecond);
int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx);
int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pGrp, bool probeGrp);

4
source/libs/executor/inc/tfill.h
View File

@ -131,11 +131,11 @@ SFillInfo* taosCreateFillInfo(TSKEY skey, int32_t numOfFillCols, int32_t numOfNo
int32_t order, const char* id);
void* taosDestroyFillInfo(struct SFillInfo* pFillInfo);
int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity);
void taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity);
int64_t getFillInfoStart(struct SFillInfo* pFillInfo);
bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData,
int32_t rowIndex);
int32_t rowIndex);
#ifdef __cplusplus
}
#endif

41
source/libs/executor/src/aggregateoperator.c
View File

@ -21,6 +21,7 @@
#include "tname.h"
#include "executorInt.h"
#include "index.h"
#include "operator.h"
#include "query.h"
#include "querytask.h"
@ -29,7 +30,6 @@
#include "tglobal.h"
#include "thash.h"
#include "ttypes.h"
#include "index.h"
typedef struct {
bool hasAgg;
@ -54,13 +54,13 @@ static void destroyAggOperatorInfo(void* param);
static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId);
static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock);
static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock);
static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock);
static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx);
static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx);
static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator);
static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize,
const char* pKey);
const char* pKey);
static int32_t addNewResultRowBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size);
@ -134,7 +134,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
return pOperator;
_error:
_error:
if (pInfo != NULL) {
destroyAggOperatorInfo(pInfo);
}
@ -166,6 +166,8 @@ void destroyAggOperatorInfo(void* param) {
* if false, fill results of ONE GROUP
* */
static bool nextGroupedResult(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SAggOperatorInfo* pAggInfo = pOperator->info;
@ -175,7 +177,6 @@ static bool nextGroupedResult(SOperatorInfo* pOperator) {
SOperatorInfo* downstream = pOperator->pDownstream[0];
int64_t st = taosGetTimestampUs();
int32_t code = TSDB_CODE_SUCCESS;
int32_t order = pAggInfo->binfo.inputTsOrder;
SSDataBlock* pBlock = pAggInfo->pNewGroupBlock;
@ -183,11 +184,10 @@ static bool nextGroupedResult(SOperatorInfo* pOperator) {
pAggInfo->pNewGroupBlock = NULL;
tSimpleHashClear(pAggInfo->aggSup.pResultRowHashTable);
setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId);
setInputDataBlock(pSup, pBlock, order, pBlock->info.scanFlag, true);
QUERY_CHECK_CODE(code, lino, _end);
code = setInputDataBlock(pSup, pBlock, order, pBlock->info.scanFlag, true);
code = doAggregateImpl(pOperator, pSup->pCtx);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
QUERY_CHECK_CODE(code, lino, _end);
}
while (1) {
bool blockAllocated = false;
@ -222,7 +222,9 @@ static bool nextGroupedResult(SOperatorInfo* pOperator) {
}
// the pDataBlock are always the same one, no need to call this again
setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId);
setInputDataBlock(pSup, pBlock, order, pBlock->info.scanFlag, true);
code = setInputDataBlock(pSup, pBlock, order, pBlock->info.scanFlag, true);
QUERY_CHECK_CODE(code, lino, _end);
code = doAggregateImpl(pOperator, pSup->pCtx);
if (code != 0) {
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
@ -238,6 +240,13 @@ static bool nextGroupedResult(SOperatorInfo* pOperator) {
}
initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
return pBlock != NULL;
}
@ -250,7 +259,7 @@ SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
}
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
bool hasNewGroups = false;
bool hasNewGroups = false;
do {
hasNewGroups = nextGroupedResult(pOperator);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
@ -336,7 +345,6 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc
colInfo.info.type = TSDB_DATA_TYPE_NULL;
colInfo.info.bytes = 1;
SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i];
for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) {
SFunctParam* pFuncParam = &pOneExpr->base.pParam[j];
@ -395,8 +403,9 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset;
SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId,
sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup, true);
SResultRow* pResultRow =
doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, sizeof(groupId), true,
groupId, pTaskInfo, false, &pAggInfo->aggSup, true);
/*
* not assign result buffer yet, add new result buffer
* all group belong to one result set, and each group result has different group id so set the id to be one
@ -486,7 +495,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
qError("failed to get buff page size, rowSize:%d", pAggSup->resultRowSize);
return code;
}
if (!osTempSpaceAvailable()) {
code = TSDB_CODE_NO_DISKSPACE;
qError("Init stream agg supporter failed since %s, key:%s, tempDir:%s", tstrerror(code), pKey, tsTempDir);

65
source/libs/executor/src/countwindowoperator.c
View File

@ -25,13 +25,13 @@
#include "ttime.h"
typedef struct SCountWindowResult {
int32_t winRows;
SResultRow row;
int32_t winRows;
SResultRow row;
} SCountWindowResult;
typedef struct SCountWindowSupp {
SArray* pWinStates;
int32_t stateIndex;
SArray* pWinStates;
int32_t stateIndex;
} SCountWindowSupp;
typedef struct SCountWindowOperatorInfo {
@ -61,13 +61,11 @@ void destroyCountWindowOperatorInfo(void* param) {
taosMemoryFreeClear(param);
}
static void clearWinStateBuff(SCountWindowResult* pBuff) {
pBuff->winRows = 0;
}
static void clearWinStateBuff(SCountWindowResult* pBuff) { pBuff->winRows = 0; }
static SCountWindowResult* getCountWinStateInfo(SCountWindowSupp* pCountSup) {
SCountWindowResult* pBuffInfo = taosArrayGet(pCountSup->pWinStates, pCountSup->stateIndex);
int32_t size = taosArrayGetSize(pCountSup->pWinStates);
int32_t size = taosArrayGetSize(pCountSup->pWinStates);
// coverity scan
ASSERTS(size > 0, "WinStates is empty");
if (size > 0) {
@ -76,7 +74,8 @@ static SCountWindowResult* getCountWinStateInfo(SCountWindowSupp* pCountSup) {
return pBuffInfo;
}
static SCountWindowResult* setCountWindowOutputBuff(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SResultRow** pResult) {
static SCountWindowResult* setCountWindowOutputBuff(SExprSupp* pExprSup, SCountWindowSupp* pCountSup,
SResultRow** pResult) {
SCountWindowResult* pBuff = getCountWinStateInfo(pCountSup);
(*pResult) = &pBuff->row;
setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
@ -100,9 +99,9 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
for (int32_t i = 0; i < pBlock->info.rows;) {
SCountWindowResult* pBuffInfo = setCountWindowOutputBuff(pExprSup, &pInfo->countSup, &pInfo->pRow);
int32_t prevRows = pBuffInfo->winRows;
int32_t num = updateCountWindowInfo(i, pBlock->info.rows, pInfo->windowCount, &pBuffInfo->winRows);
int32_t step = num;
int32_t prevRows = pBuffInfo->winRows;
int32_t num = updateCountWindowInfo(i, pBlock->info.rows, pInfo->windowCount, &pBuffInfo->winRows);
int32_t step = num;
if (prevRows == 0) {
pInfo->pRow->win.skey = tsCols[i];
}
@ -136,12 +135,13 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
return code;
}
static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SFilterInfo* pFilterInfo, SSDataBlock* pBlock) {
static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo,
SFilterInfo* pFilterInfo, SSDataBlock* pBlock) {
SResultRow* pResultRow = NULL;
for (int32_t i = 0; i < taosArrayGetSize(pCountSup->pWinStates); i++) {
SCountWindowResult* pBuff = setCountWindowOutputBuff(pExprSup, pCountSup, &pResultRow);
if (pBuff->winRows == 0) {
continue;;
continue;
}
doUpdateNumOfRows(pExprSup->pCtx, pResultRow, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
copyResultrowToDataBlock(pExprSup->pExprInfo, pExprSup->numOfExprs, pResultRow, pExprSup->pCtx, pBlock,
@ -154,6 +154,8 @@ static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, S
}
static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SCountWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pExprSup = &pOperator->exprSupp;
@ -170,8 +172,11 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
}
pRes->info.scanFlag = pBlock->info.scanFlag;
setInputDataBlock(pExprSup, pBlock, order, MAIN_SCAN, true);
blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);
code = setInputDataBlock(pExprSup, pBlock, order, MAIN_SCAN, true);
QUERY_CHECK_CODE(code, lino, _end);
code = blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);
QUERY_CHECK_CODE(code, lino, _end);
// there is an scalar expression that needs to be calculated right before apply the group aggregation.
if (pInfo->scalarSup.pExprInfo != NULL) {
@ -196,11 +201,19 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
}
buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
return pRes->info.rows == 0 ? NULL : pRes;
}
SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SCountWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SCountWindowOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -209,7 +222,6 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
pOperator->exprSupp.hasWindowOrGroup = true;
int32_t code = TSDB_CODE_SUCCESS;
SCountWinodwPhysiNode* pCountWindowNode = (SCountWinodwPhysiNode*)physiNode;
pInfo->tsSlotId = ((SColumnNode*)pCountWindowNode->window.pTspk)->slotId;
@ -218,21 +230,17 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountWindowNode->window.pExprs, NULL, &numOfScalarExpr);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
}
size_t keyBufSize = 0;
size_t keyBufSize = 0;
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pCountWindowNode->window.pFuncs, NULL, &num);
initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pCountWindowNode->window.node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity);
@ -243,7 +251,7 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
pInfo->binfo.outputTsOrder = physiNode->outputTsOrder;
pInfo->windowCount = pCountWindowNode->windowCount;
pInfo->windowSliding = pCountWindowNode->windowSliding;
//sizeof(SCountWindowResult)
// sizeof(SCountWindowResult)
int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize;
int32_t numOfItem = 1;
if (pInfo->windowCount != pInfo->windowSliding) {
@ -257,11 +265,10 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
pInfo->countSup.stateIndex = 0;
code = filterInitFromNode((SNode*)pCountWindowNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
QUERY_CHECK_CODE(code, lino, _error);
setOperatorInfo(pOperator, "CountWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, true, OP_NOT_OPENED, pInfo,
pTaskInfo);

92
source/libs/executor/src/dataDeleter.c
View File

@ -53,7 +53,7 @@ typedef struct SDataDeleterHandle {
TdThreadMutex mutex;
} SDataDeleterHandle;
static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInput, SDataDeleterBuf* pBuf) {
static int32_t toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInput, SDataDeleterBuf* pBuf) {
int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots);
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
@ -65,14 +65,23 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
pBuf->useSize = sizeof(SDataCacheEntry);
SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0);
if (NULL == pColRes) {
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pColSKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 1);
if (NULL == pColSKey) {
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pColEKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 2);
if (NULL == pColEKey) {
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SDeleterRes* pRes = (SDeleterRes*)pEntry->data;
pRes->suid = pHandle->pParam->suid;
pRes->uidList = pHandle->pParam->pUidList;
strcpy(pRes->tableName, pHandle->pDeleter->tableFName);
strcpy(pRes->tsColName, pHandle->pDeleter->tsColName);
TAOS_STRCPY(pRes->tableName, pHandle->pDeleter->tableFName);
TAOS_STRCPY(pRes->tsColName, pHandle->pDeleter->tsColName);
pRes->affectedRows = *(int64_t*)pColRes->pData;
if (pRes->affectedRows) {
@ -88,16 +97,18 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
pBuf->useSize += pEntry->dataLen;
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
(void)atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
(void)atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
return TSDB_CODE_SUCCESS;
}
static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) {
static int32_t allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) {
uint32_t capacity = pDeleter->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueItemSize(pDeleter->pDataBlocks) > capacity) {
qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity,
taosQueueItemSize(pDeleter->pDataBlocks));
return false;
return TSDB_CODE_OUT_OF_MEMORY;
}
pBuf->allocSize = sizeof(SDataCacheEntry) + sizeof(SDeleterRes);
@ -105,55 +116,66 @@ static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDa
pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
if (pBuf->pData == NULL) {
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno));
return terrno;
}
return NULL != pBuf->pData;
return TSDB_CODE_SUCCESS;
}
static int32_t updateStatus(SDataDeleterHandle* pDeleter) {
taosThreadMutexLock(&pDeleter->mutex);
(void)taosThreadMutexLock(&pDeleter->mutex);
int32_t blockNums = taosQueueItemSize(pDeleter->pDataBlocks);
int32_t status =
(0 == blockNums ? DS_BUF_EMPTY
: (blockNums < pDeleter->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL));
pDeleter->status = status;
taosThreadMutexUnlock(&pDeleter->mutex);
(void)taosThreadMutexUnlock(&pDeleter->mutex);
return status;
}
static int32_t getStatus(SDataDeleterHandle* pDeleter) {
taosThreadMutexLock(&pDeleter->mutex);
(void)taosThreadMutexLock(&pDeleter->mutex);
int32_t status = pDeleter->status;
taosThreadMutexUnlock(&pDeleter->mutex);
(void)taosThreadMutexUnlock(&pDeleter->mutex);
return status;
}
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
SDataDeleterBuf* pBuf;
SDataDeleterBuf* pBuf = NULL;
int32_t code = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0, (void**)&pBuf);
if (code) {
return code;
}
if (!allocBuf(pDeleter, pInput, pBuf)) {
code = allocBuf(pDeleter, pInput, pBuf);
if (code) {
taosFreeQitem(pBuf);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
toDataCacheEntry(pDeleter, pInput, pBuf);
taosWriteQitem(pDeleter->pDataBlocks, pBuf);
QRY_ERR_JRET(toDataCacheEntry(pDeleter, pInput, pBuf));
QRY_ERR_JRET(taosWriteQitem(pDeleter->pDataBlocks, pBuf));
*pContinue = (DS_BUF_LOW == updateStatus(pDeleter) ? true : false);
return TSDB_CODE_SUCCESS;
_return:
taosFreeQitem(pBuf);
return code;
}
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
taosThreadMutexLock(&pDeleter->mutex);
(void)taosThreadMutexLock(&pDeleter->mutex);
pDeleter->queryEnd = true;
pDeleter->useconds = useconds;
taosThreadMutexUnlock(&pDeleter->mutex);
(void)taosThreadMutexUnlock(&pDeleter->mutex);
}
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) {
@ -165,9 +187,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw
}
SDataDeleterBuf* pBuf = NULL;
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
(void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) {
memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf));
TAOS_MEMCPY(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf));
taosFreeQitem(pBuf);
}
@ -192,35 +214,35 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
}
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen);
TAOS_MEMCPY(pOutput->pData, pEntry->data, pEntry->dataLen);
pDeleter->pParam->pUidList = NULL;
pOutput->numOfRows = pEntry->numOfRows;
pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed;
atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
(void)atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
taosMemoryFreeClear(pDeleter->nextOutput.pData); // todo persistent
pOutput->bufStatus = updateStatus(pDeleter);
taosThreadMutexLock(&pDeleter->mutex);
(void)taosThreadMutexLock(&pDeleter->mutex);
pOutput->queryEnd = pDeleter->queryEnd;
pOutput->useconds = pDeleter->useconds;
pOutput->precision = pDeleter->pSchema->precision;
taosThreadMutexUnlock(&pDeleter->mutex);
(void)taosThreadMutexUnlock(&pDeleter->mutex);
return TSDB_CODE_SUCCESS;
}
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize);
taosMemoryFreeClear(pDeleter->nextOutput.pData);
taosArrayDestroy(pDeleter->pParam->pUidList);
taosMemoryFree(pDeleter->pParam);
while (!taosQueueEmpty(pDeleter->pDataBlocks)) {
SDataDeleterBuf* pBuf = NULL;
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
(void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) {
taosMemoryFreeClear(pBuf->pData);
@ -228,9 +250,10 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
}
}
taosCloseQueue(pDeleter->pDataBlocks);
taosThreadMutexDestroy(&pDeleter->mutex);
(void)taosThreadMutexDestroy(&pDeleter->mutex);
taosMemoryFree(pDeleter->pManager);
return TSDB_CODE_SUCCESS;
}
@ -247,8 +270,8 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData
SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle));
if (NULL == deleter) {
code = terrno;
taosMemoryFree(pParam);
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
@ -276,17 +299,22 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData
if (code) {
goto _end;
}
taosThreadMutexInit(&deleter->mutex, NULL);
code = taosThreadMutexInit(&deleter->mutex, NULL);
if (code) {
goto _end;
}
*pHandle = deleter;
return code;
_end:
if (deleter != NULL) {
destroyDataSinker((SDataSinkHandle*)deleter);
(void)destroyDataSinker((SDataSinkHandle*)deleter);
taosMemoryFree(deleter);
} else {
taosMemoryFree(pManager);
}
return code;
}

89
source/libs/executor/src/dataDispatcher.c
View File

@ -63,7 +63,7 @@ typedef struct SDataDispatchHandle {
// The length of bitmap is decided by number of rows of this data block, and the length of each column data is
// recorded in the first segment, next to the struct header
// clang-format on
static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
int32_t numOfCols = 0;
SNode* pNode;
@ -88,6 +88,9 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
if (pHandle->pCompressBuf == NULL) {
// allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size
pHandle->pCompressBuf = taosMemoryMalloc(pBuf->allocSize + 8);
if (NULL == pHandle->pCompressBuf) {
QRY_RET(terrno);
}
pHandle->bufSize = pBuf->allocSize + 8;
} else {
if (pHandle->bufSize < pBuf->allocSize + 8) {
@ -96,9 +99,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
if (p != NULL) {
pHandle->pCompressBuf = p;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
qError("failed to prepare compress buf:%d, code: out of memory", pHandle->bufSize);
return;
qError("failed to prepare compress buf:%d, code: %x", pHandle->bufSize, terrno);
return terrno;
}
}
}
@ -114,7 +116,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
pEntry->compressed = 0;
pEntry->dataLen = dataLen;
pEntry->rawLen = dataLen;
memcpy(pEntry->data, pHandle->pCompressBuf, dataLen);
TAOS_MEMCPY(pEntry->data, pHandle->pCompressBuf, dataLen);
}
} else {
pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols);
@ -124,11 +126,13 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
pBuf->useSize += pEntry->dataLen;
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
(void)atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
(void)atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
return TSDB_CODE_SUCCESS;
}
static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) {
static int32_t allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) {
/*
uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) {
@ -142,69 +146,73 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput,
pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
if (pBuf->pData == NULL) {
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno));
qError("SinkNode failed to malloc memory, size:%d, code:%x", pBuf->allocSize, terrno);
return terrno;
}
return NULL != pBuf->pData;
return TSDB_CODE_SUCCESS;
}
static int32_t updateStatus(SDataDispatchHandle* pDispatcher) {
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
int32_t blockNums = taosQueueItemSize(pDispatcher->pDataBlocks);
int32_t status =
(0 == blockNums ? DS_BUF_EMPTY
: (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL));
pDispatcher->status = status;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
return status;
}
static int32_t getStatus(SDataDispatchHandle* pDispatcher) {
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
int32_t status = pDispatcher->status;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
return status;
}
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
int32_t code = 0;
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
SDataDispatchBuf* pBuf;
SDataDispatchBuf* pBuf = NULL;
code = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0, (void**)&pBuf);
if (code) {
return code;
}
if (!allocBuf(pDispatcher, pInput, pBuf)) {
code = allocBuf(pDispatcher, pInput, pBuf);
if (code) {
taosFreeQitem(pBuf);
return TSDB_CODE_OUT_OF_MEMORY;
}
toDataCacheEntry(pDispatcher, pInput, pBuf);
code = taosWriteQitem(pDispatcher->pDataBlocks, pBuf);
if (code != 0) {
return code;
}
QRY_ERR_JRET(toDataCacheEntry(pDispatcher, pInput, pBuf));
QRY_ERR_JRET(taosWriteQitem(pDispatcher->pDataBlocks, pBuf));
int32_t status = updateStatus(pDispatcher);
*pContinue = (status == DS_BUF_LOW || status == DS_BUF_EMPTY);
return TSDB_CODE_SUCCESS;
_return:
taosFreeQitem(pBuf);
return code;
}
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
pDispatcher->queryEnd = true;
pDispatcher->useconds = useconds;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
}
static void resetDispatcher(struct SDataSinkHandle* pHandle) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
pDispatcher->queryEnd = false;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
}
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd) {
@ -216,9 +224,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRow
}
SDataDispatchBuf* pBuf = NULL;
taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
(void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) {
memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf));
TAOS_MEMCPY(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf));
taosFreeQitem(pBuf);
}
@ -243,33 +251,34 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
}
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen);
TAOS_MEMCPY(pOutput->pData, pEntry->data, pEntry->dataLen);
pOutput->numOfRows = pEntry->numOfRows;
pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed;
atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
(void)atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent
pOutput->bufStatus = updateStatus(pDispatcher);
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
pOutput->queryEnd = pDispatcher->queryEnd;
pOutput->useconds = pDispatcher->useconds;
pOutput->precision = pDispatcher->pSchema->precision;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
return TSDB_CODE_SUCCESS;
}
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize);
taosMemoryFreeClear(pDispatcher->nextOutput.pData);
while (!taosQueueEmpty(pDispatcher->pDataBlocks)) {
SDataDispatchBuf* pBuf = NULL;
taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
(void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) {
taosMemoryFreeClear(pBuf->pData);
taosFreeQitem(pBuf);
@ -280,7 +289,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
taosMemoryFreeClear(pDispatcher->pCompressBuf);
pDispatcher->bufSize = 0;
taosThreadMutexDestroy(&pDispatcher->mutex);
(void)taosThreadMutexDestroy(&pDispatcher->mutex);
taosMemoryFree(pDispatcher->pManager);
return TSDB_CODE_SUCCESS;
}
@ -297,7 +306,6 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle));
if (NULL == dispatcher) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return;
}
@ -318,7 +326,11 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
terrno = code;
goto _return;
}
taosThreadMutexInit(&dispatcher->mutex, NULL);
code = taosThreadMutexInit(&dispatcher->mutex, NULL);
if (code) {
terrno = code;
goto _return;
}
if (NULL == dispatcher->pDataBlocks) {
taosMemoryFree(dispatcher);
@ -330,6 +342,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFree(pManager);
return terrno;
}

95
source/libs/executor/src/dataInserter.c
View File

@ -58,10 +58,17 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
SSubmitRspParam* pParam = (SSubmitRspParam*)param;
SDataInserterHandle* pInserter = pParam->pInserter;
pInserter->submitRes.code = code;
if (code) {
pInserter->submitRes.code = code;
}
if (code == TSDB_CODE_SUCCESS) {
pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2));
if (NULL == pInserter->submitRes.pRsp) {
pInserter->submitRes.code = terrno;
goto _return;
}
SDecoder coder = {0};
tDecoderInit(&coder, pMsg->pData, pMsg->len);
code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp);
@ -77,6 +84,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
for (int32_t i = 0; i < numOfTables; ++i) {
SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i);
if (NULL == pRsp) {
pInserter->submitRes.code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
goto _return;
}
if (TSDB_CODE_SUCCESS != pRsp->code) {
code = pRsp->code;
taosMemoryFree(pInserter->submitRes.pRsp);
@ -94,8 +105,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
}
_return:
tsem_post(&pInserter->ready);
(void)tsem_post(&pInserter->ready);
taosMemoryFree(pMsg->pData);
return TSDB_CODE_SUCCESS;
}
@ -105,11 +118,15 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
taosMemoryFreeClear(pMsg);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
SSubmitRspParam* pParam = taosMemoryCalloc(1, sizeof(SSubmitRspParam));
if (NULL == pParam) {
taosMemoryFreeClear(pMsg);
taosMemoryFreeClear(pMsgSendInfo);
return terrno;
}
pParam->pInserter = pInserter;
pMsgSendInfo->param = pParam;
@ -133,7 +150,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int
len += sizeof(SSubmitReq2Msg);
pBuf = taosMemoryMalloc(len);
if (NULL == pBuf) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
((SSubmitReq2Msg*)pBuf)->header.vgId = htonl(vgId);
((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len);
@ -149,6 +166,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int
} else {
taosMemoryFree(pBuf);
}
return code;
}
@ -162,12 +180,10 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
if (NULL == pReq) {
if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
}
@ -208,6 +224,10 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
}
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx);
if (NULL == pColInfoData) {
terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
goto _end;
}
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
switch (pColInfoData->info.type) {
@ -217,13 +237,17 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
ASSERT(pColInfoData->info.type == pCol->type);
if (colDataIsNull_s(pColInfoData, j)) {
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv);
if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
} else {
void* data = colDataGetVarData(pColInfoData, j);
SValue sv = (SValue){
.type = pCol->type, .nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value
SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv);
if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
}
break;
}
@ -245,7 +269,9 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
}
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type
taosArrayPush(pVals, &cv);
if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
} else {
if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && !needSortMerge) {
if (*(int64_t*)var <= lastTs) {
@ -256,9 +282,11 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
}
SValue sv = {.type = pCol->type};
memcpy(&sv.val, var, tDataTypes[pCol->type].bytes);
TAOS_MEMCPY(&sv.val, var, tDataTypes[pCol->type].bytes);
SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv);
if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
}
} else {
uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
@ -274,7 +302,9 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
goto _end;
}
taosArrayPush(tbData.aRowP, &pRow);
if (NULL == taosArrayPush(tbData.aRowP, &pRow)) {
goto _end;
}
}
if (needSortMerge) {
@ -284,9 +314,12 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
}
}
taosArrayPush(pReq->aSubmitTbData, &tbData);
if (NULL == taosArrayPush(pReq->aSubmitTbData, &tbData)) {
goto _end;
}
_end:
taosArrayDestroy(pVals);
if (terrno != 0) {
*ppReq = NULL;
@ -294,9 +327,11 @@ _end:
tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq);
}
return terrno;
}
*ppReq = pReq;
return TSDB_CODE_SUCCESS;
}
@ -312,7 +347,9 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i);
if (NULL == pDataBlock) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid);
if (code) {
if (pReq) {
@ -334,7 +371,9 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
if (!pInserter->explain) {
taosArrayPush(pInserter->pDataBlocks, &pInput->pData);
if (NULL == taosArrayPush(pInserter->pDataBlocks, &pInput->pData)) {
return terrno;
}
void* pMsg = NULL;
int32_t msgLen = 0;
int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen);
@ -350,7 +389,7 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput,
return code;
}
tsem_wait(&pInserter->ready);
QRY_ERR_RET(tsem_wait(&pInserter->ready));
if (pInserter->submitRes.code) {
return pInserter->submitRes.code;
@ -364,10 +403,10 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput,
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
taosThreadMutexLock(&pInserter->mutex);
(void)taosThreadMutexLock(&pInserter->mutex);
pInserter->queryEnd = true;
pInserter->useconds = useconds;
taosThreadMutexUnlock(&pInserter->mutex);
(void)taosThreadMutexUnlock(&pInserter->mutex);
}
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) {
@ -378,12 +417,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize);
taosArrayDestroy(pInserter->pDataBlocks);
taosMemoryFree(pInserter->pSchema);
taosMemoryFree(pInserter->pParam);
taosHashCleanup(pInserter->pCols);
taosThreadMutexDestroy(&pInserter->mutex);
(void)taosThreadMutexDestroy(&pInserter->mutex);
taosMemoryFree(pInserter->pManager);
return TSDB_CODE_SUCCESS;
@ -401,7 +440,6 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle));
if (NULL == inserter) {
taosMemoryFree(pParam);
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return;
}
@ -432,28 +470,31 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
}
inserter->pDataBlocks = taosArrayInit(1, POINTER_BYTES);
taosThreadMutexInit(&inserter->mutex, NULL);
if (NULL == inserter->pDataBlocks) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return;
}
QRY_ERR_JRET(taosThreadMutexInit(&inserter->mutex, NULL));
inserter->fullOrderColList = pInserterNode->pCols->length == inserter->pSchema->numOfCols;
inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT),
false, HASH_NO_LOCK);
if (NULL == inserter->pCols) {
goto _return;
}
SNode* pNode = NULL;
int32_t i = 0;
FOREACH(pNode, pInserterNode->pCols) {
SColumnNode* pCol = (SColumnNode*)pNode;
taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId));
QRY_ERR_JRET(taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId)));
if (inserter->fullOrderColList && pCol->colId != inserter->pSchema->columns[i].colId) {
inserter->fullOrderColList = false;
}
++i;
}
tsem_init(&inserter->ready, 0, 0);
QRY_ERR_JRET(tsem_init(&inserter->ready, 0, 0));
*pHandle = inserter;
return TSDB_CODE_SUCCESS;
@ -461,7 +502,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
_return:
if (inserter) {
destroyDataSinker((SDataSinkHandle*)inserter);
(void)destroyDataSinker((SDataSinkHandle*)inserter);
taosMemoryFree(inserter);
} else {
taosMemoryFree(pManager);

9
source/libs/executor/src/dataSinkMgt.c
View File

@ -23,20 +23,20 @@ SDataSinkStat gDataSinkStat = {0};
int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI, void** ppSinkManager) {
SDataSinkManager* pSinkManager = taosMemoryMalloc(sizeof(SDataSinkManager));
if (NULL == pSinkManager) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pSinkManager->cfg = *cfg;
pSinkManager->pAPI = pAPI;
*ppSinkManager = pSinkManager;
return 0; // to avoid compiler eror
return TSDB_CODE_SUCCESS; // to avoid compiler eror
}
int32_t dsDataSinkGetCacheSize(SDataSinkStat* pStat) {
pStat->cachedSize = atomic_load_64(&gDataSinkStat.cachedSize);
return 0;
return TSDB_CODE_SUCCESS;
}
int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void* pParam, const char* id) {
@ -56,6 +56,7 @@ int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, D
taosMemoryFree(pSinkManager);
qError("invalid input node type:%d, %s", nodeType(pDataSink), id);
return TSDB_CODE_QRY_INVALID_INPUT;
}
@ -97,6 +98,6 @@ void dsScheduleProcess(void* ahandle, void* pItem) {
void dsDestroyDataSinker(DataSinkHandle handle) {
SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle;
pHandleImpl->fDestroy(pHandleImpl);
(void)pHandleImpl->fDestroy(pHandleImpl);
taosMemoryFree(pHandleImpl);
}

222
source/libs/executor/src/dynqueryctrloperator.c
View File

@ -105,7 +105,7 @@ static FORCE_INLINE bool tableNeedCache(int64_t uid, SStbJoinPrevJoinCtx* pPrev,
return (NULL == num) ? false : true;
}
static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) {
static int32_t updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) {
SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev;
SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post;
SStbJoinTableList* pNode = pPrev->pListHead;
@ -141,32 +141,38 @@ static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin)
pPost->leftNeedCache = tableNeedCache(*leftUid, pPrev, pPost, false, pStbJoin->basic.batchFetch);
pPost->rightNeedCache = tableNeedCache(*rightUid, pPrev, pPost, true, pStbJoin->basic.batchFetch);
if (pPost->rightNeedCache && rightPrevUid != pPost->rightCurrUid) {
tSimpleHashPut(pPrev->rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid), NULL, 0);
if (!pStbJoin->basic.batchFetch && pPost->rightNeedCache && rightPrevUid != pPost->rightCurrUid) {
QRY_ERR_RET(tSimpleHashPut(pPrev->rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid), NULL, 0));
pStbJoin->execInfo.rightCacheNum++;
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid, bool needCache, SOperatorParam* pChild) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild, OP_GET_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (pChild) {
(*ppRes)->pChildren = taosArrayInit(1, POINTER_BYTES);
if (NULL == (*ppRes)->pChildren) {
code = terrno;
freeOperatorParam(pChild, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild)) {
code = terrno;
freeOperatorParam(pChild, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
} else {
(*ppRes)->pChildren = NULL;
@ -174,9 +180,10 @@ static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t down
SGcOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcOperatorParam));
if (NULL == pGc) {
code = terrno;
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pGc->sessionId = atomic_add_fetch_64(&gSessionId, 1);
@ -194,16 +201,18 @@ static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t down
static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
(*ppRes)->pChildren = NULL;
SGcNotifyOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcNotifyOperatorParam));
if (NULL == pGc) {
code = terrno;
freeOperatorParam(*ppRes, OP_NOTIFY_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pGc->downstreamIdx = downstreamIdx;
@ -221,13 +230,13 @@ static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_
static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
(*ppRes)->pChildren = NULL;
SExchangeOperatorParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorParam));
if (NULL == pExc) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pExc->multiParams = false;
@ -237,7 +246,7 @@ static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downst
pExc->basic.uidList = taosArrayInit(1, sizeof(int64_t));
if (NULL == pExc->basic.uidList) {
taosMemoryFree(pExc);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
taosArrayPush(pExc->basic.uidList, pUid);
@ -252,14 +261,14 @@ static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downst
static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, SSHashObj* pVg) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
(*ppRes)->pChildren = NULL;
SExchangeOperatorBatchParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorBatchParam));
if (NULL == pExc) {
taosMemoryFreeClear(*ppRes);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pExc->multiParams = true;
@ -267,7 +276,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d
if (NULL == pExc->pBatchs) {
taosMemoryFree(pExc);
taosMemoryFreeClear(*ppRes);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
tSimpleHashSetFreeFp(pExc->pBatchs, freeExchangeGetBasicOperatorParam);
@ -283,7 +292,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d
basic.uidList = pUidList;
basic.tableSeq = false;
tSimpleHashPut(pExc->pBatchs, pVgId, sizeof(*pVgId), &basic, sizeof(basic));
QRY_ERR_RET(tSimpleHashPut(pExc->pBatchs, pVgId, sizeof(*pVgId), &basic, sizeof(basic)));
qTrace("build downstreamIdx %d batch scan, vgId:%d, uidNum:%" PRId64, downstreamIdx, *pVgId, (int64_t)taosArrayGetSize(pUidList));
*(SArray**)p = NULL;
@ -298,39 +307,45 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d
static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initParam, SOperatorParam* pChild0, SOperatorParam* pChild1) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
(*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES);
if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) {
code = terrno;
freeOperatorParam(pChild1, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
SSortMergeJoinOperatorParam* pJoin = taosMemoryMalloc(sizeof(SSortMergeJoinOperatorParam));
if (NULL == pJoin) {
code = terrno;
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pJoin->initDownstream = initParam;
@ -343,31 +358,36 @@ static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initPara
static int32_t buildMergeJoinNotifyOperatorParam(SOperatorParam** ppRes, SOperatorParam* pChild0, SOperatorParam* pChild1) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild0, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
(*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES);
if (NULL == *ppRes) {
code = terrno;
taosMemoryFreeClear(*ppRes);
freeOperatorParam(pChild0, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (pChild0 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) {
code = terrno;
freeOperatorParam(*ppRes, OP_NOTIFY_PARAM);
freeOperatorParam(pChild0, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (pChild1 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) {
code = terrno;
freeOperatorParam(*ppRes, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
(*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN;
@ -407,9 +427,11 @@ static int32_t buildBatchTableScanOperatorParam(SOperatorParam** ppRes, int32_t
static int32_t buildSingleTableScanOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) {
SArray* pUidList = taosArrayInit(1, sizeof(int64_t));
if (NULL == pUidList) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (NULL == taosArrayPush(pUidList, pUid)) {
return terrno;
}
taosArrayPush(pUidList, pUid);
int32_t code = buildTableScanOperatorParam(ppRes, pUidList, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true);
taosArrayDestroy(pUidList);
@ -435,7 +457,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
qDebug("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64,
rowIdx, pPrev->tableNum, *leftVg, *leftUid, *rightVg, *rightUid);
updatePostJoinCurrTableInfo(&pInfo->stbJoin);
QRY_ERR_RET(updatePostJoinCurrTableInfo(&pInfo->stbJoin));
if (pInfo->stbJoin.basic.batchFetch) {
if (pPrev->leftHash) {
@ -538,53 +560,56 @@ static int32_t notifySeqJoinTableCacheEnd(SOperatorInfo* pOperator, SStbJoinPost
return optrDefaultNotifyFn(pOperator->pDownstream[1], pMergeJoinParam);
}
static void handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) {
static int32_t handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) {
SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post;
pPost->isStarted = false;
if (pStbJoin->basic.batchFetch) {
return;
return TSDB_CODE_SUCCESS;
}
if (pPost->leftNeedCache) {
uint32_t* num = tSimpleHashGet(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
if (num && --(*num) <= 0) {
tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
notifySeqJoinTableCacheEnd(pOperator, pPost, true);
(void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, true));
}
}
if (!pPost->rightNeedCache) {
void* v = tSimpleHashGet(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
if (NULL != v) {
tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
notifySeqJoinTableCacheEnd(pOperator, pPost, false);
(void)tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, false));
}
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void seqJoinContinueCurrRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
static FORCE_INLINE int32_t seqJoinContinueCurrRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinPostJoinCtx* pPost = &pInfo->stbJoin.ctx.post;
SStbJoinPrevJoinCtx* pPrev = &pInfo->stbJoin.ctx.prev;
if (!pPost->isStarted) {
return;
return TSDB_CODE_SUCCESS;
}
qDebug("%s dynQueryCtrl continue to retrieve block from post op", GET_TASKID(pOperator->pTaskInfo));
*ppRes = getNextBlockFromDownstream(pOperator, 1);
if (NULL == *ppRes) {
handleSeqJoinCurrRetrieveEnd(pOperator, &pInfo->stbJoin);
QRY_ERR_RET(handleSeqJoinCurrRetrieveEnd(pOperator, &pInfo->stbJoin));
pPrev->pListHead->readIdx++;
} else {
pInfo->stbJoin.execInfo.postBlkNum++;
pInfo->stbJoin.execInfo.postBlkRows += (*ppRes)->info.rows;
return;
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, int32_t keySize, void* pVal, int32_t valSize) {
@ -592,35 +617,38 @@ static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, in
if (NULL == ppArray) {
SArray* pArray = taosArrayInit(10, valSize);
if (NULL == pArray) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (NULL == taosArrayPush(pArray, pVal)) {
taosArrayDestroy(pArray);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (tSimpleHashPut(pHash, pKey, keySize, &pArray, POINTER_BYTES)) {
taosArrayDestroy(pArray);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
if (NULL == taosArrayPush(*ppArray, pVal)) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnceHash, void* pKey, int32_t keySize) {
int32_t code = TSDB_CODE_SUCCESS;
uint32_t* pNum = tSimpleHashGet(pHash, pKey, keySize);
if (NULL == pNum) {
uint32_t n = 1;
if (tSimpleHashPut(pHash, pKey, keySize, &n, sizeof(n))) {
return TSDB_CODE_OUT_OF_MEMORY;
code = tSimpleHashPut(pHash, pKey, keySize, &n, sizeof(n));
if (code) {
return code;
}
if (tSimpleHashPut(pOnceHash, pKey, keySize, NULL, 0)) {
return TSDB_CODE_OUT_OF_MEMORY;
code = tSimpleHashPut(pOnceHash, pKey, keySize, NULL, 0);
if (code) {
return code;
}
return TSDB_CODE_SUCCESS;
}
@ -633,7 +661,7 @@ static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnc
break;
default:
if (1 == (*pNum)) {
tSimpleHashRemove(pOnceHash, pKey, keySize);
(void)tSimpleHashRemove(pOnceHash, pKey, keySize);
}
(*pNum)++;
break;
@ -655,23 +683,40 @@ static void freeStbJoinTableList(SStbJoinTableList* pList) {
}
static int32_t appendStbJoinTableList(SStbJoinPrevJoinCtx* pCtx, int64_t rows, int32_t* pLeftVg, int64_t* pLeftUid, int32_t* pRightVg, int64_t* pRightUid) {
int32_t code = TSDB_CODE_SUCCESS;
SStbJoinTableList* pNew = taosMemoryMalloc(sizeof(SStbJoinTableList));
if (NULL == pNew) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pNew->pLeftVg = taosMemoryMalloc(rows * sizeof(*pLeftVg));
pNew->pLeftUid = taosMemoryMalloc(rows * sizeof(*pLeftUid));
pNew->pRightVg = taosMemoryMalloc(rows * sizeof(*pRightVg));
pNew->pRightUid = taosMemoryMalloc(rows * sizeof(*pRightUid));
if (NULL == pNew->pLeftVg || NULL == pNew->pLeftUid || NULL == pNew->pRightVg || NULL == pNew->pRightUid) {
if (NULL == pNew->pLeftVg) {
code = terrno;
freeStbJoinTableList(pNew);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pNew->pLeftUid = taosMemoryMalloc(rows * sizeof(*pLeftUid));
if (NULL == pNew->pLeftUid) {
code = terrno;
freeStbJoinTableList(pNew);
return code;
}
pNew->pRightVg = taosMemoryMalloc(rows * sizeof(*pRightVg));
if (NULL == pNew->pRightVg) {
code = terrno;
freeStbJoinTableList(pNew);
return code;
}
pNew->pRightUid = taosMemoryMalloc(rows * sizeof(*pRightUid));
if (NULL == pNew->pRightUid) {
code = terrno;
freeStbJoinTableList(pNew);
return code;
}
memcpy(pNew->pLeftVg, pLeftVg, rows * sizeof(*pLeftVg));
memcpy(pNew->pLeftUid, pLeftUid, rows * sizeof(*pLeftUid));
memcpy(pNew->pRightVg, pRightVg, rows * sizeof(*pRightVg));
memcpy(pNew->pRightUid, pRightUid, rows * sizeof(*pRightUid));
TAOS_MEMCPY(pNew->pLeftVg, pLeftVg, rows * sizeof(*pLeftVg));
TAOS_MEMCPY(pNew->pLeftUid, pLeftUid, rows * sizeof(*pLeftUid));
TAOS_MEMCPY(pNew->pRightVg, pRightVg, rows * sizeof(*pRightVg));
TAOS_MEMCPY(pNew->pRightUid, pRightUid, rows * sizeof(*pRightUid));
pNew->readIdx = 0;
pNew->uidNum = rows;
@ -693,9 +738,21 @@ static void doBuildStbJoinTableHash(SOperatorInfo* pOperator, SSDataBlock* pBloc
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SColumnInfoData* pVg0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[0]);
if (NULL == pVg0) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pVg1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[1]);
if (NULL == pVg1) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pUid0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[0]);
if (NULL == pUid0) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pUid1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[1]);
if (NULL == pUid1) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pStbJoin->basic.batchFetch) {
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
@ -731,6 +788,8 @@ static void doBuildStbJoinTableHash(SOperatorInfo* pOperator, SSDataBlock* pBloc
}
}
_return:
if (TSDB_CODE_SUCCESS != code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
@ -754,7 +813,7 @@ static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) {
uint64_t* pUid = NULL;
int32_t iter = 0;
while (NULL != (pUid = tSimpleHashIterate(pStbJoin->ctx.prev.onceTable, pUid, &iter))) {
tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid));
(void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid));
}
pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache);
@ -789,7 +848,7 @@ static void buildStbJoinTableList(SOperatorInfo* pOperator) {
pStbJoin->ctx.prev.joinBuild = true;
}
static void seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
static int32_t seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev;
@ -805,17 +864,17 @@ static void seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppR
seqJoinLaunchNewRetrieveImpl(pOperator, ppRes);
if (*ppRes) {
return;
return TSDB_CODE_SUCCESS;
}
handleSeqJoinCurrRetrieveEnd(pOperator, pStbJoin);
QRY_ERR_RET(handleSeqJoinCurrRetrieveEnd(pOperator, pStbJoin));
pPrev->pListHead->readIdx++;
}
*ppRes = NULL;
setOperatorCompleted(pOperator);
return;
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo* pStbJoin, SSDataBlock* pBlock) {
@ -825,6 +884,7 @@ static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo*
SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SSDataBlock* pRes = NULL;
@ -846,18 +906,23 @@ SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) {
}
}
seqJoinContinueCurrRetrieve(pOperator, &pRes);
QRY_ERR_JRET(seqJoinContinueCurrRetrieve(pOperator, &pRes));
if (pRes) {
goto _return;
}
seqJoinLaunchNewRetrieve(pOperator, &pRes);
QRY_ERR_JRET(seqJoinLaunchNewRetrieve(pOperator, &pRes));
_return:
if (pOperator->cost.openCost == 0) {
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
}
if (code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
}
return pRes ? seqStableJoinComposeRes(pStbJoin, pRes) : NULL;
}
@ -866,24 +931,24 @@ int32_t initSeqStbJoinTableHash(SStbJoinPrevJoinCtx* pPrev, bool batchFetch) {
if (batchFetch) {
pPrev->leftHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pPrev->leftHash) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pPrev->rightHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pPrev->rightHash) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
} else {
pPrev->leftCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->leftCache) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pPrev->rightCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->rightCache) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pPrev->onceTable = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->onceTable) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
@ -893,13 +958,16 @@ int32_t initSeqStbJoinTableHash(SStbJoinPrevJoinCtx* pPrev, bool batchFetch) {
SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo) {
SDynQueryCtrlOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDynQueryCtrlOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
__optr_fn_t nextFp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator == NULL || pInfo == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
__optr_fn_t nextFp = NULL;
SDynQueryCtrlOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDynQueryCtrlOperatorInfo));
if (pInfo == NULL) {
code = terrno;
goto _error;
}
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pOperator == NULL) {
code = terrno;
goto _error;
}
@ -913,7 +981,7 @@ SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32
pInfo->qType = pPhyciNode->qType;
switch (pInfo->qType) {
case DYN_QTYPE_STB_HASH:
memcpy(&pInfo->stbJoin.basic, &pPhyciNode->stbJoin, sizeof(pPhyciNode->stbJoin));
TAOS_MEMCPY(&pInfo->stbJoin.basic, &pPhyciNode->stbJoin, sizeof(pPhyciNode->stbJoin));
pInfo->stbJoin.outputBlkId = pPhyciNode->node.pOutputDataBlockDesc->dataBlockId;
code = initSeqStbJoinTableHash(&pInfo->stbJoin.ctx.prev, pInfo->stbJoin.basic.batchFetch);
if (TSDB_CODE_SUCCESS != code) {
@ -934,12 +1002,14 @@ SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32
return pOperator;
_error:
if (pInfo != NULL) {
destroyDynQueryCtrlOperator(pInfo);
}
taosMemoryFree(pOperator);
pTaskInfo->code = code;
return NULL;
}

46
source/libs/executor/src/eventwindowoperator.c
View File

@ -60,6 +60,8 @@ static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SEventWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SEventWindowOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -71,29 +73,21 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
SEventWinodwPhysiNode* pEventWindowNode = (SEventWinodwPhysiNode*)physiNode;
int32_t tsSlotId = ((SColumnNode*)pEventWindowNode->window.pTspk)->slotId;
int32_t code = filterInitFromNode((SNode*)pEventWindowNode->pStartCond, &pInfo->pStartCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = filterInitFromNode((SNode*)pEventWindowNode->pStartCond, &pInfo->pStartCondInfo, 0);
QUERY_CHECK_CODE(code, lino, _error);
code = filterInitFromNode((SNode*)pEventWindowNode->pEndCond, &pInfo->pEndCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
if (pEventWindowNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pEventWindowNode->window.pExprs, NULL, &numOfScalarExpr);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
}
code = filterInitFromNode((SNode*)pEventWindowNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
@ -103,12 +97,13 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pEventWindowNode->window.node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity);
QUERY_CHECK_NULL(pResBlock, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
code = blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _error);
initBasicInfo(&pInfo->binfo, pResBlock);
initResultRowInfo(&pInfo->binfo.resultRowInfo);
@ -118,7 +113,8 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pEventWindowNode->window.watermark,
.calTrigger = pEventWindowNode->window.triggerType};
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->tsSlotId = tsSlotId;
@ -173,6 +169,8 @@ void destroyEWindowOperatorInfo(void* param) {
}
static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SEventWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -191,8 +189,11 @@ static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator) {
}
pRes->info.scanFlag = pBlock->info.scanFlag;
setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);
code = setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
QUERY_CHECK_CODE(code, lino, _end);
code = blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);
QUERY_CHECK_CODE(code, lino, _end);
// there is an scalar expression that needs to be calculated right before apply the group aggregation.
if (pInfo->scalarSup.pExprInfo != NULL) {
@ -210,6 +211,11 @@ static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator) {
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
return pRes->info.rows == 0 ? NULL : pRes;
}

18
source/libs/executor/src/executil.c
View File

@ -1627,7 +1627,8 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu
SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
for (int32_t i = 0; i < numOfOutput; ++i) {
const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) {
if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)
|| (strcmp(pName, "_group_const_value") == 0)) {
pValCtx[num++] = &pCtx[i];
} else if (fmIsSelectFunc(pCtx[i].functionId)) {
void* data = taosHashGet(pSelectFuncs, pName, strlen(pName));
@ -2021,18 +2022,27 @@ uint64_t tableListGetTableGroupId(const STableListInfo* pTableList, uint64_t tab
// TODO handle the group offset info, fix it, the rule of group output will be broken by this function
int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t gid) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pTableList->map == NULL) {
pTableList->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
QUERY_CHECK_NULL(pTableList->map, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
STableKeyInfo keyInfo = {.uid = uid, .groupId = gid};
taosArrayPush(pTableList->pTableList, &keyInfo);
void* tmp = taosArrayPush(pTableList->pTableList, &keyInfo);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
int32_t slot = (int32_t)taosArrayGetSize(pTableList->pTableList) - 1;
taosHashPut(pTableList->map, &uid, sizeof(uid), &slot, sizeof(slot));
code = taosHashPut(pTableList->map, &uid, sizeof(uid), &slot, sizeof(slot));
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
qDebug("uid:%" PRIu64 ", groupId:%" PRIu64 " added into table list, slot:%d, total:%d", uid, gid, slot, slot + 1);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo,

258
source/libs/executor/src/executor.c
View File

@ -15,14 +15,14 @@
#include "executor.h"
#include "executorInt.h"
#include "trpc.h"
#include "wal.h"
#include "operator.h"
#include "planner.h"
#include "querytask.h"
#include "tdatablock.h"
#include "tref.h"
#include "trpc.h"
#include "tudf.h"
#include "wal.h"
#include "storageapi.h"
@ -31,15 +31,17 @@ int32_t exchangeObjRefPool = -1;
static void cleanupRefPool() {
int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0);
taosCloseRef(ref);
(void)taosCloseRef(ref);
}
static void initRefPool() {
exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo);
atexit(cleanupRefPool);
(void)atexit(cleanupRefPool);
}
static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
if (pOperator->numOfDownstream == 0) {
qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id);
@ -60,33 +62,44 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf
if (type == STREAM_INPUT__MERGED_SUBMIT) {
for (int32_t i = 0; i < numOfBlocks; i++) {
SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData));
taosArrayPush(pInfo->pBlockLists, pReq);
void* tmp = taosArrayPush(pInfo->pBlockLists, pReq);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
} else if (type == STREAM_INPUT__DATA_SUBMIT) {
taosArrayPush(pInfo->pBlockLists, &input);
void* tmp = taosArrayPush(pInfo->pBlockLists, &input);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
} else if (type == STREAM_INPUT__DATA_BLOCK) {
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
SPackedData tmp = {.pDataBlock = pDataBlock};
taosArrayPush(pInfo->pBlockLists, &tmp);
SPackedData tmp = {.pDataBlock = pDataBlock};
void* tmpItem = taosArrayPush(pInfo->pBlockLists, &tmp);
QUERY_CHECK_NULL(tmpItem, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
pInfo->blockType = STREAM_INPUT__DATA_BLOCK;
} else if (type == STREAM_INPUT__CHECKPOINT) {
SPackedData tmp = {.pDataBlock = input};
taosArrayPush(pInfo->pBlockLists, &tmp);
void* tmpItem = taosArrayPush(pInfo->pBlockLists, &tmp);
QUERY_CHECK_NULL(tmpItem, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
pInfo->blockType = STREAM_INPUT__CHECKPOINT;
} else if (type == STREAM_INPUT__REF_DATA_BLOCK) {
for (int32_t i = 0; i < numOfBlocks; ++i) {
for (int32_t i = 0; i < numOfBlocks; ++i) {
SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData));
taosArrayPush(pInfo->pBlockLists, pReq);
void* tmp = taosArrayPush(pInfo->pBlockLists, pReq);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
pInfo->blockType = STREAM_INPUT__DATA_BLOCK;
}
return TSDB_CODE_SUCCESS;
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
}
return code;
}
static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) {
@ -124,7 +137,10 @@ void resetTaskInfo(qTaskInfo_t tinfo) {
clearStreamBlock(pTaskInfo->pRoot);
}
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, const char* id) {
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type,
const char* id) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
if (pOperator->numOfDownstream == 0) {
qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id);
@ -147,24 +163,30 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
if (type == STREAM_INPUT__MERGED_SUBMIT) {
for (int32_t i = 0; i < numOfBlocks; i++) {
SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData));
taosArrayPush(pInfo->pBlockLists, pReq);
void* tmp = taosArrayPush(pInfo->pBlockLists, pReq);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
} else if (type == STREAM_INPUT__DATA_SUBMIT) {
taosArrayPush(pInfo->pBlockLists, input);
void* tmp = taosArrayPush(pInfo->pBlockLists, input);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
} else if (type == STREAM_INPUT__DATA_BLOCK) {
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
SPackedData tmp = {.pDataBlock = pDataBlock};
taosArrayPush(pInfo->pBlockLists, &tmp);
void* tmpItem = taosArrayPush(pInfo->pBlockLists, &tmp);
QUERY_CHECK_NULL(tmpItem, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
pInfo->blockType = STREAM_INPUT__DATA_BLOCK;
} else if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
SPackedData tmp = {.pDataBlock = input};
taosArrayPush(pInfo->pBlockLists, &tmp);
void* tmpItem = taosArrayPush(pInfo->pBlockLists, &tmp);
QUERY_CHECK_NULL(tmpItem, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
pInfo->blockType = STREAM_INPUT__CHECKPOINT;
} else {
ASSERT(0);
@ -172,6 +194,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
return TSDB_CODE_SUCCESS;
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI* pAPI) {
@ -330,16 +358,25 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v
return NULL;
}
qStreamInfoResetTimewindowFilter(pTaskInfo);
code = qStreamInfoResetTimewindowFilter(pTaskInfo);
if (code != TSDB_CODE_SUCCESS) {
nodesDestroyNode((SNode*)pPlan);
qDestroyTask(pTaskInfo);
terrno = code;
return NULL;
}
return pTaskInfo;
}
static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr,
SStorageAPI* pAPI) {
static int32_t filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr,
SStorageAPI* pAPI, SArray** ppArrayRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
int32_t numOfUids = taosArrayGetSize(tableIdList);
if (numOfUids == 0) {
return qa;
(*ppArrayRes) = qa;
goto _end;
}
STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info;
@ -395,11 +432,18 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
}
// handle multiple partition
taosArrayPush(qa, id);
void* tmp = taosArrayPush(qa, id);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
pAPI->metaReaderFn.clearReader(&mr);
return qa;
(*ppArrayRes) = qa;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) {
@ -416,7 +460,12 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
SStreamScanInfo* pScanInfo = pInfo->info;
if (isAdd) { // add new table id
SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id, &pTaskInfo->storageAPI);
SArray* qa = NULL;
code = filterUnqualifiedTables(pScanInfo, tableIdList, id, &pTaskInfo->storageAPI, &qa);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(qa);
return code;
}
int32_t numOfQualifiedTables = taosArrayGetSize(qa);
qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id);
code = pTaskInfo->storageAPI.tqReaderFn.tqReaderAddTables(pScanInfo->tqReader, qa);
@ -459,7 +508,13 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
}
}
tableListAddTableInfo(pTableListInfo, keyInfo.uid, keyInfo.groupId);
code = tableListAddTableInfo(pTableListInfo, keyInfo.uid, keyInfo.groupId);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(keyBuf);
taosArrayDestroy(qa);
taosWUnLockLatch(&pTaskInfo->lock);
return code;
}
}
taosWUnLockLatch(&pTaskInfo->lock);
@ -509,16 +564,14 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
return TSDB_CODE_SUCCESS;
}
bool qIsDynamicExecTask(qTaskInfo_t tinfo) {
return ((SExecTaskInfo*)tinfo)->dynamicTask;
}
bool qIsDynamicExecTask(qTaskInfo_t tinfo) { return ((SExecTaskInfo*)tinfo)->dynamicTask; }
void destroyOperatorParam(SOperatorParam* pParam) {
if (NULL == pParam) {
return;
}
//TODO
// TODO
}
void qUpdateOperatorParam(qTaskInfo_t tinfo, void* pParam) {
@ -531,7 +584,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, int8_t compressResult, char* sql,
EOPTR_EXEC_MODEL model) {
SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
taosThreadOnce(&initPoolOnce, initRefPool);
(void)taosThreadOnce(&initPoolOnce, initRefPool);
qDebug("start to create task, TID:0x%" PRIx64 " QID:0x%" PRIx64 ", vgId:%d", taskId, pSubplan->id.queryId, vgId);
@ -543,7 +596,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
if (handle) {
SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50, .compress = compressResult};
void* pSinkManager = NULL;
void* pSinkManager = NULL;
code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI, &pSinkManager);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to dsDataSinkMgtInit, code:%s, %s", tstrerror(code), (*pTask)->id.str);
@ -575,6 +628,8 @@ static void freeBlock(void* param) {
}
int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
int64_t threadId = taosGetSelfPthreadId();
@ -605,7 +660,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
int32_t ret = setjmp(pTaskInfo->env);
if (ret != TSDB_CODE_SUCCESS) {
pTaskInfo->code = ret;
cleanUpUdfs();
(void)cleanUpUdfs();
qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
atomic_store_64(&pTaskInfo->owner, 0);
@ -627,10 +682,10 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot);
}
if(pRes == NULL) {
if (pRes == NULL) {
st = taosGetTimestampUs();
}
int32_t rowsThreshold = pTaskInfo->pSubplan->rowsThreshold;
if (!pTaskInfo->pSubplan->dynamicRowThreshold || 4096 <= pTaskInfo->pSubplan->rowsThreshold) {
rowsThreshold = 4096;
@ -640,18 +695,21 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
SSDataBlock* p = NULL;
if (blockIndex >= taosArrayGetSize(pTaskInfo->pResultBlockList)) {
SSDataBlock* p1 = createOneDataBlock(pRes, true);
taosArrayPush(pTaskInfo->pResultBlockList, &p1);
void* tmp = taosArrayPush(pTaskInfo->pResultBlockList, &p1);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
p = p1;
} else {
p = *(SSDataBlock**)taosArrayGet(pTaskInfo->pResultBlockList, blockIndex);
copyDataBlock(p, pRes);
code = copyDataBlock(p, pRes);
QUERY_CHECK_CODE(code, lino, _end);
}
blockIndex += 1;
current += p->info.rows;
ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT);
taosArrayPush(pResList, &p);
void* tmp = taosArrayPush(pResList, &p);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
if (current >= rowsThreshold) {
break;
@ -671,7 +729,9 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
*useconds = pTaskInfo->cost.elapsedTime;
}
cleanUpUdfs();
_end:
(void)cleanUpUdfs();
uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows;
qDebug("%s task suspended, %d rows in %d blocks returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",
@ -730,7 +790,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
int32_t ret = setjmp(pTaskInfo->env);
if (ret != TSDB_CODE_SUCCESS) {
pTaskInfo->code = ret;
cleanUpUdfs();
(void)cleanUpUdfs();
qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
atomic_store_64(&pTaskInfo->owner, 0);
return pTaskInfo->code;
@ -748,7 +808,8 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
*useconds = pTaskInfo->cost.elapsedTime;
}
cleanUpUdfs();
int32_t tmpRes = cleanUpUdfs();
qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);
int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0;
uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows;
@ -762,9 +823,13 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo) {
taosWLockLatch(&pTaskInfo->stopInfo.lock);
taosArrayPush(pTaskInfo->stopInfo.pStopInfo, pInfo);
void* tmp = taosArrayPush(pTaskInfo->stopInfo.pStopInfo, pInfo);
taosWUnLockLatch(&pTaskInfo->stopInfo.lock);
if (!tmp) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
}
@ -798,8 +863,8 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) {
SExchangeOpStopInfo* pStop = taosArrayGet(pTaskInfo->stopInfo.pStopInfo, i);
SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pStop->refId);
if (pExchangeInfo) {
tsem_post(&pExchangeInfo->ready);
taosReleaseRef(exchangeObjRefPool, pStop->refId);
(void)tsem_post(&pExchangeInfo->ready);
(void)taosReleaseRef(exchangeObjRefPool, pStop->refId);
}
}
@ -829,12 +894,12 @@ int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) {
qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo));
setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED);
while(1) {
while (1) {
taosWLockLatch(&pTaskInfo->lock);
if (qTaskIsExecuting(pTaskInfo)) { // let's wait for 100 ms and try again
if (qTaskIsExecuting(pTaskInfo)) { // let's wait for 100 ms and try again
taosWUnLockLatch(&pTaskInfo->lock);
taosMsleep(100);
} else { // not running now
} else { // not running now
pTaskInfo->code = rspCode;
taosWUnLockLatch(&pTaskInfo->lock);
return TSDB_CODE_SUCCESS;
@ -907,7 +972,7 @@ int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {
}
}
int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow) {
int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange* pVerRange, STimeWindow* pWindow) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
@ -917,14 +982,14 @@ int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRan
pStreamInfo->fillHistoryWindow = *pWindow;
pStreamInfo->recoverStep = STREAM_RECOVER_STEP__PREPARE1;
qDebug("%s step 1. set param for stream scanner for scan-history data, verRange:%" PRId64 " - %" PRId64 ", window:%" PRId64
" - %" PRId64,
qDebug("%s step 1. set param for stream scanner for scan-history data, verRange:%" PRId64 " - %" PRId64
", window:%" PRId64 " - %" PRId64,
GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey,
pWindow->ekey);
return 0;
}
int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow) {
int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange* pVerRange, STimeWindow* pWindow) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
@ -955,9 +1020,10 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
while (1) {
int32_t type = pOperator->operatorType;
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL ||
type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
@ -974,7 +1040,7 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
@ -989,7 +1055,7 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
pInfo->ignoreExpiredData = false;
} else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) {
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
@ -1004,7 +1070,7 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
pInfo->ignoreExpiredData = false;
} else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) {
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
@ -1019,7 +1085,7 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
pInfo->ignoreExpiredData = false;
} else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT) {
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
STimeWindowAggSupp* pSup = &pInfo->twAggSup;
ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
@ -1137,20 +1203,21 @@ void qStreamSetSourceExcluded(qTaskInfo_t tinfo, int8_t sourceExcluded) {
}
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
int32_t code = TSDB_CODE_SUCCESS;
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SOperatorInfo* pOperator = pTaskInfo->pRoot;
const char* id = GET_TASKID(pTaskInfo);
if(subType == TOPIC_SUB_TYPE__COLUMN && pOffset->type == TMQ_OFFSET__LOG){
if (subType == TOPIC_SUB_TYPE__COLUMN && pOffset->type == TMQ_OFFSET__LOG) {
pOperator = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id);
if (pOperator == NULL) {
return -1;
}
SStreamScanInfo* pInfo = pOperator->info;
SStoreTqReader* pReaderAPI = &pTaskInfo->storageAPI.tqReaderFn;
SWalReader* pWalReader = pReaderAPI->tqReaderGetWalReader(pInfo->tqReader);
SStoreTqReader* pReaderAPI = &pTaskInfo->storageAPI.tqReaderFn;
SWalReader* pWalReader = pReaderAPI->tqReaderGetWalReader(pInfo->tqReader);
walReaderVerifyOffset(pWalReader, pOffset);
}
// if pOffset equal to current offset, means continue consume
@ -1229,9 +1296,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
int64_t oldSkey = pScanBaseInfo->cond.twindows.skey;
// let's start from the next ts that returned to consumer.
if(pTaskInfo->storageAPI.tqReaderFn.tqGetTablePrimaryKey(pInfo->tqReader)){
if (pTaskInfo->storageAPI.tqReaderFn.tqGetTablePrimaryKey(pInfo->tqReader)) {
pScanBaseInfo->cond.twindows.skey = ts;
}else{
} else {
pScanBaseInfo->cond.twindows.skey = ts + 1;
}
pScanInfo->scanTimes = 0;
@ -1249,8 +1316,17 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s",
uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
} else {
pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
code = pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
return code;
}
code = pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
return code;
}
qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s",
uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
}
@ -1289,20 +1365,32 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
}
pAPI->snapshotFn.taosXSetTablePrimaryKey(sContext, mtInfo.uid);
initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
if(pAPI->snapshotFn.taosXGetTablePrimaryKey(sContext)){
code = initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
return code;
}
if (pAPI->snapshotFn.taosXGetTablePrimaryKey(sContext)) {
pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
}else{
} else {
pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts + 1;
}
tableListAddTableInfo(pTableListInfo, mtInfo.uid, 0);
code = tableListAddTableInfo(pTableListInfo, mtInfo.uid, 0);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
return code;
}
STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
int32_t size = tableListGetSize(pTableListInfo);
pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL,
(void**)&pInfo->dataReader, NULL, NULL);
code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size,
NULL, (void**)&pInfo->dataReader, NULL, NULL);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
return code;
}
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
@ -1353,12 +1441,14 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
}
}
pSendInfo->fp(pSendInfo->param, &buf, pMsg->code);
(void)pSendInfo->fp(pSendInfo->param, &buf, pMsg->code);
rpcFreeCont(pMsg->pCont);
destroySendMsgInfo(pSendInfo);
}
SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = tinfo;
SArray* plist = getTableListInfo(pTaskInfo);
@ -1366,30 +1456,50 @@ SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) {
STableListInfo* pTableListInfo = taosArrayGetP(plist, 0);
SArray* pUidList = taosArrayInit(10, sizeof(uint64_t));
QUERY_CHECK_NULL(pUidList, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
int32_t numOfTables = tableListGetSize(pTableListInfo);
for (int32_t i = 0; i < numOfTables; ++i) {
STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i);
taosArrayPush(pUidList, &pKeyInfo->uid);
void* tmp = taosArrayPush(pUidList, &pKeyInfo->uid);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
taosArrayDestroy(plist);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
return pUidList;
}
static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamScanInfo* pScanInfo = pOperator->info;
STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info;
taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo);
void* tmp = taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pScanInfo = pOperator->info;
taosArrayPush(pList, &pScanInfo->base.pTableListInfo);
void* tmp = taosArrayPush(pList, &pScanInfo->base.pTableListInfo);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
} else {
if (pOperator->pDownstream != NULL && pOperator->pDownstream[0] != NULL) {
extractTableList(pList, pOperator->pDownstream[0]);
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
}
SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) {
@ -1400,13 +1510,13 @@ SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) {
}
int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tInfo;
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tInfo;
pTaskInfo->pRoot->fpSet.releaseStreamStateFn(pTaskInfo->pRoot);
return 0;
}
int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tInfo;
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tInfo;
pTaskInfo->pRoot->fpSet.reloadStreamStateFn(pTaskInfo->pRoot);
return 0;
}

223
source/libs/executor/src/executorInt.c
View File

@ -72,14 +72,14 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) {
#define realloc u_realloc
#endif
static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock);
static int32_t setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock);
static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol);
static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
static void doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup);
SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize) {
@ -192,8 +192,12 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
// add a new result set for a new group
SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset};
tSimpleHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos,
sizeof(SResultRowPosition));
int32_t code = tSimpleHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos,
sizeof(SResultRowPosition));
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
}
// 2. set the new time window to be the new active time window
@ -209,11 +213,14 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
}
// query_range_start, query_range_end, window_duration, window_start, window_end
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) {
int32_t initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) {
pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP;
pColData->info.bytes = sizeof(int64_t);
colInfoDataEnsureCapacity(pColData, 5, false);
int32_t code = colInfoDataEnsureCapacity(pColData, 5, false);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
colDataSetInt64(pColData, 0, &pQueryWindow->skey);
colDataSetInt64(pColData, 1, &pQueryWindow->ekey);
@ -221,35 +228,46 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
colDataSetInt64(pColData, 2, &interval); // this value may be variable in case of 'n' and 'y'.
colDataSetInt64(pColData, 3, &pQueryWindow->skey);
colDataSetInt64(pColData, 4, &pQueryWindow->ekey);
return TSDB_CODE_SUCCESS;
}
static void doSetInputDataBlockInfo(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) {
static int32_t doSetInputDataBlockInfo(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SqlFunctionCtx* pCtx = pExprSup->pCtx;
for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) {
pCtx[i].order = order;
pCtx[i].input.numOfRows = pBlock->info.rows;
setBlockSMAInfo(&pCtx[i], &pExprSup->pExprInfo[i], pBlock);
code = setBlockSMAInfo(&pCtx[i], &pExprSup->pExprInfo[i], pBlock);
QUERY_CHECK_CODE(code, lino, _end);
pCtx[i].pSrcBlock = pBlock;
pCtx[i].scanFlag = scanFlag;
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void setInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol) {
int32_t setInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol) {
if (pBlock->pBlockAgg != NULL) {
doSetInputDataBlockInfo(pExprSup, pBlock, order, scanFlag);
return doSetInputDataBlockInfo(pExprSup, pBlock, order, scanFlag);
} else {
doSetInputDataBlock(pExprSup, pBlock, order, scanFlag, createDummyCol);
return doSetInputDataBlock(pExprSup, pBlock, order, scanFlag, createDummyCol);
}
}
static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t paramIndex,
int32_t numOfRows) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SColumnInfoData* pColInfo = NULL;
if (pInput->pData[paramIndex] == NULL) {
pColInfo = taosMemoryCalloc(1, sizeof(SColumnInfoData));
if (pColInfo == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
QUERY_CHECK_NULL(pColInfo, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
// Set the correct column info (data type and bytes)
pColInfo->info.type = pFuncParam->param.nType;
@ -260,7 +278,8 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc
pColInfo = pInput->pData[paramIndex];
}
colInfoDataEnsureCapacity(pColInfo, numOfRows, false);
code = colInfoDataEnsureCapacity(pColInfo, numOfRows, false);
QUERY_CHECK_CODE(code, lino, _end);
int8_t type = pFuncParam->param.nType;
if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) {
@ -277,12 +296,17 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc
char* tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE);
STR_WITH_SIZE_TO_VARSTR(tmp, pFuncParam->param.pz, pFuncParam->param.nLen);
for (int32_t i = 0; i < numOfRows; ++i) {
colDataSetVal(pColInfo, i, tmp, false);
code = colDataSetVal(pColInfo, i, tmp, false);
QUERY_CHECK_CODE(code, lino, _end);
}
taosMemoryFree(tmp);
}
return TSDB_CODE_SUCCESS;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
@ -302,7 +326,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
pInput->colDataSMAIsSet = false;
SExprInfo* pOneExpr = &pExprSup->pExprInfo[i];
bool hasPk = pOneExpr->pExpr->nodeType == QUERY_NODE_FUNCTION && pOneExpr->pExpr->_function.pFunctNode->hasPk;
bool hasPk = pOneExpr->pExpr->nodeType == QUERY_NODE_FUNCTION && pOneExpr->pExpr->_function.pFunctNode->hasPk;
pCtx[i].hasPrimaryKey = hasPk;
int16_t tsParamIdx = (!hasPk) ? pOneExpr->base.numOfParams - 1 : pOneExpr->base.numOfParams - 2;
@ -320,13 +344,13 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
// NOTE: the last parameter is the primary timestamp column
// todo: refactor this
if (fmIsImplicitTsFunc(pCtx[i].functionId) && (j == tsParamIdx)) {
pInput->pPTS = pInput->pData[j]; // in case of merge function, this is not always the ts column data.
}
if (hasPk && (j == pkParamIdx)) {
pInput->pPrimaryKey = pInput->pData[j];
}
}
ASSERT(pInput->pData[j] != NULL);
} else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
// todo avoid case: top(k, 12), 12 is the value parameter.
@ -419,7 +443,9 @@ static int32_t doCreateConstantValColumnSMAInfo(SInputColumnInfoData* pInput, SF
return TSDB_CODE_SUCCESS;
}
void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock) {
int32_t setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t numOfRows = pBlock->info.rows;
SInputColumnInfoData* pInput = &pCtx->input;
@ -443,12 +469,19 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
// the data in the corresponding SColumnInfoData will not be used.
pInput->pData[j] = taosArrayGet(pBlock->pDataBlock, slotId);
} else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
doCreateConstantValColumnSMAInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows);
code = doCreateConstantValColumnSMAInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows);
QUERY_CHECK_CODE(code, lino, _end);
}
}
} else {
pInput->colDataSMAIsSet = false;
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
/////////////////////////////////////////////////////////////////////////////////////////////
@ -468,7 +501,8 @@ STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) {
return win;
}
int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset) {
int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput,
int32_t* rowEntryInfoOffset) {
bool init = false;
for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].resultInfo = getResultEntryInfo(pResult, i, rowEntryInfoOffset);
@ -488,7 +522,7 @@ int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t n
if (!pResInfo->initialized) {
if (pCtx[i].functionId != -1) {
bool ini = pCtx[i].fpSet.init(&pCtx[i], pResInfo);
if (!ini && fmIsUserDefinedFunc(pCtx[i].functionId)){
if (!ini && fmIsUserDefinedFunc(pCtx[i].functionId)) {
pResInfo->initialized = false;
return TSDB_CODE_UDF_FUNC_EXEC_FAILURE;
}
@ -517,6 +551,8 @@ void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
}
int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pFilterInfo == NULL || pBlock->info.rows == 0) {
return TSDB_CODE_SUCCESS;
}
@ -524,16 +560,12 @@ int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* p
SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock};
SColumnInfoData* p = NULL;
int32_t code = filterSetDataFromSlotId(pFilterInfo, &param1);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
code = filterSetDataFromSlotId(pFilterInfo, &param1);
QUERY_CHECK_CODE(code, lino, _err);
int32_t status = 0;
code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
QUERY_CHECK_CODE(code, lino, _err);
extractQualifiedTupleByFilterResult(pBlock, p, status);
@ -544,7 +576,8 @@ int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* p
if (pInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pInfo->dstSlotId);
if (pColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
blockDataUpdateTsWindow(pBlock, pInfo->dstSlotId);
code = blockDataUpdateTsWindow(pBlock, pInfo->dstSlotId);
QUERY_CHECK_CODE(code, lino, _err);
break;
}
}
@ -598,12 +631,15 @@ void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExpr
void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultRow* pRow, SqlFunctionCtx* pCtx,
SSDataBlock* pBlock, const int32_t* rowEntryOffset, SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
for (int32_t j = 0; j < numOfExprs; ++j) {
int32_t slotId = pExprInfo[j].base.resSchema.slotId;
pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);
if (pCtx[j].fpSet.finalize) {
if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_key") == 0) {
if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_key") == 0 ||
strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_const_value") == 0) {
// for groupkey along with functions that output multiple lines(e.g. Histogram)
// need to match groupkey result for each output row of that function.
if (pCtx[j].resultInfo->numOfRes != 0) {
@ -611,11 +647,13 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR
}
}
blockDataEnsureCapacity(pBlock, pBlock->info.rows + pCtx[j].resultInfo->numOfRes);
int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
if (TAOS_FAILED(code)) {
qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
code = blockDataEnsureCapacity(pBlock, pBlock->info.rows + pCtx[j].resultInfo->numOfRes);
QUERY_CHECK_CODE(code, lino, _end);
int32_t winCode = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
if (TAOS_FAILED(winCode)) {
qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(winCode));
T_LONG_JMP(pTaskInfo->env, winCode);
}
} else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
// do nothing
@ -625,10 +663,17 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
for (int32_t k = 0; k < pRow->numOfRows; ++k) {
colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
QUERY_CHECK_CODE(code, lino, _end);
}
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
}
// todo refactor. SResultRow has direct pointer in miainfo
@ -671,9 +716,10 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos
return 0;
}
int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold,
bool ignoreGroup) {
void doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, bool ignoreGroup) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExprInfo* pExprInfo = pSup->pExprInfo;
int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
@ -724,7 +770,8 @@ int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock,
if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - iter) > 1 ? 1 : 0);
blockDataEnsureCapacity(pBlock, newSize);
code = blockDataEnsureCapacity(pBlock, newSize);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", newSize,
pBlock->info.capacity, GET_TASKID(pTaskInfo));
// todo set the pOperator->resultInfo size
@ -744,14 +791,22 @@ int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock,
}
qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
pBlock->info.id.groupId);
pBlock->info.id.groupId);
pBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pBlock, 0);
return 0;
code = blockDataUpdateTsWindow(pBlock, 0);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
}
int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup) {
void doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExprInfo* pExprInfo = pSup->pExprInfo;
int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
@ -792,7 +847,8 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - i) > 1 ? 1 : 0);
blockDataEnsureCapacity(pBlock, newSize);
code = blockDataEnsureCapacity(pBlock, newSize);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", newSize,
pBlock->info.capacity, GET_TASKID(pTaskInfo));
// todo set the pOperator->resultInfo size
@ -811,8 +867,14 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
pBlock->info.id.groupId);
pBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pBlock, 0);
return 0;
code = blockDataUpdateTsWindow(pBlock, 0);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
}
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
@ -909,9 +971,9 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock) {
initResultRowInfo(&pInfo->resultRowInfo);
}
static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, SExprInfo* pExpr, int32_t numOfOutput) {
static void destroySqlFunctionCtx(SqlFunctionCtx* pCtx, SExprInfo* pExpr, int32_t numOfOutput) {
if (pCtx == NULL) {
return NULL;
return;
}
for (int32_t i = 0; i < numOfOutput; ++i) {
@ -939,7 +1001,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, SExprInfo* pExpr, int32
}
taosMemoryFreeClear(pCtx);
return NULL;
return;
}
int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore) {
@ -970,7 +1032,10 @@ void cleanupExprSupp(SExprSupp* pSupp) {
taosMemoryFree(pSupp->rowEntryInfoOffset);
}
void cleanupBasicInfo(SOptrBasicInfo* pInfo) { blockDataDestroy(pInfo->pRes); pInfo->pRes = NULL;}
void cleanupBasicInfo(SOptrBasicInfo* pInfo) {
blockDataDestroy(pInfo->pRes);
pInfo->pRes = NULL;
}
bool groupbyTbname(SNodeList* pGroupList) {
bool bytbname = false;
@ -1019,7 +1084,12 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo*
for (int32_t i = 0; i < numOfTables; ++i) {
STableKeyInfo* pTable = tableListGetInfo(pTableListInfo, i);
taosArrayPush(pDeleterParam->pUidList, &pTable->uid);
void* tmp = taosArrayPush(pDeleterParam->pUidList, &pTable->uid);
if (!tmp) {
taosArrayDestroy(pDeleterParam->pUidList);
taosMemoryFree(pDeleterParam);
return TSDB_CODE_OUT_OF_MEMORY;
}
}
*pParam = pDeleterParam;
@ -1056,7 +1126,7 @@ void freeOperatorParamImpl(SOperatorParam* pParam, SOperatorParamType type) {
taosArrayDestroy(pParam->pChildren);
taosMemoryFree(pParam->value);
taosMemoryFree(pParam);
}
@ -1077,25 +1147,15 @@ void freeExchangeGetOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_GET_PARAM);
}
void freeExchangeNotifyOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM);
}
void freeExchangeNotifyOperatorParam(SOperatorParam* pParam) { freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM); }
void freeGroupCacheGetOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_GET_PARAM);
}
void freeGroupCacheGetOperatorParam(SOperatorParam* pParam) { freeOperatorParamImpl(pParam, OP_GET_PARAM); }
void freeGroupCacheNotifyOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM);
}
void freeGroupCacheNotifyOperatorParam(SOperatorParam* pParam) { freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM); }
void freeMergeJoinGetOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_GET_PARAM);
}
void freeMergeJoinGetOperatorParam(SOperatorParam* pParam) { freeOperatorParamImpl(pParam, OP_GET_PARAM); }
void freeMergeJoinNotifyOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM);
}
void freeMergeJoinNotifyOperatorParam(SOperatorParam* pParam) { freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM); }
void freeTableScanGetOperatorParam(SOperatorParam* pParam) {
STableScanOperatorParam* pTableScanParam = (STableScanOperatorParam*)pParam->value;
@ -1103,16 +1163,13 @@ void freeTableScanGetOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_GET_PARAM);
}
void freeTableScanNotifyOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM);
}
void freeTableScanNotifyOperatorParam(SOperatorParam* pParam) { freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM); }
void freeOperatorParam(SOperatorParam* pParam, SOperatorParamType type) {
if (NULL == pParam) {
return;
}
switch (pParam->opType) {
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
type == OP_GET_PARAM ? freeExchangeGetOperatorParam(pParam) : freeExchangeNotifyOperatorParam(pParam);
@ -1133,7 +1190,7 @@ void freeOperatorParam(SOperatorParam* pParam, SOperatorParamType type) {
}
void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamType type, bool allFree) {
SOperatorParam** ppParam = NULL;
SOperatorParam** ppParam = NULL;
SOperatorParam*** pppDownstramParam = NULL;
switch (type) {
case OP_GET_PARAM:
@ -1166,22 +1223,22 @@ void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamType
}
}
FORCE_INLINE SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam) {
FORCE_INLINE SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx,
bool clearParam) {
if (pOperator->pDownstreamGetParams && pOperator->pDownstreamGetParams[idx]) {
qDebug("DynOp: op %s start to get block from downstream %s", pOperator->name, pOperator->pDownstream[idx]->name);
SSDataBlock* pBlock = pOperator->pDownstream[idx]->fpSet.getNextExtFn(pOperator->pDownstream[idx], pOperator->pDownstreamGetParams[idx]);
SSDataBlock* pBlock = pOperator->pDownstream[idx]->fpSet.getNextExtFn(pOperator->pDownstream[idx],
pOperator->pDownstreamGetParams[idx]);
if (clearParam) {
freeOperatorParam(pOperator->pDownstreamGetParams[idx], OP_GET_PARAM);
pOperator->pDownstreamGetParams[idx] = NULL;
}
return pBlock;
}
return pOperator->pDownstream[idx]->fpSet.getNextFn(pOperator->pDownstream[idx]);
}
bool compareVal(const char* v, const SStateKeys* pKey) {
if (IS_VAR_DATA_TYPE(pKey->type)) {
if (varDataLen(v) != varDataLen(pKey->pData)) {

93
source/libs/executor/src/filloperator.c
View File

@ -26,11 +26,11 @@
#include "ttime.h"
#include "function.h"
#include "operator.h"
#include "querynodes.h"
#include "querytask.h"
#include "tdatablock.h"
#include "tfill.h"
#include "operator.h"
#include "querytask.h"
typedef struct STimeRange {
TSKEY skey;
@ -65,9 +65,9 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp
pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;
SSDataBlock* pResBlock = pInfo->pFinalRes;
// int32_t order = TSDB_ORDER_ASC;
// int32_t order = TSDB_ORDER_ASC;
int32_t scanFlag = MAIN_SCAN;
// getTableScanInfo(pOperator, &order, &scanFlag, false);
// getTableScanInfo(pOperator, &order, &scanFlag, false);
taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo));
blockDataCleanup(pInfo->pRes);
@ -75,7 +75,8 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp
revisedFillStartKey(pInfo, pInfo->existNewGroupBlock, order);
int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey;
int64_t ts = (order == TSDB_ORDER_ASC) ? pInfo->existNewGroupBlock->info.window.ekey
: pInfo->existNewGroupBlock->info.window.skey;
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts);
taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes);
@ -92,6 +93,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp
static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
SResultInfo* pResultInfo, int32_t order) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (taosFillHasMoreResults(pInfo->pFillInfo)) {
int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows;
taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows);
@ -106,18 +108,32 @@ static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOpera
}
void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SFillOperatorInfo* pInfo = pOperator->info;
SExprSupp* pSup = &pOperator->exprSupp;
setInputDataBlock(pSup, pBlock, order, scanFlag, false);
projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, NULL);
code = setInputDataBlock(pSup, pBlock, order, scanFlag, false);
QUERY_CHECK_CODE(code, lino, _end);
code = projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, NULL);
QUERY_CHECK_CODE(code, lino, _end);
// reset the row value before applying the no-fill functions to the input data block, which is "pBlock" in this case.
pInfo->pRes->info.rows = 0;
SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp;
setInputDataBlock(pNoFillSupp, pBlock, order, scanFlag, false);
code = setInputDataBlock(pNoFillSupp, pBlock, order, scanFlag, false);
QUERY_CHECK_CODE(code, lino, _end);
projectApplyFunctions(pNoFillSupp->pExprInfo, pInfo->pRes, pBlock, pNoFillSupp->pCtx, pNoFillSupp->numOfExprs, NULL);
code = projectApplyFunctions(pNoFillSupp->pExprInfo, pInfo->pRes, pBlock, pNoFillSupp->pCtx, pNoFillSupp->numOfExprs,
NULL);
QUERY_CHECK_CODE(code, lino, _end);
pInfo->pRes->info.id.groupId = pBlock->info.id.groupId;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
}
static void fillResetPrevForNewGroup(SFillInfo* pFillInfo) {
@ -134,13 +150,13 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, i
if (order == TSDB_ORDER_ASC) {
int64_t skey = pBlock->info.window.skey;
if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the
ASSERT( taosFillNotStarted(pInfo->pFillInfo));
ASSERT(taosFillNotStarted(pInfo->pFillInfo));
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey);
} else if (pInfo->pFillInfo->start < skey) {
int64_t t = skey;
int64_t t = skey;
SInterval* pInterval = &pInfo->pFillInfo->interval;
while(1) {
while (1) {
int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
if (prev <= pInfo->pFillInfo->start) {
t = prev;
@ -155,13 +171,13 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, i
} else {
int64_t ekey = pBlock->info.window.ekey;
if (ekey > pInfo->pFillInfo->start) {
ASSERT( taosFillNotStarted(pInfo->pFillInfo));
ASSERT(taosFillNotStarted(pInfo->pFillInfo));
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey);
} else if (ekey < pInfo->pFillInfo->start) {
int64_t t = ekey;
int64_t t = ekey;
SInterval* pInterval = &pInfo->pFillInfo->interval;
int64_t prev = t;
while(1) {
while (1) {
int64_t next = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
if (next >= pInfo->pFillInfo->start) {
prev = t;
@ -180,6 +196,8 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, i
}
static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -219,11 +237,14 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
} else {
pResBlock->info.scanFlag = pBlock->info.scanFlag;
pBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId);
code = blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId);
QUERY_CHECK_CODE(code, lino, _end);
blockDataCleanup(pInfo->pRes);
blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows);
code = blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
QUERY_CHECK_CODE(code, lino, _end);
code = blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows);
QUERY_CHECK_CODE(code, lino, _end);
doApplyScalarCalculation(pOperator, pBlock, order, pBlock->info.scanFlag);
if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) {
@ -234,7 +255,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block
pInfo->totalInputRows += pInfo->pRes->info.rows;
int64_t ts = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey;
int64_t ts = (order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts);
taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes);
} else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block
@ -276,6 +297,13 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
return NULL;
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
return NULL;
}
static SSDataBlock* doFill(SOperatorInfo* pOperator) {
@ -294,7 +322,11 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
break;
}
doFilter(fillResult, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo);
int32_t code = doFilter(fillResult, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
if (fillResult->info.rows > 0) {
break;
}
@ -329,8 +361,8 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t
int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey;
// STimeWindow w = {0};
// getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC);
// STimeWindow w = {0};
// getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC);
pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo,
pInfo->primaryTsCol, order, id);
@ -409,7 +441,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
goto _error;
}
code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, &pTaskInfo->storageAPI.functionStore);
code =
initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -425,7 +458,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
SResultInfo* pResultInfo = &pOperator->resultInfo;
initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
@ -446,7 +482,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
}
pInfo->pFinalRes = createOneDataBlock(pInfo->pRes, false);
blockDataEnsureCapacity(pInfo->pFinalRes, pOperator->resultInfo.capacity);
code = blockDataEnsureCapacity(pInfo->pFinalRes, pOperator->resultInfo.capacity);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
@ -455,8 +494,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->exprSupp.numOfExprs = pInfo->numOfExpr;
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, optrDefaultBufFn, NULL,
optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
return pOperator;

201
source/libs/executor/src/groupcacheoperator.c
View File

@ -30,9 +30,9 @@
static void removeGroupCacheFile(SGroupCacheFileInfo* pFileInfo) {
if (pFileInfo->fd.fd) {
taosCloseFile(&pFileInfo->fd.fd);
(void)taosCloseFile(&pFileInfo->fd.fd);
pFileInfo->fd.fd = NULL;
taosThreadMutexDestroy(&pFileInfo->fd.mutex);
(void)taosThreadMutexDestroy(&pFileInfo->fd.mutex);
}
pFileInfo->deleted = true;
}
@ -88,7 +88,7 @@ static void logGroupCacheExecInfo(SGroupCacheOperatorInfo* pGrpCacheOperator) {
static void freeSGcSessionCtx(void* p) {
SGcSessionCtx* pSession = p;
if (pSession->semInit) {
tsem_destroy(&pSession->waitSem);
(void)tsem_destroy(&pSession->waitSem);
}
}
@ -148,6 +148,10 @@ void blockDataDeepCleanup(SSDataBlock* pDataBlock) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
if (NULL == p) {
qError("fail to get %dth col in dataBlock, numOfCols:%d", i, (int32_t)numOfCols);
continue;
}
taosMemoryFreeClear(p->pData);
if (IS_VAR_DATA_TYPE(p->info.type)) {
taosMemoryFreeClear(p->varmeta.offset);
@ -196,10 +200,14 @@ static FORCE_INLINE int32_t initOpenCacheFile(SGroupCacheFileFd* pFileFd, char*
TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE|TD_FILE_AUTO_DEL);
//TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE);
if (NULL == newFd) {
return TAOS_SYSTEM_ERROR(errno);
QRY_ERR_RET(TAOS_SYSTEM_ERROR(errno));
}
pFileFd->fd = newFd;
taosThreadMutexInit(&pFileFd->mutex, NULL);
int32_t code = taosThreadMutexInit(&pFileFd->mutex, NULL);
if (code) {
qError("taosThreadMutexInit failed, code:%x", code);
QRY_ERR_RET(code);
}
qTrace("file path %s created", filename);
@ -218,13 +226,16 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S
SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) {
sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", fileId);
(void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", fileId);
SGroupCacheFileInfo newFile = {0};
taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile));
if (taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile))) {
QRY_ERR_RET(terrno);
}
pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) {
return TSDB_CODE_OUT_OF_MEMORY;
qError("fail to get file %d from pCacheFile", fileId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
}
@ -240,14 +251,14 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S
}
}
taosThreadMutexLock(&pTmp->fd.mutex);
(void)taosThreadMutexLock(&pTmp->fd.mutex);
*ppFd = &pTmp->fd;
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void releaseFdToFileCtx(SGroupCacheFileFd* pFd) {
taosThreadMutexUnlock(&pFd->mutex);
(void)taosThreadMutexUnlock(&pFd->mutex);
}
static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamCtx* pCtx, SGcBlkBufInfo* pHead) {
@ -275,7 +286,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC
int64_t blkId = pHead->basic.blkId;
pHead = pHead->next;
taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
(void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
continue;
}
@ -295,7 +306,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC
int64_t blkId = pHead->basic.blkId;
pHead = pHead->next;
taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
(void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
continue;
}
@ -321,7 +332,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC
int64_t blkId = pHead->basic.blkId;
pHead = pHead->next;
taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
(void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
}
_return:
@ -330,7 +341,7 @@ _return:
taosHashRelease(pGrpHash, pGroup);
}
atomic_val_compare_exchange_32(&pGCache->blkCache.writeDownstreamId, pCtx->id, -1);
(void)atomic_val_compare_exchange_32(&pGCache->blkCache.writeDownstreamId, pCtx->id, -1);
return code;
}
@ -341,7 +352,8 @@ static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstr
}
pBufInfo = taosHashGet(pCache->pDirtyBlk, &pBufInfo->basic.blkId, sizeof(pBufInfo->basic.blkId));
if (NULL == pBufInfo) {
return TSDB_CODE_OUT_OF_MEMORY;
qError("fail to get blk %" PRId64 " from pCache->pDirtyBlk", pBufInfo->basic.blkId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t code = TSDB_CODE_SUCCESS;
SGcBlkBufInfo* pWriteHead = NULL;
@ -424,7 +436,7 @@ static int32_t addBlkToBufCache(struct SOperatorInfo* pOperator, SSDataBlock* pB
qError("group cache add block to cache failed, size:%" PRId64, bufSize);
return TSDB_CODE_OUT_OF_MEMORY;
}
blockDataToBuf(pBufInfo->pBuf, pBlock);
QRY_ERR_RET(blockDataToBuf(pBufInfo->pBuf, pBlock));
SGcFileCacheCtx* pFileCtx = pGCache->batchFetch ? &pCtx->fileCtx : &pGroup->pVgCtx->fileCtx;
@ -449,6 +461,10 @@ void blockDataDeepClear(SSDataBlock* pDataBlock) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
if (NULL == p) {
qError("fail to get %d col from pDataBlock, numOfCols:%d", i, (int32_t)numOfCols);
continue;
}
p->pData = NULL;
if (IS_VAR_DATA_TYPE(p->info.type)) {
p->varmeta.offset = NULL;
@ -473,8 +489,9 @@ static int32_t buildGroupCacheBaseBlock(SSDataBlock** ppDst, SSDataBlock* pSrc)
taosMemoryFree(*ppDst);
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info));
TAOS_MEMCPY(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info));
blockDataDeepClear(*ppDst);
return TSDB_CODE_SUCCESS;
}
@ -490,11 +507,17 @@ static int32_t acquireBaseBlockFromList(SGcDownstreamCtx* pCtx, SSDataBlock** pp
return TSDB_CODE_SUCCESS;
}
static void releaseBaseBlockToList(SGcDownstreamCtx* pCtx, SSDataBlock* pBlock) {
static int32_t releaseBaseBlockToList(SGcDownstreamCtx* pCtx, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
blockDataDeepCleanup(pBlock);
taosWLockLatch(&pCtx->blkLock);
taosArrayPush(pCtx->pFreeBlock, &pBlock);
if (NULL == taosArrayPush(pCtx->pFreeBlock, &pBlock)) {
code = terrno;
}
taosWUnLockLatch(&pCtx->blkLock);
return code;
}
@ -573,14 +596,15 @@ static int32_t retrieveBlkFromBufCache(SGroupCacheOperatorInfo* pGCache, SGroupC
return code;
}
taosHashPut(pCache->pReadBlk, &sessionId, sizeof(sessionId), ppRes, POINTER_BYTES);
QRY_ERR_RET(taosHashPut(pCache->pReadBlk, &sessionId, sizeof(sessionId), ppRes, POINTER_BYTES));
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void initGcVgroupCtx(SOperatorInfo* pOperator, SGcVgroupCtx* pVgCtx, int32_t downstreamId, int32_t vgId, SArray* pTbList) {
pVgCtx->pTbList = pTbList;
pVgCtx->id = vgId;
snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d",
(void)snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d",
tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, downstreamId, vgId);
pVgCtx->fileCtx.baseFilename[sizeof(pVgCtx->fileCtx.baseFilename) - 1] = 0;
@ -594,15 +618,27 @@ static int32_t addNewGroupToVgHash(SOperatorInfo* pOperator, SSHashObj* pHash, S
if (NULL == pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pList, pNew);
if (NULL == taosArrayPush(pList, pNew)) {
QRY_ERR_RET(terrno);
}
SGcVgroupCtx vgCtx = {0};
initGcVgroupCtx(pOperator, &vgCtx, pNew->pGroup->downstreamIdx, pNew->vgId, pList);
tSimpleHashPut(pHash, &pNew->vgId, sizeof(pNew->vgId), &vgCtx, sizeof(vgCtx));
QRY_ERR_RET(tSimpleHashPut(pHash, &pNew->vgId, sizeof(pNew->vgId), &vgCtx, sizeof(vgCtx)));
pNew->pGroup->pVgCtx = tSimpleHashGet(pHash, &pNew->vgId, sizeof(pNew->vgId));
if (NULL == pNew->pGroup->pVgCtx) {
qError("fail to get vg %d ctx from vgHash", pNew->vgId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
return TSDB_CODE_SUCCESS;
}
taosArrayPush(pVgCtx->pTbList, pNew);
if (NULL == taosArrayPush(pVgCtx->pTbList, pNew)) {
QRY_ERR_RET(terrno);
}
return TSDB_CODE_SUCCESS;
}
@ -620,6 +656,11 @@ static FORCE_INLINE int32_t appendNewGroupToDownstream(struct SOperatorInfo* pOp
for (int32_t i = 0; i < num; ++i) {
SGcNewGroupInfo* pNew = taosArrayGet(pCtx->pNewGrpList, i);
if (NULL == pNew) {
qError("fail to get vg %d SGcNewGroupInfo from pNewGrpList", i);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (!pGCache->batchFetch) {
code = addNewGroupToVgHash(pOperator, pCtx->pVgTbHash, pNew);
if (code) {
@ -672,7 +713,9 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p
if (code) {
return code;
}
taosArrayPush(pGCache->pDownstreams[downstreamIdx].pFreeBlock, &pGCache->pDownstreams[downstreamIdx].pBaseBlock);
if (NULL == taosArrayPush(pGCache->pDownstreams[downstreamIdx].pFreeBlock, &pGCache->pDownstreams[downstreamIdx].pBaseBlock)) {
QRY_ERR_RET(terrno);
}
}
}
@ -681,26 +724,36 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p
return code;
}
static void notifyWaitingSessions(SArray* pWaitQueue) {
static int32_t notifyWaitingSessions(SArray* pWaitQueue) {
if (NULL == pWaitQueue || taosArrayGetSize(pWaitQueue) <= 0) {
return;
return TSDB_CODE_SUCCESS;
}
int32_t n = taosArrayGetSize(pWaitQueue);
for (int32_t i = 0; i < n; ++i) {
SGcSessionCtx* pSession = taosArrayGetP(pWaitQueue, i);
tsem_post(&pSession->waitSem);
if (NULL == pSession) {
qError("fail to get %d SGcSessionCtx in pWaitQueue, total:%d", i, n);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
QRY_ERR_RET(tsem_post(&pSession->waitSem));
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void handleGroupFetchDone(SGroupCacheData* pGroup) {
static FORCE_INLINE int32_t handleGroupFetchDone(SGroupCacheData* pGroup) {
int32_t code = TSDB_CODE_SUCCESS;
pGroup->pBlock = NULL;
atomic_store_8((int8_t*)&pGroup->fetchDone, true);
taosThreadMutexLock(&pGroup->mutex);
notifyWaitingSessions(pGroup->waitQueue);
(void)taosThreadMutexLock(&pGroup->mutex);
code = notifyWaitingSessions(pGroup->waitQueue);
taosArrayClear(pGroup->waitQueue);
taosThreadMutexUnlock(&pGroup->mutex);
(void)taosThreadMutexUnlock(&pGroup->mutex);
return code;
}
static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int32_t downstreamId, int32_t vgId) {
@ -714,14 +767,15 @@ static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int
SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) {
sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%u", fileId);
(void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%u", fileId);
SGroupCacheFileInfo newFile = {0};
newFile.groupNum = 1;
taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile));
QRY_ERR_RET(taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile)));
pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) {
return TSDB_CODE_OUT_OF_MEMORY;
qError("fail to get file %d in pCacheFile", fileId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
} else {
pTmp->groupNum++;
@ -746,7 +800,7 @@ static int32_t handleVgroupTableFetchDone(SGcDownstreamCtx* pCtx, SGroupCacheDat
if (NULL == pNew || pNew->uid == uid) {
break;
}
handleGroupFetchDone(pNew->pGroup);
QRY_ERR_RET(handleGroupFetchDone(pNew->pGroup));
}
groupCacheSwitchNewFile(&pGroup->pVgCtx->fileCtx, pGroup->downstreamIdx, pGroup->vgId, true);
@ -764,15 +818,21 @@ static int32_t handleVgroupTableFetchDone(SGcDownstreamCtx* pCtx, SGroupCacheDat
}
static FORCE_INLINE void initNewGroupData(SGcDownstreamCtx* pCtx, SGroupCacheData* pGroup, int32_t downstreamIdx, int32_t vgId, bool batchFetch, bool needCache) {
taosThreadMutexInit(&pGroup->mutex, NULL);
static FORCE_INLINE int32_t initNewGroupData(SGcDownstreamCtx* pCtx, SGroupCacheData* pGroup, int32_t downstreamIdx, int32_t vgId, bool batchFetch, bool needCache) {
QRY_ERR_RET(taosThreadMutexInit(&pGroup->mutex, NULL));
pGroup->downstreamIdx = downstreamIdx;
pGroup->vgId = vgId;
pGroup->fileId = -1;
pGroup->blkList.pList = taosArrayInit(10, sizeof(SGcBlkBufBasic));
if (NULL == pGroup->blkList.pList) {
QRY_ERR_RET(terrno);
}
pGroup->startOffset = -1;
pGroup->needCache = needCache;
pGroup->pVgCtx = tSimpleHashGet(pCtx->pVgTbHash, &pGroup->vgId, sizeof(pGroup->vgId));
return TSDB_CODE_SUCCESS;
}
static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam* pParam, SGroupCacheData** ppGrp, int32_t vgId, int64_t uid) {
@ -801,7 +861,7 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
if (NULL == *ppGrp) {
return TSDB_CODE_OUT_OF_MEMORY;
}
initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache);
QRY_ERR_RET(initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache));
qDebug("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache);
@ -829,7 +889,9 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
static int32_t addBlkToGroupCache(bool batchFetch, SGroupCacheData* pGroup, SGcBlkBufInfo* pNewBlk, int64_t* pIdx) {
taosWLockLatch(&pGroup->blkList.lock);
taosArrayPush(pGroup->blkList.pList, &pNewBlk->basic);
if (NULL == taosArrayPush(pGroup->blkList.pList, &pNewBlk->basic)) {
QRY_ERR_RET(terrno);
}
*pIdx = taosArrayGetSize(pGroup->blkList.pList) - 1;
taosWUnLockLatch(&pGroup->blkList.lock);
@ -889,7 +951,7 @@ static int32_t handleGroupCacheRetrievedBlk(struct SOperatorInfo* pOperator, SSD
pGroup->pBlock = pBlock;
}
notifyWaitingSessions(pGroup->waitQueue);
QRY_ERR_RET(notifyWaitingSessions(pGroup->waitQueue));
if (pGroup == pSession->pGroupData) {
if (pGroup->needCache) {
pSession->lastBlkId = newBlkIdx;
@ -909,7 +971,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes
SHashObj* pGrpHash = pGCache->globalGrp ? pGCache->pGrpHash : pCtx->pGrpHash;
SGroupCacheData* pGroup = NULL;
while (NULL != (pGroup = taosHashIterate(pGrpHash, pGroup))) {
handleGroupFetchDone(pGroup);
QRY_ERR_RET(handleGroupFetchDone(pGroup));
}
pCtx->fetchDone = true;
} else {
@ -920,7 +982,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes
uidNum = taosArrayGetSize(pVgCtx->pTbList);
for (int32_t i = 0; i < uidNum; ++i) {
SGcNewGroupInfo* pNew = taosArrayGet(pVgCtx->pTbList, i);
handleGroupFetchDone(pNew->pGroup);
QRY_ERR_RET(handleGroupFetchDone(pNew->pGroup));
}
taosArrayClear(pVgCtx->pTbList);
}
@ -961,8 +1023,8 @@ static int32_t getCacheBlkFromDownstreamOperator(struct SOperatorInfo* pOperator
}
SGcSessionCtx* pWaitCtx = *ppWaitCtx;
pWaitCtx->newFetch = true;
taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId));
tsem_post(&pWaitCtx->waitSem);
(void)taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId));
QRY_ERR_RET(tsem_post(&pWaitCtx->waitSem));
return code;
}
@ -1028,36 +1090,47 @@ static int32_t groupCacheSessionWait(struct SOperatorInfo* pOperator, SGcDownstr
SGroupCacheOperatorInfo* pGCache = pOperator->info;
SGroupCacheData* pGroup = pSession->pGroupData;
int32_t code = TSDB_CODE_SUCCESS;
bool inLock = true;
if (NULL == pGroup->waitQueue) {
pGroup->waitQueue = taosArrayInit(1, POINTER_BYTES);
if (NULL == pGroup->waitQueue) {
taosThreadMutexUnlock(&pSession->pGroupData->mutex);
return TSDB_CODE_OUT_OF_MEMORY;
QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
taosArrayPush(pGroup->waitQueue, &pSession);
if (NULL == taosArrayPush(pGroup->waitQueue, &pSession)) {
QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
if (!pSession->semInit) {
tsem_init(&pSession->waitSem, 0, 0);
QRY_ERR_JRET(tsem_init(&pSession->waitSem, 0, 0));
pSession->semInit = true;
}
taosThreadMutexUnlock(&pSession->pGroupData->mutex);
(void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
inLock = false;
taosHashPut(pCtx->pWaitSessions, &sessionId, sizeof(sessionId), &pSession, POINTER_BYTES);
QRY_ERR_JRET(taosHashPut(pCtx->pWaitSessions, &sessionId, sizeof(sessionId), &pSession, POINTER_BYTES));
tsem_wait(&pSession->waitSem);
(void)tsem_wait(&pSession->waitSem);
if (pSession->newFetch) {
pSession->newFetch = false;
return getCacheBlkFromDownstreamOperator(pOperator, pCtx, sessionId, pSession, ppRes);
}
taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId));
(void)taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId));
bool got = false;
return getBlkFromSessionCacheImpl(pOperator, sessionId, pSession, ppRes, &got);
_return:
if (inLock) {
(void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
}
return code;
}
@ -1077,7 +1150,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s
if ((atomic_load_64(&pCtx->fetchSessionId) == sessionId)
|| (-1 == atomic_val_compare_exchange_64(&pCtx->fetchSessionId, -1, sessionId))) {
if (locked) {
taosThreadMutexUnlock(&pSession->pGroupData->mutex);
(void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
locked = false;
}
@ -1095,7 +1168,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s
break;
}
taosThreadMutexLock(&pSession->pGroupData->mutex);
(void)taosThreadMutexLock(&pSession->pGroupData->mutex);
locked = true;
};
@ -1103,7 +1176,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s
_return:
if (locked) {
taosThreadMutexUnlock(&pSession->pGroupData->mutex);
(void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
}
return code;
@ -1165,6 +1238,10 @@ static int32_t initGroupCacheSession(struct SOperatorInfo* pOperator, SOperatorP
}
*ppSession = taosHashGet(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
if (NULL == *ppSession) {
qError("fail to get session %" PRId64 " from pSessions", pGcParam->sessionId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
return TSDB_CODE_SUCCESS;
}
@ -1187,15 +1264,15 @@ static int32_t getBlkFromGroupCache(struct SOperatorInfo* pOperator, SSDataBlock
} else if (pSession->pGroupData->needCache) {
SSDataBlock** ppBlock = taosHashGet(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
if (ppBlock) {
releaseBaseBlockToList(pCtx, *ppBlock);
taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
QRY_ERR_RET(releaseBaseBlockToList(pCtx, *ppBlock));
(void)taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
}
}
code = getBlkFromSessionCache(pOperator, pGcParam->sessionId, pSession, ppRes);
if (NULL == *ppRes) {
qDebug("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows);
taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
(void)taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
} else {
pSession->resRows += (*ppRes)->info.rows;
qDebug("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows);
@ -1241,7 +1318,7 @@ static void freeRemoveGroupCacheData(void* p) {
taosArrayDestroy(pGroup->waitQueue);
taosArrayDestroy(pGroup->blkList.pList);
taosThreadMutexDestroy(&pGroup->mutex);
(void)taosThreadMutexDestroy(&pGroup->mutex);
qTrace("group removed");
}
@ -1271,7 +1348,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) {
int32_t defaultVg = 0;
SGcVgroupCtx vgCtx = {0};
initGcVgroupCtx(pOperator, &vgCtx, pCtx->id, defaultVg, NULL);
tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx));
QRY_ERR_RET(tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx)));
}
pCtx->pNewGrpList = taosArrayInit(10, sizeof(SGcNewGroupInfo));
@ -1302,7 +1379,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) {
return TSDB_CODE_OUT_OF_MEMORY;
}
snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d",
(void)snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d",
tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pCtx->id);
pCtx->fileCtx.baseFilename[sizeof(pCtx->fileCtx.baseFilename) - 1] = 0;
pCtx->fileCtx.baseNameLen = strlen(pCtx->fileCtx.baseFilename);

325
source/libs/executor/src/groupoperator.c
View File

@ -67,7 +67,7 @@ static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDa
static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity);
static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData,
int32_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup);
static SArray* extractColumnInfo(SNodeList* pNodeList);
static int32_t extractColumnInfo(SNodeList* pNodeList, SArray** pArrayRes);
static void freeGroupKey(void* param) {
SGroupKeys* pKey = (SGroupKeys*)param;
@ -111,7 +111,10 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char**
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush((*pGroupColVals), &key);
void* tmp = taosArrayPush((*pGroupColVals), &key);
if (!tmp) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols;
@ -349,12 +352,16 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
}
static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SGroupbyOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pRes = pInfo->binfo.pRes;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSDataBlock* pRes = pInfo->binfo.pRes;
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
code = doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
QUERY_CHECK_CODE(code, lino, _end);
if (!hasRemainResults(&pInfo->groupResInfo)) {
setOperatorCompleted(pOperator);
@ -367,6 +374,12 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) {
}
pOperator->resultInfo.totalRows += pRes->info.rows;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
return (pRes->info.rows == 0) ? NULL : pRes;
}
@ -412,6 +425,9 @@ void doBuildResultDatablockByHash(SOperatorInfo* pOperator, SOptrBasicInfo* pbIn
}
static SSDataBlock* buildGroupResultDataBlockByHash(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SGroupbyOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pRes = pInfo->binfo.pRes;
@ -419,7 +435,9 @@ static SSDataBlock* buildGroupResultDataBlockByHash(SOperatorInfo* pOperator) {
while (1) {
doBuildResultDatablockByHash(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
code = doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
QUERY_CHECK_CODE(code, lino, _end);
if (!hasRemainResultByHash(pOperator)) {
setOperatorCompleted(pOperator);
// clean hash after completed
@ -433,6 +451,12 @@ static SSDataBlock* buildGroupResultDataBlockByHash(SOperatorInfo* pOperator) {
}
pOperator->resultInfo.totalRows += pRes->info.rows;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
return (pRes->info.rows == 0) ? NULL : pRes;
}
@ -441,6 +465,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
return NULL;
}
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SGroupbyOperatorInfo* pInfo = pOperator->info;
@ -462,15 +488,14 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
pInfo->binfo.pRes->info.scanFlag = pBlock->info.scanFlag;
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(&pOperator->exprSupp, pBlock, order, pBlock->info.scanFlag, true);
code = setInputDataBlock(&pOperator->exprSupp, pBlock, order, pBlock->info.scanFlag, true);
QUERY_CHECK_CODE(code, lino, _end);
// there is an scalar expression that needs to be calculated right before apply the group aggregation.
if (pInfo->scalarSup.pExprInfo != NULL) {
pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
pInfo->scalarSup.numOfExprs, NULL);
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
pInfo->scalarSup.numOfExprs, NULL);
QUERY_CHECK_CODE(code, lino, _end);
}
doHashGroupbyAgg(pOperator, pBlock);
@ -491,11 +516,19 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
pGroupResInfo->dataPos = NULL;
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
return buildGroupResultDataBlockByHash(pOperator);
}
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -514,32 +547,28 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr);
}
pInfo->pGroupCols = extractColumnInfo(pAggNode->pGroupKeys);
pInfo->pGroupCols = NULL;
code = extractColumnInfo(pAggNode->pGroupKeys, &pInfo->pGroupCols);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _error);
code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
initResultRowInfo(&pInfo->binfo.resultRowInfo);
setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo);
@ -551,9 +580,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
return pOperator;
@ -567,6 +594,8 @@ _error:
}
SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBlock* pDataBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pDataBlock == NULL) {
return NULL;
}
@ -595,27 +624,33 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc
int32_t slotId = pExpr->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, slotId);
SColumnInfoData colInfo = {.hasNull = true, .info = pSrc->info};
blockDataAppendColInfo(pDstBlock, &colInfo);
code = blockDataAppendColInfo(pDstBlock, &colInfo);
QUERY_CHECK_CODE(code, lino, _end);
SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, i);
if (pDataBlock->pBlockAgg && pDataBlock->pBlockAgg[slotId].colId != -1) {
pDstBlock->pBlockAgg[i] = pDataBlock->pBlockAgg[slotId];
} else {
int32_t code = doEnsureCapacity(pDst, &pDstBlock->info, pDataBlock->info.rows, false);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
blockDataDestroy(pDstBlock);
return NULL;
}
code = doEnsureCapacity(pDst, &pDstBlock->info, pDataBlock->info.rows, false);
QUERY_CHECK_CODE(code, lino, _end);
colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info);
code = colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info);
QUERY_CHECK_CODE(code, lino, _end);
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
blockDataDestroy(pDstBlock);
return NULL;
}
return pDstBlock;
}
static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SPartitionOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -709,24 +744,37 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
}
if (pGroupInfo->blockForNotLoaded == NULL) {
pGroupInfo->blockForNotLoaded = taosArrayInit(0, sizeof(SSDataBlock*));
QUERY_CHECK_NULL(pGroupInfo->blockForNotLoaded, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
pGroupInfo->offsetForNotLoaded = 0;
}
dataNotLoadBlock->info.id.groupId = pGroupInfo->groupId;
dataNotLoadBlock->info.dataLoad = 0;
taosArrayPush(pGroupInfo->blockForNotLoaded, &dataNotLoadBlock);
void* tmp = taosArrayPush(pGroupInfo->blockForNotLoaded, &dataNotLoadBlock);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
break;
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
}
void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SDataGroupInfo* p = taosHashGet(pInfo->pGroupSet, pInfo->keyBuf, len);
void* pPage = NULL;
if (p == NULL) { // it is a new group
SDataGroupInfo gi = {0};
gi.pPageList = taosArrayInit(100, sizeof(int32_t));
taosHashPut(pInfo->pGroupSet, pInfo->keyBuf, len, &gi, sizeof(SDataGroupInfo));
QUERY_CHECK_NULL(gi.pPageList, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
code = taosHashPut(pInfo->pGroupSet, pInfo->keyBuf, len, &gi, sizeof(SDataGroupInfo));
QUERY_CHECK_CODE(code, lino, _end);
p = taosHashGet(pInfo->pGroupSet, pInfo->keyBuf, len);
@ -736,7 +784,9 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
return pPage;
}
taosArrayPush(p->pPageList, &pageId);
void* tmp = taosArrayPush(p->pPageList, &pageId);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
*(int32_t*)pPage = 0;
} else {
int32_t* curId = taosArrayGetLast(p->pPageList);
@ -759,12 +809,21 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
return NULL;
}
taosArrayPush(p->pPageList, &pageId);
void* tmp = taosArrayPush(p->pPageList, &pageId);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
memset(pPage, 0, getBufPageSize(pInfo->pBuf));
}
}
*pGroupInfo = p;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}
return pPage;
}
@ -832,6 +891,8 @@ static SSDataBlock* buildPartitionResultForNotLoadBlock(SDataGroupInfo* pGroupIn
}
static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SPartitionOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -877,8 +938,11 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
return buildPartitionResult(pOperator);
}
blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity);
blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity);
code = blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity);
QUERY_CHECK_CODE(code, lino, _end);
code = blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity);
QUERY_CHECK_CODE(code, lino, _end);
pInfo->pageIndex += 1;
releaseBufPage(pInfo->pBuf, page);
@ -891,13 +955,21 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
if (pInfo->pOrderInfoArr) {
pInfo->binfo.pRes->info.rows += pInfo->remainRows;
blockDataTrimFirstRows(pInfo->binfo.pRes, pInfo->orderedRows);
code = blockDataTrimFirstRows(pInfo->binfo.pRes, pInfo->orderedRows);
QUERY_CHECK_CODE(code, lino, _end);
pInfo->orderedRows = blockDataGetSortedRows(pInfo->binfo.pRes, pInfo->pOrderInfoArr);
pInfo->remainRows = pInfo->binfo.pRes->info.rows - pInfo->orderedRows;
pInfo->binfo.pRes->info.rows = pInfo->orderedRows;
}
blockDataUpdateTsWindow(pInfo->binfo.pRes, 0);
code = blockDataUpdateTsWindow(pInfo->binfo.pRes, 0);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
pOperator->resultInfo.totalRows += pInfo->binfo.pRes->info.rows;
return pInfo->binfo.pRes;
@ -908,8 +980,9 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) {
return NULL;
}
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SPartitionOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pRes = pInfo->binfo.pRes;
@ -948,7 +1021,8 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) {
void* pGroupIter = taosHashIterate(pInfo->pGroupSet, NULL);
while (pGroupIter != NULL) {
SDataGroupInfo* pGroupInfo = pGroupIter;
taosArrayPush(groupArray, pGroupInfo);
void* tmp = taosArrayPush(groupArray, pGroupInfo);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
pGroupIter = taosHashIterate(pInfo->pGroupSet, pGroupIter);
}
@ -960,7 +1034,15 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) {
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
pOperator->status = OP_RES_TO_RETURN;
blockDataEnsureCapacity(pRes, 4096);
code = blockDataEnsureCapacity(pRes, 4096);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
return buildPartitionResult(pOperator);
}
@ -1002,6 +1084,8 @@ static void destroyPartitionOperatorInfo(void* param) {
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SPartitionOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -1022,7 +1106,8 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
pTaskInfo->code = terrno;
goto _error;
}
taosArrayPush(pInfo->pOrderInfoArr, &order);
void* tmp = taosArrayPush(pInfo->pOrderInfoArr, &order);
QUERY_CHECK_NULL(tmp, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
}
if (pPartNode->pExprs != NULL) {
@ -1048,7 +1133,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
uint32_t defaultBufsz = 0;
pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->node.pOutputDataBlockDesc);
int32_t code = getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz);
code = getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
pTaskInfo->code = code;
@ -1101,6 +1186,7 @@ _error:
if (pInfo != NULL) {
destroyPartitionOperatorInfo(pInfo);
}
pTaskInfo->code = code;
taosMemoryFreeClear(pOperator);
return NULL;
}
@ -1156,7 +1242,8 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL);
char* pSrcData = NULL;
if (!isNull) pSrcData = colDataGetData(pSrcCol, rowIndex);
colDataSetVal(pDestCol, pDest->info.rows, pSrcData, isNull);
code = colDataSetVal(pDestCol, pDest->info.rows, pSrcData, isNull);
QUERY_CHECK_CODE(code, lino, _end);
}
pDest->info.rows++;
}
@ -1166,7 +1253,7 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
int32_t winCode = TSDB_CODE_SUCCESS;
code = pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname,
false, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode == TSDB_CODE_SUCCESS) {
memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -1177,7 +1264,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
pParInfo->rowIds = NULL;
pDest->info.dataLoad = 1;
blockDataUpdateTsWindow(pDest, pInfo->tsColIndex);
code = blockDataUpdateTsWindow(pDest, pInfo->tsColIndex);
QUERY_CHECK_CODE(code, lino, _end);
pDest->info.id.groupId = pParInfo->groupId;
pOperator->resultInfo.totalRows += pDest->info.rows;
pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte);
@ -1199,7 +1288,7 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag
void* pValue = NULL;
int32_t winCode = TSDB_CODE_SUCCESS;
code = pAPI->streamStateGetParName(pState, groupId, &pValue, true, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId);
@ -1209,7 +1298,7 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag
if (pTableSup->numOfExprs > 0) {
code = projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs,
NULL);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX);
memset(tbName, 0, TSDB_TABLE_NAME_LEN);
@ -1222,7 +1311,7 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag
len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1);
memcpy(tbName, varDataVal(pData), len);
code = pAPI->streamStatePutParName(pState, groupId, tbName);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
memcpy(pTmpBlock->info.parTbName, tbName, len);
pDestBlock->info.rows--;
@ -1234,7 +1323,7 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag
if (pTagSup->numOfExprs > 0) {
code = projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
pDestBlock->info.rows--;
} else {
memcpy(pDestBlock->info.parTbName, pTmpBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
@ -1242,7 +1331,7 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag
void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX);
code = colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
pDestBlock->info.rows++;
blockDataDestroy(pTmpBlock);
} else {
@ -1269,14 +1358,14 @@ static int32_t buildStreamCreateTableResult(SOperatorInfo* pOperator) {
}
blockDataCleanup(pInfo->pCreateTbRes);
code = blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions));
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->pTbNameIte != NULL) {
SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte;
int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0);
code = appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, pParInfo->groupId,
pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte);
}
@ -1288,6 +1377,8 @@ _end:
}
static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
pInfo->pInputDataBlock = pBlock;
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
recordNewGroupKeys(pInfo->partitionSup.pGroupCols, pInfo->partitionSup.pGroupColVals, pBlock, i);
@ -1295,15 +1386,25 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat
SPartitionDataInfo* pParData =
(SPartitionDataInfo*)taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen);
if (pParData) {
taosArrayPush(pParData->rowIds, &i);
void* tmp = taosArrayPush(pParData->rowIds, &i);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
} else {
SPartitionDataInfo newParData = {0};
newParData.groupId = calcGroupId(pInfo->partitionSup.keyBuf, keyLen);
newParData.rowIds = taosArrayInit(64, sizeof(int32_t));
taosArrayPush(newParData.rowIds, &i);
taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData, sizeof(SPartitionDataInfo));
void* tmp = taosArrayPush(newParData.rowIds, &i);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
code =
taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData, sizeof(SPartitionDataInfo));
QUERY_CHECK_CODE(code, lino, _end);
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
}
static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
@ -1318,7 +1419,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
if (hasRemainTbName(pInfo)) {
code = buildStreamCreateTableResult(pOperator);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->pCreateTbRes && pInfo->pCreateTbRes->info.rows > 0) {
return pInfo->pCreateTbRes;
}
@ -1345,7 +1446,9 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
pInfo->binfo.pRes->info.type = pBlock->info.type;
break;
case STREAM_DELETE_DATA: {
copyDataBlock(pInfo->pDelRes, pBlock);
code = copyDataBlock(pInfo->pDelRes, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
return pInfo->pDelRes;
@ -1362,8 +1465,9 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
// there is an scalar expression that needs to be calculated right before apply the group aggregation.
if (pInfo->scalarSup.pExprInfo != NULL) {
projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
pInfo->scalarSup.numOfExprs, NULL);
code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
pInfo->scalarSup.numOfExprs, NULL);
QUERY_CHECK_CODE(code, lino, _end);
}
taosHashClear(pInfo->pPartitions);
doStreamHashPartitionImpl(pInfo, pBlock);
@ -1373,7 +1477,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL);
pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, NULL);
code = buildStreamCreateTableResult(pOperator);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->pCreateTbRes && pInfo->pCreateTbRes->info.rows > 0) {
return pInfo->pCreateTbRes;
}
@ -1429,6 +1533,8 @@ int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pPar
}
SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.hasVarCol = false;
pBlock->info.id.groupId = 0;
@ -1446,14 +1552,16 @@ SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) {
}
pBlock->info.rowSize += infoData.info.bytes;
// sub table name
taosArrayPush(pBlock->pDataBlock, &infoData);
void* tmp = taosArrayPush(pBlock->pDataBlock, &infoData);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
SColumnInfoData gpIdData = {0};
gpIdData.info.type = TSDB_DATA_TYPE_UBIGINT;
gpIdData.info.bytes = 8;
pBlock->info.rowSize += gpIdData.info.bytes;
// group id
taosArrayPush(pBlock->pDataBlock, &gpIdData);
tmp = taosArrayPush(pBlock->pDataBlock, &gpIdData);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
for (int32_t i = 0; i < tag->numOfExprs; i++) {
SColumnInfoData tagCol = {0};
@ -1461,10 +1569,17 @@ SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) {
tagCol.info.bytes = tag->pExprInfo[i].base.resSchema.bytes;
tagCol.info.precision = tag->pExprInfo[i].base.resSchema.precision;
// tag info
taosArrayPush(pBlock->pDataBlock, &tagCol);
tmp = taosArrayPush(pBlock->pDataBlock, &tagCol);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
pBlock->info.rowSize += tagCol.info.bytes;
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
blockDataDestroy(pBlock);
return NULL;
}
return pBlock;
}
@ -1490,42 +1605,33 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
int32_t num = 0;
SExprInfo* pCalExprInfo = createExprInfo(pPartNode->part.pExprs, NULL, &num);
code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
}
pInfo->tbnameCalSup.numOfExprs = 0;
if (pPartNode->pSubtable != NULL) {
SExprInfo* pSubTableExpr = taosMemoryCalloc(1, sizeof(SExprInfo));
if (pSubTableExpr == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
QUERY_CHECK_NULL(pSubTableExpr, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
pInfo->tbnameCalSup.pExprInfo = pSubTableExpr;
createExprFromOneNode(pSubTableExpr, pPartNode->pSubtable, 0);
code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
}
pInfo->tagCalSup.numOfExprs = 0;
if (pPartNode->pTags != NULL) {
int32_t numOfTags;
SExprInfo* pTagExpr = createExpr(pPartNode->pTags, &numOfTags);
if (pTagExpr == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
QUERY_CHECK_NULL(pTagExpr, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
code = initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
}
if (pInfo->tbnameCalSup.numOfExprs != 0 || pInfo->tagCalSup.numOfExprs != 0) {
pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup);
QUERY_CHECK_NULL(pInfo->pCreateTbRes, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
} else {
pInfo->pCreateTbRes = NULL;
}
@ -1533,18 +1639,15 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
int32_t keyLen = 0;
code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf,
pInfo->partitionSup.pGroupCols);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
pInfo->partitionSup.needCalc = true;
pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->part.node.pOutputDataBlockDesc);
if (pInfo->binfo.pRes == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
QUERY_CHECK_NULL(pInfo->binfo.pRes, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
blockDataEnsureCapacity(pInfo->binfo.pRes, 4096);
code = blockDataEnsureCapacity(pInfo->binfo.pRes, 4096);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->parIte = NULL;
pInfo->pTbNameIte = NULL;
@ -1555,6 +1658,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
taosHashSetFreeFp(pInfo->pPartitions, freePartItem);
pInfo->tsColIndex = 0;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
QUERY_CHECK_NULL(pInfo->pDelRes, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pPartNode->part.pTargets, NULL, &numOfCols);
@ -1569,10 +1673,10 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
return pOperator;
@ -1584,12 +1688,15 @@ _error:
return NULL;
}
SArray* extractColumnInfo(SNodeList* pNodeList) {
int32_t extractColumnInfo(SNodeList* pNodeList, SArray** pArrayRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
size_t numOfCols = LIST_LENGTH(pNodeList);
SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn));
if (pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
(*pArrayRes) = NULL;
QUERY_CHECK_CODE(code, lino, _end);
}
for (int32_t i = 0; i < numOfCols; ++i) {
@ -1599,7 +1706,8 @@ SArray* extractColumnInfo(SNodeList* pNodeList) {
SColumnNode* pColNode = (SColumnNode*)pNode->pExpr;
SColumn c = extractColumnFromColumnNode(pColNode);
taosArrayPush(pList, &c);
void* tmp = taosArrayPush(pList, &c);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
} else if (nodeType(pNode->pExpr) == QUERY_NODE_VALUE) {
SValueNode* pValNode = (SValueNode*)pNode->pExpr;
SColumn c = {0};
@ -1610,9 +1718,16 @@ SArray* extractColumnInfo(SNodeList* pNodeList) {
c.scale = pValNode->node.resType.scale;
c.precision = pValNode->node.resType.precision;
taosArrayPush(pList, &c);
void* tmp = taosArrayPush(pList, &c);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
}
return pList;
(*pArrayRes) = pList;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}

4
source/libs/executor/src/hashjoin.c
View File

@ -90,7 +90,7 @@ int32_t hLeftJoinHandleSeqRowRemains(struct SOperatorInfo* pOperator, SHJoinOper
while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL);
HJ_ERR_RET(doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL));
if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));
@ -170,7 +170,7 @@ int32_t hLeftJoinHandleSeqProbeRows(struct SOperatorInfo* pOperator, SHJoinOpera
while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL);
HJ_ERR_RET(doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL));
if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));

84
source/libs/executor/src/hashjoinoperator.c
View File

@ -238,10 +238,12 @@ static int32_t hJoinInitValColsInfo(SHJoinTableCtx* pTable, SNodeList* pList) {
if (NULL == pTable->valVarCols) {
pTable->valVarCols = taosArrayInit(pTable->valNum, sizeof(int32_t));
if (NULL == pTable->valVarCols) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
taosArrayPush(pTable->valVarCols, &i);
if (NULL == taosArrayPush(pTable->valVarCols, &i)) {
return terrno;
}
}
pTable->valCols[i].bytes = pColNode->node.resType.bytes;
if (!pTable->valCols[i].keyCol && !pTable->valCols[i].vardata) {
@ -332,7 +334,7 @@ static int32_t hJoinInitTableInfo(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode*
return code;
}
memcpy(&pTable->inputStat, pStat, sizeof(*pStat));
TAOS_MEMCPY(&pTable->inputStat, pStat, sizeof(*pStat));
HJ_ERR_RET(hJoinInitPrimExprCtx(pTable->primExpr, &pTable->primCtx, pTable));
@ -416,7 +418,9 @@ static FORCE_INLINE int32_t hJoinAddPageToBufs(SArray* pRowBufs) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pRowBufs, &page);
if (NULL == taosArrayPush(pRowBufs, &page)) {
return terrno;
}
return TSDB_CODE_SUCCESS;
}
@ -464,12 +468,21 @@ static void hJoinDestroyKeyHash(SSHashObj** ppHash) {
*ppHash = NULL;
}
static FORCE_INLINE char* hJoinRetrieveColDataFromRowBufs(SArray* pRowBufs, SBufRowInfo* pRow) {
static FORCE_INLINE int32_t hJoinRetrieveColDataFromRowBufs(SArray* pRowBufs, SBufRowInfo* pRow, char** ppData) {
*ppData = NULL;
if ((uint16_t)-1 == pRow->pageId) {
return NULL;
return TSDB_CODE_SUCCESS;
}
SBufPageInfo *pPage = taosArrayGet(pRowBufs, pRow->pageId);
return pPage->data + pRow->offset;
if (NULL == pPage) {
qError("fail to get %d page, total:%d", pRow->pageId, (int32_t)taosArrayGetSize(pRowBufs));
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
*ppData = pPage->data + pRow->offset;
return TSDB_CODE_SUCCESS;
}
static int32_t hJoinCopyResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum, SBufRowInfo* pStart, SSDataBlock* pRes) {
@ -479,9 +492,11 @@ static int32_t hJoinCopyResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum
int32_t probeIdx = 0;
SBufRowInfo* pRow = pStart;
int32_t code = 0;
char* pData = NULL;
for (int32_t r = 0; r < rowNum; ++r) {
char* pData = hJoinRetrieveColDataFromRowBufs(pJoin->pRowBufs, pRow);
HJ_ERR_RET(hJoinRetrieveColDataFromRowBufs(pJoin->pRowBufs, pRow, &pData));
char* pValData = pData + pBuild->valBitMapSize;
char* pKeyData = pProbe->keyData;
buildIdx = buildValIdx = probeIdx = 0;
@ -544,7 +559,7 @@ int32_t hJoinCopyNMatchRowsToBlock(SHJoinOperatorInfo* pJoin, SSDataBlock* pRes,
SColumnInfoData* pSrc = taosArrayGet(pJoin->ctx.pProbeData->pDataBlock, pProbe->valCols[probeIdx].srcSlot);
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pProbe->valCols[probeIdx].dstSlot);
colDataAssignNRows(pDst, pRes->info.rows, pSrc, startIdx, rows);
QRY_ERR_RET(colDataAssignNRows(pDst, pRes->info.rows, pSrc, startIdx, rows));
probeIdx++;
}
@ -606,11 +621,11 @@ bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *p
}
if (pTable->keyCols[i].vardata) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData);
} else {
pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx;
memcpy(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
bufLen += pTable->keyCols[i].bytes;
}
}
@ -627,6 +642,10 @@ bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *p
static int32_t hJoinSetKeyColsData(SSDataBlock* pBlock, SHJoinTableCtx* pTable) {
for (int32_t i = 0; i < pTable->keyNum; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot);
if (NULL == pCol) {
qError("fail to get %d col, total:%d", pTable->keyCols[i].srcSlot, (int32_t)taosArrayGetSize(pBlock->pDataBlock));
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata);
return TSDB_CODE_INVALID_PARA;
@ -654,6 +673,10 @@ static int32_t hJoinSetValColsData(SSDataBlock* pBlock, SHJoinTableCtx* pTable)
continue;
}
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->valCols[i].srcSlot);
if (NULL == pCol) {
qError("fail to get %d col, total:%d", pTable->valCols[i].srcSlot, (int32_t)taosArrayGetSize(pBlock->pDataBlock));
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pTable->valCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->valCols[i].srcSlot, pCol->info.type, pTable->valCols[i].vardata);
return TSDB_CODE_INVALID_PARA;
@ -683,7 +706,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32
char *pData = NULL;
size_t bufLen = pTable->valBitMapSize;
memset(pTable->valData, 0, pTable->valBitMapSize);
TAOS_MEMSET(pTable->valData, 0, pTable->valBitMapSize);
for (int32_t i = 0, m = 0; i < pTable->valNum; ++i) {
if (pTable->valCols[i].keyCol) {
continue;
@ -693,7 +716,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32
colDataSetNull_f(pTable->valData, m);
} else {
pData = pTable->valCols[i].data + pTable->valCols[i].offset[rowIdx];
memcpy(pTable->valData + bufLen, pData, varDataTLen(pData));
TAOS_MEMCPY(pTable->valData + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData);
}
} else {
@ -701,7 +724,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32
colDataSetNull_f(pTable->valData, m);
} else {
pData = pTable->valCols[i].data + pTable->valCols[i].bytes * rowIdx;
memcpy(pTable->valData + bufLen, pData, pTable->valCols[i].bytes);
TAOS_MEMCPY(pTable->valData + bufLen, pData, pTable->valCols[i].bytes);
bufLen += pTable->valCols[i].bytes;
}
}
@ -1009,7 +1032,11 @@ static SSDataBlock* hJoinMainProcess(struct SOperatorInfo* pOperator) {
}
if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) {
doFilter(pRes, pJoin->pFinFilter, NULL);
code = doFilter(pRes, pJoin->pFinFilter, NULL);
if (code) {
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
}
if (pRes->info.rows > 0) {
return pRes;
@ -1037,7 +1064,11 @@ static SSDataBlock* hJoinMainProcess(struct SOperatorInfo* pOperator) {
}
if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) {
doFilter(pRes, pJoin->pFinFilter, NULL);
code = doFilter(pRes, pJoin->pFinFilter, NULL);
if (code) {
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
}
if (pRes->info.rows > 0) {
@ -1119,13 +1150,25 @@ static uint32_t hJoinGetFinBlkCapacity(SHJoinOperatorInfo* pJoin, SHashJoinPhysi
int32_t hJoinInitResBlocks(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* pJoinNode) {
pJoin->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
if (NULL == pJoin->finBlk) {
QRY_ERR_RET(terrno);
}
ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0);
blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode));
int32_t code = blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode));
if (TSDB_CODE_SUCCESS != code) {
QRY_ERR_RET(terrno);
}
if (NULL != pJoin->pPreFilter) {
pJoin->midBlk = createOneDataBlock(pJoin->finBlk, false);
blockDataEnsureCapacity(pJoin->midBlk, pJoin->finBlk->info.capacity);
if (NULL == pJoin->finBlk) {
QRY_ERR_RET(terrno);
}
code = blockDataEnsureCapacity(pJoin->midBlk, pJoin->finBlk->info.capacity);
if (TSDB_CODE_SUCCESS != code) {
QRY_ERR_RET(terrno);
}
}
pJoin->blkThreshold = pJoin->finBlk->info.capacity * HJOIN_BLK_THRESHOLD_RATIO;
@ -1152,8 +1195,8 @@ SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t n
setOperatorInfo(pOperator, "HashJoinOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo);
hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0]);
hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1]);
HJ_ERR_JRET(hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0]));
HJ_ERR_JRET(hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1]));
hJoinSetBuildAndProbeTable(pInfo, pJoinNode);
@ -1183,6 +1226,7 @@ SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t n
return pOperator;
_return:
if (pInfo != NULL) {
destroyHashJoinOperator(pInfo);
}

333
source/libs/executor/src/mergejoin.c
View File

@ -72,9 +72,15 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) {
if ((!pCtx->seqWinGrp) && 0 == cache->grpIdx && probeRows * buildTotalRows <= rowsLeft) {
SMJoinGrpRows* pFirstBuild = taosArrayGet(cache->grps, 0);
if (NULL == pFirstBuild) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pFirstBuild->readIdx == pFirstBuild->beginIdx) {
for (; cache->grpIdx < buildGrpNum; ++cache->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx = buildGrp->beginIdx;
}
@ -89,6 +95,9 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) {
probeGrp->endIdx = probeGrp->readIdx;
for (; cache->grpIdx < buildGrpNum && rowsLeft > 0; ++cache->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
@ -134,7 +143,8 @@ static int32_t mOuterJoinHashFullCart(SMJoinMergeCtx* pCtx) {
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (build->grpRowIdx >= 0) {
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (build->grpRowIdx < 0) {
probeGrp->readIdx++;
}
@ -171,7 +181,8 @@ static int32_t mOuterJoinHashFullCart(SMJoinMergeCtx* pCtx) {
}
build->grpRowIdx = 0;
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (!contLoop) {
if (build->grpRowIdx < 0) {
probeGrp->readIdx++;
@ -193,15 +204,27 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeRows = GRP_REMAIN_ROWS(probeGrp);
int32_t probeEndIdx = probeGrp->endIdx;
if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) {
SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0);
if (NULL == pFirstBuild) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pFirstBuild->readIdx == pFirstBuild->beginIdx) {
for (; build->grpIdx < buildGrpNum; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx = buildGrp->beginIdx;
}
@ -215,6 +238,9 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
probeGrp->endIdx = probeGrp->readIdx;
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
@ -254,6 +280,10 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeEndIdx = probeGrp->endIdx;
int32_t rowsLeft = pCtx->midBlk->info.capacity;
@ -274,6 +304,10 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (startRowIdx < 0) {
startRowIdx = buildGrp->readIdx;
}
@ -351,13 +385,17 @@ static int32_t mOuterJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, bool* contLoop)
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t startRowIdx = 0;
//blockDataCleanup(pCtx->midBlk);
do {
startRowIdx = build->grpRowIdx;
mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build);
MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL));
if (pCtx->midBlk->info.rows > 0) {
if (build->rowBitmapSize > 0) {
@ -406,6 +444,10 @@ static int32_t mOuterJoinHashSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
bool contLoop = false;
if (build->grpRowIdx >= 0) {
@ -500,7 +542,15 @@ static bool mLeftJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoi
if (buildGot) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
if (NULL == pProbeCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
buildGot = false;
@ -649,15 +699,27 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeRows = GRP_REMAIN_ROWS(probeGrp);
int32_t probeEndIdx = probeGrp->endIdx;
if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) {
SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0);
if (NULL == pFirstBuild) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pFirstBuild->readIdx == pFirstBuild->beginIdx) {
for (; build->grpIdx < buildGrpNum; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx = buildGrp->beginIdx;
}
@ -671,6 +733,9 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) {
probeGrp->endIdx = probeGrp->readIdx;
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
@ -711,9 +776,13 @@ static int32_t mInnerJoinHashCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (build->grpRowIdx >= 0) {
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (build->grpRowIdx < 0) {
probeGrp->readIdx++;
}
@ -734,7 +803,8 @@ static int32_t mInnerJoinHashCart(SMJoinMergeCtx* pCtx) {
if (NULL != pGrp) {
build->pHashCurGrp = *pGrp;
build->grpRowIdx = 0;
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (!contLoop) {
if (build->grpRowIdx < 0) {
probeGrp->readIdx++;
@ -772,7 +842,15 @@ static bool mInnerJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJo
if (buildGot) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
if (NULL == pProbeCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
buildGot = false;
@ -903,6 +981,10 @@ static int32_t mFullJoinMergeCart(SMJoinMergeCtx* pCtx) {
static FORCE_INLINE int32_t mFullJoinOutputHashRow(SMJoinMergeCtx* pCtx, SMJoinHashGrpRows* pGrpRows, int32_t idx) {
SMJoinGrpRows grp = {0};
SMJoinRowPos* pPos = taosArrayGet(pGrpRows->pRows, idx);
if (NULL == pPos) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
grp.blk = pPos->pBlk;
grp.readIdx = pPos->pos;
grp.endIdx = pPos->pos;
@ -1042,6 +1124,10 @@ static int32_t mFullJoinHandleMergeGrpRemains(SMJoinMergeCtx* pCtx) {
grpDone = false;
SMJoinGrpRows* pGrpRows = taosArrayGet(build->eqGrps, pNMatch->grpIdx);
if (NULL == pGrpRows) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pGrpRows->allRowsMatch) {
continue;
}
@ -1283,7 +1369,7 @@ static int32_t mSemiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p
do {
blockDataCleanup(pCtx->midBlk);
mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build);
MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL));
if (pCtx->midBlk->info.rows > 0) {
MJ_ERR_RET(mJoinFilterAndKeepSingleRow(pCtx->midBlk, pCtx->pJoin->pPreFilter));
@ -1312,6 +1398,9 @@ static int32_t mSemiJoinHashSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0;
int32_t probeEndIdx = probeGrp->endIdx;
@ -1343,6 +1432,10 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0;
for (; !GRP_DONE(probeGrp) && !BLK_IS_FULL(pCtx->finBlk); ++probeGrp->readIdx) {
@ -1358,7 +1451,7 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) {
build->pHashCurGrp = *(SArray**)pGrp;
ASSERT(1 == taosArrayGetSize(build->pHashCurGrp));
build->grpRowIdx = 0;
mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, NULL));
ASSERT(build->grpRowIdx < 0);
}
@ -1372,6 +1465,10 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows* buildGrp = NULL;
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeEndIdx = probeGrp->endIdx;
@ -1387,6 +1484,9 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
blockDataCleanup(pCtx->midBlk);
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
@ -1448,6 +1548,10 @@ static int32_t mSemiJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, 0);
if (NULL == buildGrp || NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t probeRows = GRP_REMAIN_ROWS(probeGrp);
int32_t probeEndIdx = probeGrp->endIdx;
@ -1584,6 +1688,10 @@ static int32_t mAntiJoinHashFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0;
int32_t probeEndIdx = probeGrp->endIdx;
@ -1616,7 +1724,7 @@ static int32_t mAntiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p
do {
blockDataCleanup(pCtx->midBlk);
mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build);
MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL));
if (pCtx->midBlk->info.rows > 0) {
MJ_ERR_RET(mJoinFilterAndNoKeepRows(pCtx->midBlk, pCtx->pJoin->pPreFilter));
@ -1642,6 +1750,10 @@ static int32_t mAntiJoinHashSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0;
int32_t probeEndIdx = probeGrp->endIdx;
@ -1682,6 +1794,10 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows* buildGrp = NULL;
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeEndIdx = probeGrp->endIdx;
@ -1697,6 +1813,10 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
blockDataCleanup(pCtx->midBlk);
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
rowsLeft -= GRP_REMAIN_ROWS(buildGrp);
@ -1889,6 +2009,9 @@ int32_t mAsofBackwardAddEqRowsToCache(struct SOperatorInfo* pOperator, SMJoinWin
grp.blk = pTable->blk;
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) {
return TSDB_CODE_SUCCESS;
@ -2102,7 +2225,9 @@ int32_t mAsofBackwardHandleGrpRemains(SMJoinWindowCtx* pCtx) {
return (pCtx->lastEqGrp) ? mAsofBackwardDumpUpdateEqRows(pCtx, pCtx->pJoin, false, true) : mAsofBackwardDumpGrpCache(pCtx);
}
static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) {
static int32_t mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) {
*newBlock = false;
bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe);
bool buildGot = false;
@ -2116,7 +2241,7 @@ static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo*
mJoinSetDone(pOperator);
}
return false;
return TSDB_CODE_SUCCESS;
}
break;
@ -2124,13 +2249,18 @@ static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo*
if (buildGot && NULL == pCtx->cache.outBlk) {
pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false);
blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit);
if (NULL == pCtx->cache.outBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit));
}
pCtx->probeGrp.blk = pJoin->probe->blk;
pCtx->buildGrp.blk = pJoin->build->blk;
return true;
*newBlock = true;
return TSDB_CODE_SUCCESS;
}
@ -2142,6 +2272,7 @@ SSDataBlock* mAsofBackwardJoinDo(struct SOperatorInfo* pOperator) {
int64_t buildTs = 0;
SColumnInfoData* pBuildCol = NULL;
SColumnInfoData* pProbeCol = NULL;
bool newBlock = false;
blockDataCleanup(pCtx->finBlk);
@ -2154,7 +2285,8 @@ SSDataBlock* mAsofBackwardJoinDo(struct SOperatorInfo* pOperator) {
}
do {
if (!mAsofBackwardRetrieve(pOperator, pJoin, pCtx)) {
MJ_ERR_JRET(mAsofBackwardRetrieve(pOperator, pJoin, pCtx, &newBlock));
if (!newBlock) {
if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) {
continue;
}
@ -2240,6 +2372,10 @@ int32_t mAsofForwardTrimCacheBlk(SMJoinWindowCtx* pCtx) {
}
SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pGrp->blk == pCtx->cache.outBlk && pCtx->pJoin->build->blkRowIdx > 0) {
MJ_ERR_RET(blockDataTrimFirstRows(pGrp->blk, pCtx->pJoin->build->blkRowIdx));
pCtx->pJoin->build->blkRowIdx = 0;
@ -2262,6 +2398,10 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCache->grps);
if (grpNum >= 1) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, grpNum - 1);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pGrp->blk != pCache->outBlk) {
int32_t beginIdx = (1 == grpNum) ? build->blkRowIdx : 0;
MJ_ERR_RET(blockDataMergeNRows(pCache->outBlk, pGrp->blk, beginIdx, pGrp->blk->info.rows - beginIdx));
@ -2271,8 +2411,12 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) {
pGrp->readIdx = 0;
//pGrp->endIdx = pGrp->blk->info.rows - 1;
} else {
taosArrayPop(pCache->grps);
(void)taosArrayPop(pCache->grps);
pGrp = taosArrayGet(pCache->grps, 0);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
ASSERT(pGrp->blk == pCache->outBlk);
//pGrp->endIdx = pGrp->blk->info.rows - pGrp->beginIdx;
}
@ -2311,16 +2455,20 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
void mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) {
int32_t mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCtx->cache.grps);
if (grpNum <= 0) {
return;
return TSDB_CODE_SUCCESS;
}
SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (1 == grpNum) {
pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows - pGrp->beginIdx, pCtx->jLimit) - 1;
return;
return TSDB_CODE_SUCCESS;
}
ASSERT(pCtx->jLimit > (pGrp->blk->info.rows - pGrp->beginIdx));
@ -2329,7 +2477,13 @@ void mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) {
int64_t remainRows = pCtx->jLimit - (pGrp->endIdx - pGrp->beginIdx + 1);
pGrp = taosArrayGet(pCtx->cache.grps, 1);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows, remainRows) - 1;
return TSDB_CODE_SUCCESS;
}
int32_t mAsofForwardFillDumpGrpCache(SMJoinWindowCtx* pCtx, bool lastBuildGrp) {
@ -2338,13 +2492,16 @@ int32_t mAsofForwardFillDumpGrpCache(SMJoinWindowCtx* pCtx, bool lastBuildGrp) {
MJ_ERR_RET(mAsofForwardChkFillGrpCache(pCtx));
}
mAsofForwardUpdateBuildGrpEndIdx(pCtx);
MJ_ERR_RET(mAsofForwardUpdateBuildGrpEndIdx(pCtx));
return mWinJoinDumpGrpCache(pCtx);
}
int32_t mAsofForwardSkipEqRows(SMJoinWindowCtx* pCtx, SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk) {
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) {
*wholeBlk = false;
@ -2479,7 +2636,9 @@ int32_t mAsofForwardSkipBuildGrp(SMJoinWindowCtx* pCtx, SMJoinOperatorInfo* pJoi
return TSDB_CODE_SUCCESS;
}
static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) {
static int32_t mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) {
*newBlock = false;
bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe);
bool buildGot = false;
@ -2496,12 +2655,16 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p
mJoinSetDone(pOperator);
}
return false;
return TSDB_CODE_SUCCESS;
}
if (buildGot) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pProbeCol || NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
MJOIN_POP_TB_BLK(&pCtx->cache);
@ -2516,7 +2679,10 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p
if (buildGot && pJoin->build->newBlk) {
if (NULL == pCtx->cache.outBlk) {
pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false);
blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit);
if (NULL == pCtx->cache.outBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit));
}
MJOIN_PUSH_BLK_TO_CACHE(&pCtx->cache, pJoin->build->blk);
@ -2524,8 +2690,9 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p
}
pCtx->probeGrp.blk = pJoin->probe->blk;
*newBlock = true;
return true;
return TSDB_CODE_SUCCESS;
}
@ -2537,6 +2704,7 @@ SSDataBlock* mAsofForwardJoinDo(struct SOperatorInfo* pOperator) {
int64_t buildTs = 0;
SColumnInfoData* pBuildCol = NULL;
SColumnInfoData* pProbeCol = NULL;
bool newBlock = false;
blockDataCleanup(pCtx->finBlk);
@ -2549,7 +2717,8 @@ SSDataBlock* mAsofForwardJoinDo(struct SOperatorInfo* pOperator) {
}
do {
if (!mAsofForwardRetrieve(pOperator, pJoin, pCtx)) {
MJ_ERR_JRET(mAsofForwardRetrieve(pOperator, pJoin, pCtx, &newBlock));
if (!newBlock) {
if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) {
continue;
}
@ -2643,7 +2812,7 @@ static FORCE_INLINE void mWinJoinPopFrontGroup(SMJoinWindowCtx* pCtx, SMJoinGrpR
if (pGrp->blk == pCtx->cache.outBlk) {
blockDataCleanup(pGrp->blk);
} else if (pGrp->clonedBlk) {
blockDataDestroy(pGrp->blk);
(void)blockDataDestroy(pGrp->blk);
}
taosArrayPopFrontBatch(pCtx->cache.grps, 1);
@ -2658,6 +2827,10 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
}
SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGetLast(pGrpArray);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (!pGrp->clonedBlk) {
if (0 == pGrp->beginIdx) {
pGrp->blk = createOneDataBlock(pGrp->blk, true);
@ -2667,6 +2840,10 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
pGrp->beginIdx = 0;
pGrp->readIdx = 0;
}
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
}
pGrp->clonedBlk = true;
}
@ -2674,14 +2851,16 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) {
static int32_t mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) {
*newBlock = false;
bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe);
bool buildGot = false;
do {
if (probeGot || MJOIN_DS_NEED_INIT(pOperator, pJoin->build)) {
if (NULL == pJoin->build->blk) {
mWinJoinCloneCacheBlk(pCtx);
MJ_ERR_RET(mWinJoinCloneCacheBlk(pCtx));
}
buildGot = mJoinRetrieveBlk(pJoin, &pJoin->build->blkRowIdx, &pJoin->build->blk, pJoin->build);
@ -2692,12 +2871,16 @@ static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin
mJoinSetDone(pOperator);
}
return false;
return TSDB_CODE_SUCCESS;
}
if (buildGot && pCtx->forwardRowsAcq) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pProbeCol || NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
buildGot = false;
@ -2709,13 +2892,18 @@ static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin
} while (true);
pCtx->probeGrp.blk = pJoin->probe->blk;
return true;
*newBlock = true;
return TSDB_CODE_SUCCESS;
}
int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) {
SSDataBlock* pBlk = build->blk;
SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pCtx->ascTs) {
if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) < pCtx->winBeginTs) {
*winEnd = false;
@ -2736,6 +2924,9 @@ int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache,
if (*((int64_t*)pCol->pData + build->blkRowIdx) <= pCtx->winEndTs) {
SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx};
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pGrp->beginIdx;
@ -2772,7 +2963,10 @@ int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache,
if (*((int64_t*)pCol->pData + build->blkRowIdx) >= pCtx->winBeginTs) {
SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx};
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pGrp->beginIdx;
@ -2841,7 +3035,15 @@ int32_t mWinJoinMoveAscWinBegin(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCache->grps);
for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) < pCtx->winBeginTs) {
mWinJoinPopFrontGroup(pCtx, pGrp);
grpNum--;
@ -2877,6 +3079,7 @@ int32_t mWinJoinMoveAscWinBegin(SMJoinWindowCtx* pCtx) {
pCache->grps = pCache->grpsQueue;
pCache->rowNum = 1;
pCache->grpsQueue = NULL;
continue;
}
@ -2893,7 +3096,15 @@ int32_t mWinJoinMoveDescWinBegin(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCache->grps);
for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) > pCtx->winEndTs) {
mWinJoinPopFrontGroup(pCtx, pGrp);
@ -2930,6 +3141,7 @@ int32_t mWinJoinMoveDescWinBegin(SMJoinWindowCtx* pCtx) {
pCache->grps = pCache->grpsQueue;
pCache->rowNum = 1;
pCache->grpsQueue = NULL;
continue;
}
@ -2963,6 +3175,10 @@ void mWinJoinRemoveOverflowGrp(SMJoinWindowCtx* pCtx) {
int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) {
SSDataBlock* pBlk = build->blk;
SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx};
if (pCtx->ascTs) {
@ -2978,6 +3194,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) <= pCtx->winEndTs) {
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pBlk->info.rows - 1;
@ -3005,6 +3224,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
}
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = build->blkRowIdx - 1;
@ -3025,6 +3247,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) >= pCtx->winBeginTs) {
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pBlk->info.rows - 1;
@ -3047,7 +3272,10 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
}
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = build->blkRowIdx - 1;
@ -3107,7 +3335,15 @@ int32_t mWinJoinMoveAscWinEnd(SMJoinWindowCtx* pCtx) {
}
SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) <= pCtx->winEndTs) {
pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1;
if (pCache->rowNum >= pCtx->jLimit) {
@ -3150,7 +3386,15 @@ int32_t mWinJoinMoveDescWinEnd(SMJoinWindowCtx* pCtx) {
}
SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) >= pCtx->winBeginTs) {
pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1;
pGrp->endIdx = pGrp->blk->info.rows - 1;
@ -3194,6 +3438,10 @@ int32_t mWinJoinTrimDumpGrpCache(SMJoinWindowCtx* pCtx) {
int32_t buildGrpNum = taosArrayGetSize(cache->grps);
for (int32_t i = 0; i < buildGrpNum && skipRows > 0; ++i) {
SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, i);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (skipRows >= GRP_REMAIN_ROWS(buildGrp)) {
skipRows -= GRP_REMAIN_ROWS(buildGrp);
mWinJoinPopFrontGroup(pCtx, buildGrp);
@ -3220,6 +3468,7 @@ SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t probeTs = 0;
SColumnInfoData* pProbeCol = NULL;
bool newBlock = false;
blockDataCleanup(pCtx->finBlk);
@ -3232,7 +3481,8 @@ SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator) {
}
do {
if (!mWinJoinRetrieve(pOperator, pJoin, pCtx)) {
MJ_ERR_JRET(mWinJoinRetrieve(pOperator, pJoin, pCtx, &newBlock));
if (!newBlock) {
if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) {
continue;
}
@ -3293,7 +3543,7 @@ int32_t mJoinInitWindowCache(SMJoinWinCache* pCache, SMJoinOperatorInfo* pJoin,
pCache->grps = taosArrayInit(2, sizeof(SMJoinGrpRows));
if (NULL == pCache->grps) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
//taosArrayReserve(pTable->eqGrps, 1);
@ -3369,7 +3619,11 @@ int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* p
}
pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode));
if (NULL == pCtx->finBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)));
pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO;
@ -3409,13 +3663,20 @@ int32_t mJoinInitMergeCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJ
}
pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
if (NULL == pCtx->finBlk) {
MJ_ERR_RET(terrno);
}
ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0);
blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode));
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)));
if (pJoin->pFPreFilter) {
pCtx->midBlk = createOneDataBlock(pCtx->finBlk, false);
blockDataEnsureCapacity(pCtx->midBlk, pCtx->finBlk->info.capacity);
if (NULL == pCtx->midBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->midBlk, pCtx->finBlk->info.capacity));
}
pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO;

321
source/libs/executor/src/mergejoinoperator.c
View File

@ -30,6 +30,9 @@
int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, SMJoinGrpRows* pGrp) {
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
pGrp->beginIdx = pTable->blkRowIdx;
pGrp->readIdx = pTable->blkRowIdx;
@ -59,12 +62,16 @@ int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBl
}
void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
int32_t mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
int32_t bmLen = BitmapLen(pBlock->info.rows);
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
if (NULL == pDst) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
// it is a reserved column for scalar function, and no data in this column yet.
if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) {
continue;
@ -86,7 +93,7 @@ void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
} else {
bool isNull = colDataIsNull_f(pDst->nullbitmap, 0);
memset(pDst->nullbitmap, 0, bmLen);
TAOS_MEMSET(pDst->nullbitmap, 0, bmLen);
if (isNull) {
colDataSetNull_f(pDst->nullbitmap, 0);
}
@ -94,11 +101,14 @@ void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
}
pBlock->info.rows = 1;
return TSDB_CODE_SUCCESS;
}
void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) {
int32_t mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) {
// int32_t totalRows = pBlock->info.rows;
int32_t code = TSDB_CODE_SUCCESS;
int32_t bmLen = BitmapLen(totalRows);
char* pBitmap = NULL;
int32_t maxRows = 0;
@ -106,6 +116,10 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
if (NULL == pDst) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
// it is a reserved column for scalar function, and no data in this column yet.
if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) {
continue;
@ -135,8 +149,12 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
len = varDataTLen(p1);
}
char* p2 = taosMemoryMalloc(len);
memcpy(p2, p1, len);
colDataSetVal(pDst, numOfRows, p2, false);
TAOS_MEMCPY(p2, p1, len);
code = colDataSetVal(pDst, numOfRows, p2, false);
if (code) {
taosMemoryFreeClear(p2);
MJ_ERR_RET(terrno);
}
taosMemoryFree(p2);
}
numOfRows += 1;
@ -150,10 +168,13 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
} else {
if (pBitmap == NULL) {
pBitmap = taosMemoryCalloc(1, bmLen);
if (NULL == pBitmap) {
MJ_ERR_RET(terrno);
}
}
memcpy(pBitmap, pDst->nullbitmap, bmLen);
memset(pDst->nullbitmap, 0, bmLen);
TAOS_MEMCPY(pBitmap, pDst->nullbitmap, bmLen);
TAOS_MEMSET(pDst->nullbitmap, 0, bmLen);
int32_t j = 0;
@ -243,6 +264,8 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
if (pBitmap != NULL) {
taosMemoryFree(pBitmap);
}
return TSDB_CODE_SUCCESS;
}
@ -290,8 +313,10 @@ int32_t mJoinFilterAndMarkHashRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo
code = TSDB_CODE_SUCCESS;
_err:
colDataDestroy(p);
taosMemoryFree(p);
return code;
}
@ -300,18 +325,19 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS;
SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock};
SColumnInfoData* p = NULL;
int32_t code = filterSetDataFromSlotId(pFilterInfo, &param1);
code = filterSetDataFromSlotId(pFilterInfo, &param1);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
goto _return;
}
int32_t status = 0;
code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
goto _return;
}
int32_t rowNum = 0;
@ -320,6 +346,9 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM
if (status == FILTER_RESULT_ALL_QUALIFIED || status == FILTER_RESULT_PARTIAL_QUALIFIED) {
for (int32_t i = startGrpIdx; i < grpNum && rowNum < pBlock->info.rows; startRowIdx = 0, ++i) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, i);
if (NULL == buildGrp) {
MJ_ERR_JRET(terrno);
}
if (buildGrp->allRowsMatch) {
rowNum += buildGrp->endIdx - startRowIdx + 1;
continue;
@ -350,9 +379,11 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM
code = TSDB_CODE_SUCCESS;
_err:
_return:
colDataDestroy(p);
taosMemoryFree(p);
return code;
}
@ -366,29 +397,31 @@ int32_t mJoinFilterAndKeepSingleRow(SSDataBlock* pBlock, SFilterInfo* pFilterInf
int32_t code = filterSetDataFromSlotId(pFilterInfo, &param1);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
goto _return;
}
int32_t status = 0;
code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
goto _return;
}
if (status == FILTER_RESULT_ALL_QUALIFIED) {
pBlock->info.rows = 1;
mJoinTrimKeepFirstRow(pBlock);
MJ_ERR_JRET(mJoinTrimKeepFirstRow(pBlock));
} else if (status == FILTER_RESULT_NONE_QUALIFIED) {
pBlock->info.rows = 0;
} else if (status == FILTER_RESULT_PARTIAL_QUALIFIED) {
mJoinTrimKeepOneRow(pBlock, pBlock->info.rows, (bool*)p->pData);
MJ_ERR_JRET(mJoinTrimKeepOneRow(pBlock, pBlock->info.rows, (bool*)p->pData));
}
code = TSDB_CODE_SUCCESS;
_err:
_return:
colDataDestroy(p);
taosMemoryFree(p);
return code;
}
@ -418,8 +451,10 @@ int32_t mJoinFilterAndNoKeepRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo)
code = TSDB_CODE_SUCCESS;
_err:
colDataDestroy(p);
taosMemoryFree(p);
return code;
}
@ -480,12 +515,20 @@ int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
SMJoinColMap* pFirstCol = probe->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(pGrp->blk->pDataBlock, pFirstCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot);
colDataAssignNRows(pOutCol, currRows, pInCol, pGrp->readIdx, firstRows);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows, pInCol, pGrp->readIdx, firstRows));
}
for (int32_t c = 0; c < build->finNum; ++c) {
SMJoinColMap* pSecondCol = build->finCols + c;
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot);
if (NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
colDataSetNItemsNull(pOutCol, currRows, firstRows);
}
@ -536,6 +579,10 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
SMJoinColMap* pFirstCol = probe->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(pFirst->blk->pDataBlock, pFirstCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
for (int32_t r = 0; r < firstRows; ++r) {
if (colDataIsNull_s(pInCol, pFirst->readIdx + r)) {
colDataSetNItemsNull(pOutCol, currRows + r * secondRows, secondRows);
@ -543,7 +590,7 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
ASSERT(pRes->info.capacity >= (pRes->info.rows + firstRows * secondRows));
uint32_t startOffset = (IS_VAR_DATA_TYPE(pOutCol->info.type)) ? pOutCol->varmeta.length : ((currRows + r * secondRows) * pOutCol->info.bytes);
ASSERT((startOffset + 1 * pOutCol->info.bytes) <= pRes->info.capacity * pOutCol->info.bytes);
colDataSetNItems(pOutCol, currRows + r * secondRows, colDataGetData(pInCol, pFirst->readIdx + r), secondRows, true);
MJ_ERR_RET(colDataSetNItems(pOutCol, currRows + r * secondRows, colDataGetData(pInCol, pFirst->readIdx + r), secondRows, true));
}
}
}
@ -552,28 +599,40 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
SMJoinColMap* pSecondCol = build->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(pSecond->blk->pDataBlock, pSecondCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
for (int32_t r = 0; r < firstRows; ++r) {
colDataAssignNRows(pOutCol, currRows + r * secondRows, pInCol, pSecond->readIdx, secondRows);
MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows + r * secondRows, pInCol, pSecond->readIdx, secondRows));
}
}
pRes->info.rows = append ? (pRes->info.rows + firstRows * secondRows) : firstRows * secondRows;
return TSDB_CODE_SUCCESS;
}
bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build) {
int32_t mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build, bool* cont) {
if (NULL != cont) {
*cont = false;
}
int32_t rowsLeft = append ? (pBlk->info.capacity - pBlk->info.rows) : pBlk->info.capacity;
if (rowsLeft <= 0) {
return false;
return TSDB_CODE_SUCCESS;
}
int32_t buildGrpRows = taosArrayGetSize(build->pHashCurGrp);
int32_t grpRows = buildGrpRows - build->grpRowIdx;
if (grpRows <= 0 || build->grpRowIdx < 0) {
build->grpRowIdx = -1;
return true;
if (NULL != cont) {
*cont = true;
}
return TSDB_CODE_SUCCESS;
}
int32_t actRows = TMIN(grpRows, rowsLeft);
@ -583,10 +642,14 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S
SMJoinColMap* pFirstCol = probe->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(probeGrp->blk->pDataBlock, pFirstCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pFirstCol->dstSlot);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (colDataIsNull_s(pInCol, probeGrp->readIdx)) {
colDataSetNItemsNull(pOutCol, currRows, actRows);
} else {
colDataSetNItems(pOutCol, currRows, colDataGetData(pInCol, probeGrp->readIdx), actRows, true);
MJ_ERR_RET(colDataSetNItems(pOutCol, currRows, colDataGetData(pInCol, probeGrp->readIdx), actRows, true));
}
}
@ -595,8 +658,16 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S
SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pSecondCol->dstSlot);
for (int32_t r = 0; r < actRows; ++r) {
SMJoinRowPos* pRow = taosArrayGet(build->pHashCurGrp, build->grpRowIdx + r);
if (NULL == pRow) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pInCol = taosArrayGet(pRow->pBlk->pDataBlock, pSecondCol->srcSlot);
colDataAssignNRows(pOutCol, currRows + r, pInCol, pRow->pos, 1);
if (NULL == pInCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows + r, pInCol, pRow->pos, 1));
}
}
@ -609,16 +680,24 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S
}
if (actRows == rowsLeft) {
return false;
return TSDB_CODE_SUCCESS;
}
return true;
if (NULL != cont) {
*cont = true;
}
return TSDB_CODE_SUCCESS;
}
int32_t mJoinAllocGrpRowBitmap(SMJoinTableCtx* pTb) {
int32_t grpNum = taosArrayGetSize(pTb->eqGrps);
for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGet(pTb->eqGrps, i);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinGetRowBitmapOffset(pTb, pGrp->endIdx - pGrp->beginIdx + 1, &pGrp->rowBitmapOffset));
pGrp->rowMatchNum = 0;
}
@ -632,9 +711,9 @@ int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastB
pCtx->lastEqGrp = true;
mJoinBuildEqGroups(pJoin->probe, timestamp, NULL, true);
MJ_ERR_RET(mJoinBuildEqGroups(pJoin->probe, timestamp, NULL, true));
if (!lastBuildGrp) {
mJoinRetrieveEqGrpRows(pJoin, pJoin->build, timestamp);
MJ_ERR_RET(mJoinRetrieveEqGrpRows(pJoin, pJoin->build, timestamp));
} else {
pJoin->build->grpIdx = 0;
}
@ -661,7 +740,7 @@ int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastB
pCtx->hashJoin = false;
if (!lastBuildGrp && pJoin->build->rowBitmapSize > 0) {
mJoinAllocGrpRowBitmap(pJoin->build);
MJ_ERR_RET(mJoinAllocGrpRowBitmap(pJoin->build));
}
return (*pCtx->mergeCartFp)(pCtx);
@ -721,7 +800,7 @@ int32_t mJoinInitDownstreamInfo(SMJoinOperatorInfo* pInfo, SOperatorInfo*** pDow
*newDownstreams = true;
*pDownstream = mJoinBuildDownstreams(pInfo, *pDownstream);
if (NULL == *pDownstream) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
*numOfDownstream = 2;
}
@ -732,7 +811,7 @@ int32_t mJoinInitDownstreamInfo(SMJoinOperatorInfo* pInfo, SOperatorInfo*** pDow
static int32_t mJoinInitPrimKeyInfo(SMJoinTableCtx* pTable, int32_t slotId) {
pTable->primCol = taosMemoryMalloc(sizeof(SMJoinColInfo));
if (NULL == pTable->primCol) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pTable->primCol->srcSlot = slotId;
@ -745,7 +824,7 @@ static int32_t mJoinInitColsInfo(int32_t* colNum, int64_t* rowSize, SMJoinColInf
*pCols = taosMemoryMalloc((*colNum) * sizeof(SMJoinColInfo));
if (NULL == *pCols) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
*rowSize = 0;
@ -779,7 +858,7 @@ static int32_t mJoinInitKeyColsInfo(SMJoinTableCtx* pTable, SNodeList* pList, bo
pTable->keyBuf = taosMemoryMalloc(TMAX(rowSize, pTable->keyNullSize));
if (NULL == pTable->keyBuf) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
@ -790,7 +869,7 @@ static int32_t mJoinInitKeyColsInfo(SMJoinTableCtx* pTable, SNodeList* pList, bo
static int32_t mJoinInitFinColsInfo(SMJoinTableCtx* pTable, SNodeList* pList) {
pTable->finCols = taosMemoryMalloc(LIST_LENGTH(pList) * sizeof(SMJoinColMap));
if (NULL == pTable->finCols) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
int32_t i = 0;
@ -837,8 +916,20 @@ static int32_t mJoinInitPrimExprCtx(SNode* pNode, SMJoinPrimExprCtx* pCtx, SMJoi
}
SValueNode* pUnit = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
SValueNode* pCurrTz = (5 == pFunc->pParameterList->length) ? (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2) : NULL;
if (NULL == pUnit) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
SValueNode* pCurrTz = NULL;
if (5 == pFunc->pParameterList->length){
pCurrTz = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2);
if (NULL == pCurrTz) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
}
SValueNode* pTimeZone = (5 == pFunc->pParameterList->length) ? (SValueNode*)nodesListGetNode(pFunc->pParameterList, 4) : (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3);
if (NULL == pTimeZone) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
pCtx->truncateUnit = pUnit->typeData;
if ((NULL == pCurrTz || 1 == pCurrTz->typeData) && pCtx->truncateUnit >= (86400 * TSDB_TICK_PER_SECOND(pFunc->node.resType.precision))) {
@ -859,24 +950,32 @@ static int32_t mJoinInitTableInfo(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysi
MJ_ERR_RET(mJoinInitKeyColsInfo(pTable, (0 == idx) ? pJoinNode->pEqLeft : pJoinNode->pEqRight, JOIN_TYPE_FULL == pJoin->joinType));
MJ_ERR_RET(mJoinInitFinColsInfo(pTable, pJoinNode->pTargets));
memcpy(&pTable->inputStat, pStat, sizeof(*pStat));
TAOS_MEMCPY(&pTable->inputStat, pStat, sizeof(*pStat));
pTable->eqGrps = taosArrayInit(8, sizeof(SMJoinGrpRows));
//taosArrayReserve(pTable->eqGrps, 1);
if (NULL == pTable->eqGrps) {
return terrno;
}
if (E_JOIN_TB_BUILD == pTable->type) {
pTable->createdBlks = taosArrayInit(8, POINTER_BYTES);
if (NULL == pTable->createdBlks) {
return terrno;
}
pTable->pGrpArrays = taosArrayInit(32, POINTER_BYTES);
if (NULL == pTable->pGrpArrays) {
return terrno;
}
pTable->pGrpHash = tSimpleHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY));
if (NULL == pTable->createdBlks || NULL == pTable->pGrpArrays || NULL == pTable->pGrpHash) {
return TSDB_CODE_OUT_OF_MEMORY;
if (NULL == pTable->pGrpHash) {
return terrno;
}
if (pJoin->pFPreFilter && IS_FULL_OUTER_JOIN(pJoin->joinType, pJoin->subType)) {
pTable->rowBitmapSize = MJOIN_ROW_BITMAP_SIZE;
pTable->pRowBitmap = taosMemoryMalloc(pTable->rowBitmapSize);
if (NULL == pTable->pRowBitmap) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
@ -945,7 +1044,15 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
SMJoinPrimExprCtx* pCtx = &pTable->primCtx;
SColumnInfoData* pPrimIn = taosArrayGet(pBlock->pDataBlock, pTable->primCol->srcSlot);
if (NULL == pPrimIn) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pPrimOut) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
if (0 != pCtx->timezoneUnit) {
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] + pCtx->timezoneUnit) % pCtx->truncateUnit;
@ -961,6 +1068,7 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable) {
SSDataBlock* pTmp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
int32_t dsIdx = pTable->downStreamIdx;
if (E_JOIN_TB_PROBE == pTable->type) {
if (pTable->remainInBlk) {
@ -1028,7 +1136,12 @@ SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTa
_return:
mJoinLaunchPrimExpr(pTmp, pTable);
code = mJoinLaunchPrimExpr(pTmp, pTable);
if (code) {
pJoin->errCode = code;
T_LONG_JMP(pJoin->pOperator->pTaskInfo->env, pJoin->errCode);
}
return pTmp;
}
@ -1041,7 +1154,11 @@ static FORCE_INLINE SSDataBlock* mJoinRetrieveImpl(SMJoinOperatorInfo* pJoin, SM
if (NULL == pTmp) {
pTable->dsFetchDone = true;
} else {
mJoinLaunchPrimExpr(pTmp, pTable);
int32_t code = mJoinLaunchPrimExpr(pTmp, pTable);
if (code) {
pJoin->errCode = code;
T_LONG_JMP(pJoin->pOperator->pTaskInfo->env, pJoin->errCode);
}
}
return pTmp;
@ -1107,7 +1224,7 @@ bool mJoinRetrieveBlk(SMJoinOperatorInfo* pJoin, int32_t* pIdx, SSDataBlock** pp
static void mJoinDestroyCreatedBlks(SArray* pCreatedBlks) {
int32_t blkNum = taosArrayGetSize(pCreatedBlks);
for (int32_t i = 0; i < blkNum; ++i) {
blockDataDestroy(*(SSDataBlock**)TARRAY_GET_ELEM(pCreatedBlks, i));
(void)blockDataDestroy(*(SSDataBlock**)TARRAY_GET_ELEM(pCreatedBlks, i));
}
taosArrayClear(pCreatedBlks);
}
@ -1119,12 +1236,12 @@ int32_t mJoinGetRowBitmapOffset(SMJoinTableCtx* pTable, int32_t rowNum, int32_t
int64_t newSize = reqSize * 1.1;
pTable->pRowBitmap = taosMemoryRealloc(pTable->pRowBitmap, newSize);
if (NULL == pTable->pRowBitmap) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pTable->rowBitmapSize = newSize;
}
memset(pTable->pRowBitmap + pTable->rowBitmapOffset, 0xFFFFFFFF, bitmapLen);
TAOS_MEMSET(pTable->pRowBitmap + pTable->rowBitmapOffset, 0xFFFFFFFF, bitmapLen);
*rowBitmapOffset = pTable->rowBitmapOffset;
pTable->rowBitmapOffset += bitmapLen;
@ -1140,13 +1257,18 @@ void mJoinResetForBuildTable(SMJoinTableCtx* pTable) {
taosArrayClear(pTable->eqGrps);
if (pTable->rowBitmapSize > 0) {
pTable->rowBitmapOffset = 1;
memset(&pTable->nMatchCtx, 0, sizeof(pTable->nMatchCtx));
TAOS_MEMSET(&pTable->nMatchCtx, 0, sizeof(pTable->nMatchCtx));
}
}
int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, bool restart) {
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows* pGrp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) {
return TSDB_CODE_SUCCESS;
@ -1158,7 +1280,10 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
bool keepGrp = true;
pGrp = taosArrayReserve(pTable->eqGrps, 1);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->beginIdx = pTable->blkRowIdx++;
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pGrp->beginIdx;
@ -1207,7 +1332,12 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
if (0 == pGrp->beginIdx && pTable->multiEqGrpRows && 0 == pTable->eqRowLimit) {
pGrp->blk = createOneDataBlock(pTable->blk, true);
taosArrayPush(pTable->createdBlks, &pGrp->blk);
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
}
if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) {
MJ_ERR_RET(terrno);
}
} else {
if (!pTable->multiEqGrpRows) {
pGrp->endIdx = pGrp->beginIdx;
@ -1230,10 +1360,16 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
pTable->eqRowNum += rowNum;
pGrp->blk = blockDataExtractBlock(pTable->blk, pGrp->beginIdx, rowNum);
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
}
pGrp->endIdx -= pGrp->beginIdx;
pGrp->beginIdx = 0;
pGrp->readIdx = 0;
taosArrayPush(pTable->createdBlks, &pGrp->blk);
if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) {
MJ_ERR_RET(terrno);
}
}
}
@ -1242,19 +1378,19 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
_return:
if (pTable->noKeepEqGrpRows || !keepGrp || (!pTable->multiEqGrpRows && !restart)) {
taosArrayPop(pTable->eqGrps);
(void)taosArrayPop(pTable->eqGrps);
} else {
pTable->grpTotalRows += pGrp->endIdx - pGrp->beginIdx + 1;
}
return TSDB_CODE_SUCCESS;
return code;
}
int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable, int64_t timestamp) {
bool wholeBlk = false;
mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, true);
MJ_ERR_RET(mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, true));
while (wholeBlk && !pTable->dsFetchDone) {
pTable->blk = (*pJoin->retrieveFp)(pJoin, pTable);
@ -1267,7 +1403,7 @@ int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable
}
wholeBlk = false;
mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, false);
MJ_ERR_RET(mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, false));
}
return TSDB_CODE_SUCCESS;
@ -1276,6 +1412,10 @@ int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable
int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
for (int32_t i = 0; i < pTable->keyNum; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata);
return TSDB_CODE_INVALID_PARA;
@ -1320,15 +1460,15 @@ bool mJoinCopyKeyColsDataToBuf(SMJoinTableCtx* pTable, int32_t rowIdx, size_t *p
}
if (pTable->keyCols[0].jsonData) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, getJsonValueLen(pData));
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, getJsonValueLen(pData));
bufLen += getJsonValueLen(pData);
} else if (pTable->keyCols[i].vardata) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData);
} else {
pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx;
memcpy(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
bufLen += pTable->keyCols[i].bytes;
}
}
@ -1346,15 +1486,20 @@ static int32_t mJoinGetAvailableGrpArray(SMJoinTableCtx* pTable, SArray** ppRes)
do {
if (pTable->grpArrayIdx < taosArrayGetSize(pTable->pGrpArrays)) {
*ppRes = taosArrayGetP(pTable->pGrpArrays, pTable->grpArrayIdx++);
if (NULL == *ppRes) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
taosArrayClear(*ppRes);
return TSDB_CODE_SUCCESS;
}
SArray* pNew = taosArrayInit(4, sizeof(SMJoinRowPos));
if (NULL == pNew) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (NULL == taosArrayPush(pTable->pGrpArrays, &pNew)) {
return terrno;
}
taosArrayPush(pTable->pGrpArrays, &pNew);
} while (true);
return TSDB_CODE_SUCCESS;
@ -1368,10 +1513,14 @@ static int32_t mJoinAddRowToHash(SMJoinOperatorInfo* pJoin, size_t keyLen, SSDat
SArray* pNewGrp = NULL;
MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp));
taosArrayPush(pNewGrp, &pos);
tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, POINTER_BYTES);
if (NULL == taosArrayPush(pNewGrp, &pos)) {
return terrno;
}
MJ_ERR_RET(tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, POINTER_BYTES));
} else if (pBuild->multiRowsGrp) {
taosArrayPush(*pGrpRows, &pos);
if (NULL == taosArrayPush(*pGrpRows, &pos)) {
return terrno;
}
}
return TSDB_CODE_SUCCESS;
@ -1386,10 +1535,14 @@ static int32_t mJoinAddRowToFullHash(SMJoinOperatorInfo* pJoin, size_t keyLen, S
SMJoinHashGrpRows pNewGrp = {0};
MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp.pRows));
taosArrayPush(pNewGrp.pRows, &pos);
tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, sizeof(pNewGrp));
if (NULL == taosArrayPush(pNewGrp.pRows, &pos)) {
return terrno;
}
MJ_ERR_RET(tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, sizeof(pNewGrp)));
} else {
taosArrayPush(pGrpRows->pRows, &pos);
if (NULL == taosArrayPush(pGrpRows->pRows, &pos)) {
return terrno;
}
}
return TSDB_CODE_SUCCESS;
@ -1406,6 +1559,9 @@ int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pT
int32_t grpNum = taosArrayGetSize(pTable->eqGrps);
for (int32_t g = 0; g < grpNum; ++g) {
SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g);
if (NULL == pGrp) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable));
int32_t grpRows = GRP_REMAIN_ROWS(pGrp);
@ -1434,6 +1590,10 @@ int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable
int32_t grpNum = taosArrayGetSize(pTable->eqGrps);
for (int32_t g = 0; g < grpNum; ++g) {
SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g);
if (NULL == pGrp) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable));
int32_t grpRows = GRP_REMAIN_ROWS(pGrp);
@ -1490,8 +1650,11 @@ void mWinJoinResetWindowCache(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache) {
for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i);
if (NULL == pGrp) {
continue;
}
if (pGrp->blk != pCtx->cache.outBlk && pGrp->clonedBlk) {
blockDataDestroy(pGrp->blk);
(void)blockDataDestroy(pGrp->blk);
}
}
@ -1533,6 +1696,7 @@ void mJoinResetOperator(struct SOperatorInfo* pOperator) {
SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) {
SMJoinOperatorInfo* pJoin = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator->status == OP_EXEC_DONE) {
if (NULL == pOperator->pDownstreamGetParams || NULL == pOperator->pDownstreamGetParams[0] || NULL == pOperator->pDownstreamGetParams[1]) {
qDebug("%s merge join done", GET_TASKID(pOperator->pTaskInfo));
@ -1553,7 +1717,6 @@ SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) {
pBlock = (*pJoin->joinFp)(pOperator);
if (NULL == pBlock) {
if (pJoin->errCode) {
ASSERT(0);
T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode);
}
break;
@ -1561,7 +1724,11 @@ SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) {
pBlock->info.id.blockId = pJoin->outBlkId;
if (pJoin->pFinFilter != NULL) {
doFilter(pBlock, pJoin->pFinFilter, NULL);
code = doFilter(pBlock, pJoin->pFinFilter, NULL);
if (code) {
pJoin->errCode = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode);
}
}
if (pBlock->info.rows > 0 || pOperator->status == OP_EXEC_DONE) {
@ -1697,13 +1864,17 @@ int32_t mJoinSetImplFp(SMJoinOperatorInfo* pJoin) {
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) {
SMJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMJoinOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
bool newDownstreams = false;
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator == NULL || pInfo == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
SMJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMJoinOperatorInfo));
if (pInfo == NULL) {
code = terrno;
goto _return;
}
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pOperator == NULL) {
code = terrno;
goto _return;
}
@ -1716,8 +1887,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
MJ_ERR_JRET(mJoinHandleConds(pInfo, pJoinNode));
mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0], newDownstreams);
mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1], newDownstreams);
MJ_ERR_JRET(mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0], newDownstreams));
MJ_ERR_JRET(mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1], newDownstreams));
MJ_ERR_JRET(mJoinInitCtx(pInfo, pJoinNode));
MJ_ERR_JRET(mJoinSetImplFp(pInfo));
@ -1736,6 +1907,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
return pOperator;
_return:
if (pInfo != NULL) {
destroyMergeJoinOperator(pInfo);
}
@ -1745,6 +1917,7 @@ _return:
taosMemoryFree(pOperator);
pTaskInfo->code = code;
return NULL;
}

1866
source/libs/executor/src/scanoperator.c
View File

File diff suppressed because it is too large Load Diff

100
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -84,7 +84,7 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) {
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
winCode = TSDB_CODE_FAILED;
} else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) {
@ -95,7 +95,7 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
if (winCode == TSDB_CODE_FAILED) {
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
} else {
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
@ -105,7 +105,7 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
if (winCode == TSDB_CODE_FAILED) {
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
}
if (ts < pCurWin->winInfo.sessionWin.win.ekey) {
@ -115,7 +115,7 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
if (winCode == TSDB_CODE_SUCCESS) {
@ -173,7 +173,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI
needDelState = true;
if (pStDeleted && pWinInfo->winInfo.isOutput) {
code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
pWinInfo->winInfo.sessionWin.win.skey = pTs[start];
@ -268,11 +268,11 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
if (!pStartTsCol) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
SStreamStateCur* pCur = NULL;
COUNT_TYPE slidingRows = 0;
@ -287,7 +287,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SCountWindowInfo curWin = {0};
buffInfo.rebuildWindow = false;
code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) {
buffInfo.winBuffOp = MOVE_NEXT_WINDOW;
@ -296,9 +296,9 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
setSessionWinOutputInfo(pStUpdated, &curWin.winInfo);
slidingRows = *curWin.pWindowCount;
if (!buffInfo.rebuildWindow) {
code = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStUpdated,
pStDeleted, &buffInfo.rebuildWindow, &winRows);
TSDB_CHECK_CODE(code, lino, _end);
code = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStUpdated, pStDeleted,
&buffInfo.rebuildWindow, &winRows);
QUERY_CHECK_CODE(code, lino, _end);
}
if (buffInfo.rebuildWindow) {
SSessionKey range = {0};
@ -310,24 +310,26 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
range.win.skey = TMIN(startTsCols[i], range.win.skey);
range.win.ekey = TMAX(startTsCols[rows - 1], range.win.ekey);
uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL);
code =
appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL);
QUERY_CHECK_CODE(code, lino, _end);
break;
}
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, 0);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_COUNT_OP(pOperator)) {
code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
code = saveResult(curWin.winInfo, pStUpdated);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
curWin.winInfo.pStatePos->beUpdated = true;
@ -335,7 +337,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
getSessionHashKey(&curWin.winInfo.sessionWin, &key);
code =
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
if (isSlidingCountWindow(pAggSup)) {
@ -425,7 +427,7 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
if (!pInfo) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
// 4.checksum
@ -434,7 +436,7 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
void* pCksum = POINTER_SHIFT(buf, dataLen);
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
}
@ -447,12 +449,12 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
buf = decodeSSessionKey(buf, &key);
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo,
sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
// 2.twAggSup
@ -478,7 +480,7 @@ void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) {
pBuf = taosMemoryCalloc(1, len);
if (!pBuf) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
@ -556,7 +558,7 @@ int32_t doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock,
doDeleteSessionWindow(pAggSup, &curWin);
if (result) {
code = saveDeleteInfo(result, curWin);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
}
}
@ -575,22 +577,22 @@ int32_t deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, S
SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
if (!pWins) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
if (isSlidingCountWindow(pAggSup)) {
code = doDeleteCountWindows(pAggSup, pBlock, pWins);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
} else {
code = doDeleteTimeWindows(pAggSup, pBlock, pWins);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
removeSessionResults(pAggSup, pMapUpdate, pWins);
code = copyDeleteWindowInfo(pWins, pMapDelete);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (needAdd) {
code = copyDeleteWindowInfo(pWins, pPkDelete);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
taosArrayDestroy(pWins);
@ -653,7 +655,7 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
bool add = pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator);
code = deleteCountWinState(&pInfo->streamAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, pInfo->pPkDeleted,
add);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} else if (pBlock->info.type == STREAM_CLEAR) {
doResetCountWindows(&pInfo->streamAggSup, pBlock);
@ -661,7 +663,7 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
} else if (pBlock->info.type == STREAM_GET_ALL) {
pInfo->recvGetAll = true;
code = getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
return pBlock;
@ -669,7 +671,7 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
pAggSup->stateStore.streamStateCommit(pAggSup->pState);
doStreamCountSaveCheckpoint(pOperator);
code = copyDataBlock(pInfo->pCheckpointRes, pBlock);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} else {
ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
@ -678,10 +680,11 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
if (pInfo->scalarSupp.pExprInfo != NULL) {
SExprSupp* pExprSup = &pInfo->scalarSupp;
code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
QUERY_CHECK_CODE(code, lino, _end);
doStreamCountAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);
@ -690,20 +693,20 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
pOperator->status = OP_RES_TO_RETURN;
code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);
initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
pInfo->pUpdated = NULL;
code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator)) {
code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
SSDataBlock* opRes = buildCountResult(pOperator);
@ -727,7 +730,7 @@ void streamCountReleaseState(SOperatorInfo* pOperator) {
char* pBuff = taosMemoryCalloc(1, resSize);
if (pBuff) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
memcpy(pBuff, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
qDebug("===stream=== count window operator relase state. ");
@ -755,7 +758,7 @@ void streamCountReloadState(SOperatorInfo* pOperator) {
code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME,
strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
TSKEY ts = *(TSKEY*)pBuf;
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
@ -783,7 +786,7 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
}
pOperator->pTaskInfo = pTaskInfo;
@ -793,14 +796,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
}
SExprSupp* pExpSup = &pOperator->exprSupp;
SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->twAggSup = (STimeWindowAggSupp){
.waterMark = pCountNode->window.watermark,
@ -814,12 +817,13 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState,
sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->streamAggSup.windowCount = pCountNode->windowCount;
pInfo->streamAggSup.windowSliding = pCountNode->windowSliding;
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->binfo.pRes = pResBlock;
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
@ -834,7 +838,7 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
}
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
@ -853,7 +857,7 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), &buff, &len);
if (res == TSDB_CODE_SUCCESS) {
code = doStreamCountDecodeOpState(buff, len, pOperator, true);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
taosMemoryFree(buff);
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo,
@ -863,10 +867,10 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
if (downstream) {
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
}
return pOperator;

118
source/libs/executor/src/streameventwindowoperator.c
View File

@ -154,7 +154,7 @@ int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t gro
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
code = pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
pCurWin->pWinFlag->startFlag = start;
@ -213,7 +213,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
if (pWin->skey > pTsData[i]) {
if (pStDeleted && pWinInfo->winInfo.isOutput) {
code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->winInfo.sessionWin);
pWin->skey = pTsData[i];
@ -278,7 +278,7 @@ static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pC
setEventWindowFlag(pAggSup, &nextWinInfo);
code = compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo,
pStUpdated, pStDeleted, false);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag;
}
@ -321,7 +321,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
if (!pColDataInfo) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
tsCols = (int64_t*)pColDataInfo->pData;
} else {
@ -337,7 +337,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
}
int32_t statusStart = 0;
code = filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock};
@ -349,11 +349,11 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t statusEnd = 0;
code = filterExecute(pInfo->pEndCondInfo, pSDataBlock, &pColEnd, NULL, paramEnd.numOfCols, &statusEnd);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
int32_t rows = pSDataBlock->info.rows;
code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
for (int32_t i = 0; i < rows; i += winRows) {
if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,
@ -367,48 +367,50 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SSessionKey nextWinKey = {0};
code = setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin,
&nextWinKey);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
bool rebuild = false;
code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild, &winRows);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
ASSERT(winRows >= 1);
if (rebuild) {
uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
code = appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
QUERY_CHECK_CODE(code, lino, _end);
code = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin);
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) &&
!isWindowIncomplete(&curWin)) {
code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
SSessionKey tmpSeInfo = {0};
getSessionHashKey(&curWin.winInfo.sessionWin, &tmpSeInfo);
code = tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
continue;
}
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, 0);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
code = compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->isHistoryOp) {
code = saveResult(curWin.winInfo, pInfo->pAllUpdated);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
if (isWindowIncomplete(&curWin)) {
@ -418,12 +420,12 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator)) {
code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
code = saveResult(curWin.winInfo, pSeUpdated);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
@ -432,7 +434,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
getSessionHashKey(&curWin.winInfo.sessionWin, &key);
code =
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
}
@ -490,7 +492,7 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
if (!pInfo) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
@ -500,7 +502,7 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
qError("stream event state is invalid");
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
// 1.streamAggSup.pResultRows
@ -513,13 +515,13 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
int32_t winCode = TSDB_CODE_SUCCESS;
code = pAggSup->stateStore.streamStateSessionAddIfNotExist(
pAggSup->pState, &winfo.sessionWin, pAggSup->gap, (void**)&winfo.pStatePos, &pAggSup->resultRowSize, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
ASSERT(winCode == TSDB_CODE_SUCCESS);
buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
code =
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
// 2.twAggSup
@ -622,12 +624,12 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
bool add = pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator);
code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted,
pInfo->pPkDeleted, add);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} else if (pBlock->info.type == STREAM_GET_ALL) {
pInfo->recvGetAll = true;
code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
return pBlock;
@ -636,7 +638,7 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
doStreamEventSaveCheckpoint(pOperator);
pInfo->reCkBlock = true;
code = copyDataBlock(pInfo->pCheckpointRes, pBlock);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} else {
ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
@ -645,10 +647,11 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
if (pInfo->scalarSupp.pExprInfo != NULL) {
SExprSupp* pExprSup = &pInfo->scalarSupp;
code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
QUERY_CHECK_CODE(code, lino, _end);
doStreamEventAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
}
@ -656,10 +659,10 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
pOperator->status = OP_RES_TO_RETURN;
code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated);
@ -667,26 +670,26 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
SArray* pHisWins = taosArrayInit(16, sizeof(SEventWindowInfo));
if (!pHisWins) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
code = copyUpdateResult(&pInfo->pAllUpdated, pHisWins, sessionKeyCompareAsc);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
code = getMaxTsWins(pHisWins, pInfo->historyWins);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
taosArrayDestroy(pHisWins);
}
if (pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator)) {
code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
pInfo->pUpdated = NULL;
code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
SSDataBlock* resBlock = buildEventResult(pOperator);
if (resBlock != NULL) {
@ -733,7 +736,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
void* pBuf = NULL;
code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_EVENT_OP_STATE_NAME,
strlen(STREAM_EVENT_OP_STATE_NAME), &pBuf, &size);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
qDebug("===stream=== event window operator reload state. get result count:%d", num);
@ -757,7 +760,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,
pSeKeyBuf[i].groupId, i);
code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
// event window has been deleted
if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) {
@ -766,18 +769,18 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
setEventWindowFlag(pAggSup, &curInfo);
if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) {
code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
continue;
}
code = compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {
code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
if (!curInfo.pWinFlag->endFlag) {
@ -786,17 +789,17 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
code = saveResult(curInfo.winInfo, pInfo->pSeUpdated);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
} else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) {
code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
SSessionKey key = {0};
getSessionHashKey(&curInfo.winInfo.sessionWin, &key);
code =
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
}
taosMemoryFree(pBuf);
@ -844,24 +847,21 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
.deleteMark = getDeleteMark(&pEventNode->window, 0),
};
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
QUERY_CHECK_CODE(code, lino, _error);
SExprSupp* pExpSup = &pOperator->exprSupp;
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pEventNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
pInfo->primaryTsIndex = tsSlotId;
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState,
sizeof(bool) + sizeof(bool), 0, &pTaskInfo->storageAPI.stateStore, pHandle,
&pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
QUERY_CHECK_CODE(code, lino, _error);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);
@ -883,14 +883,18 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
if (pInfo->isHistoryOp) {
pInfo->pAllUpdated = tSimpleHashInit(64, hashFn);
QUERY_CHECK_NULL(pInfo->pAllUpdated, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
} else {
pInfo->pAllUpdated = NULL;
}
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
QUERY_CHECK_NULL(pInfo->pCheckpointRes, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
pInfo->reCkBlock = false;
pInfo->recvGetAll = false;
pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);
QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, TSDB_CODE_OUT_OF_MEMORY);
pInfo->destHasPrimaryKey = pEventNode->window.destHasPrimayKey;
setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
@ -904,7 +908,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
if (res == TSDB_CODE_SUCCESS) {
code = doStreamEventDecodeOpState(buff, len, pOperator);
taosMemoryFree(buff);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo,
@ -912,16 +916,16 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
return pOperator;

114
source/libs/executor/src/streamfilloperator.c
View File

@ -472,7 +472,7 @@ static int32_t checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t gr
(*pRes) = false;
}
code = tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
(*pRes) = true;
_end:
@ -493,7 +493,7 @@ static int32_t buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pF
uint64_t groupId = pBlock->info.id.groupId;
bool ckRes = true;
code = checkResult(pFillSup, ts, groupId, &ckRes);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (pFillSup->hasDelete && !ckRes) {
(*pRes) = true;
@ -512,7 +512,7 @@ static int32_t buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pF
if (!filled) {
SResultCellData* pCell = getResultCell(pResRow, slotId);
code = setRowCell(pColData, pBlock->info.rows, pCell);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
}
pBlock->info.rows++;
@ -540,7 +540,7 @@ static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo*
if (inWinRange(&pFillSup->winRange, &st)) {
bool res = true;
code = buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock, &res);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
pFillSup->interval.precision);
@ -561,7 +561,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
bool ckRes = true;
code = checkResult(pFillSup, pFillInfo->current, groupId, &ckRes);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if ((pFillSup->hasDelete && !ckRes) ||
!inWinRange(&pFillSup->winRange, &st)) {
@ -588,7 +588,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index);
if (!filled) {
code = setRowCell(pColData, index, pCell);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
} else {
if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) {
@ -606,7 +606,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
cur.val = taosMemoryCalloc(1, pCell->bytes);
taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type);
code = colDataSetVal(pColData, index, (const char*)cur.val, false);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
destroySPoint(&cur);
}
}
@ -636,13 +636,13 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter*
bool res = false;
if (pFillInfo->needFill == false) {
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
return;
}
if (pFillInfo->pos == FILL_POS_START) {
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (res) {
pFillInfo->pos = FILL_POS_INVALID;
}
@ -654,7 +654,7 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter*
if (pFillInfo->pos == FILL_POS_MID) {
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (res) {
pFillInfo->pos = FILL_POS_INVALID;
}
@ -671,7 +671,7 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter*
}
if (pFillInfo->pos == FILL_POS_END) {
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (res) {
pFillInfo->pos = FILL_POS_INVALID;
}
@ -731,14 +731,14 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) {
doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes);
if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) {
code = blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
return;
}
pInfo->srcRowIndex++;
}
doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes);
code = blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
blockDataCleanup(pInfo->pSrcBlock);
_end:
@ -761,14 +761,14 @@ static int32_t buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint
SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
code = colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
code = colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
colDataSetNULL(pUidCol, pBlock->info.rows);
code = colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
colDataSetNULL(pCalStartCol, pBlock->info.rows);
colDataSetNULL(pCalEndCol, pBlock->info.rows);
@ -778,7 +778,7 @@ static int32_t buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint
void* tbname = NULL;
int32_t winCode = TSDB_CODE_SUCCESS;
code = pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname, false, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
colDataSetNULL(pTableCol, pBlock->info.rows);
} else {
@ -786,7 +786,7 @@ static int32_t buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint
char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
pAPI->stateStore.streamStateFreeVal(tbname);
}
@ -808,14 +808,14 @@ static int32_t buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY
if (hasPrevWindow(pFillSup)) {
TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval);
code = buildDeleteRange(pOperator, start, endTs, groupId, delRes);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
} else if (hasNextWindow(pFillSup)) {
TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval);
code = buildDeleteRange(pOperator, startTs, end, groupId, delRes);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
} else {
code = buildDeleteRange(pOperator, startTs, endTs, groupId, delRes);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
@ -836,7 +836,7 @@ static int32_t doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, T
pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
if (!pInfo->pFillInfo->needFill) {
code = buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
} else {
STimeRange tw = {
.skey = startTs,
@ -846,7 +846,7 @@ static int32_t doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, T
void* tmp = taosArrayPush(pInfo->pFillInfo->delRanges, &tw);
if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
}
@ -960,7 +960,7 @@ static int32_t doDeleteFillResult(SOperatorInfo* pOperator) {
pAPI->stateStore.streamStateFreeCur(pCur);
code = doDeleteFillResultImpl(pOperator, ts, endTs, groupId);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
pFillInfo->current = pFillInfo->end + 1;
@ -988,17 +988,19 @@ static int32_t doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBl
blockDataCleanup(pDstBlock);
code = blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
code = setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
QUERY_CHECK_CODE(code, lino, _end);
code = projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
pDstBlock->info.rows = 0;
pSup = &pInfo->pFillSup->notFillExprSup;
setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
code = setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
QUERY_CHECK_CODE(code, lino, _end);
code = projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId;
@ -1076,7 +1078,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
blockDataCleanup(pInfo->pDelRes);
pInfo->pFillSup->hasDelete = true;
code = doDeleteFillResult(pOperator);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
@ -1088,7 +1090,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
case STREAM_INVALID:
case STREAM_PULL_DATA: {
code = doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
pInfo->srcRowIndex = -1;
@ -1104,7 +1106,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
doStreamFillImpl(pOperator);
code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
@ -1161,7 +1163,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter));
if (!pFillSup) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
pFillSup->numOfFillCols = numOfFillCols;
int32_t numOfNotFillCols = 0;
@ -1174,11 +1176,11 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
pFillSup->pAPI = pAPI;
code = initResultBuf(pFillSup);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore);
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pFillSup->pResMap = tSimpleHashInit(16, hashFn);
@ -1199,7 +1201,7 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo));
if (!pFillInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
pFillInfo->start = INT64_MIN;
@ -1210,7 +1212,7 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo));
if (!pFillInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
pFillInfo->pLinearInfo->hasNext = false;
@ -1221,13 +1223,13 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
if (!pFillInfo->pLinearInfo->pEndPoints) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
if (!pFillInfo->pLinearInfo->pNextEndPoints) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) {
@ -1236,25 +1238,25 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
value.val = taosMemoryCalloc(1, pColData->info.bytes);
if (!value.val) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
void* tmpRes = taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value);
if (!tmpRes) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
value.val = taosMemoryCalloc(1, pColData->info.bytes);
if (!value.val) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
tmpRes = taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value);
if (!tmpRes) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
}
}
@ -1266,14 +1268,14 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData));
if (!pFillInfo->pResRow) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
pFillInfo->pResRow->key = INT64_MIN;
pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize);
if (!pFillInfo->pResRow->pRowVal) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
@ -1288,7 +1290,7 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange));
if (!pFillInfo->delRanges) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
QUERY_CHECK_CODE(code, lino, _end);
}
pFillInfo->delIndex = 0;
@ -1311,7 +1313,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
}
SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval;
@ -1320,17 +1322,17 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI);
if (!pInfo->pFillSup) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
}
initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
code = blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes);
if (!pInfo->pFillInfo) {
@ -1371,11 +1373,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
if (!pInfo->pDelRes) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
}
code = blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId;
pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId;
@ -1383,13 +1385,13 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
int32_t numOfOutputCols = 0;
code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols,
COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->srcRowIndex = -1;
setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo,
@ -1399,7 +1401,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
QUERY_CHECK_CODE(code, lino, _error);
return pOperator;
_error:

522
source/libs/executor/src/streamtimewindowoperator.c
View File

File diff suppressed because it is too large Load Diff

3
source/libs/executor/src/sysscanoperator.c
View File

@ -966,7 +966,8 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo,
char* tagVarChar = NULL;
if (tagData != NULL) {
if (tagType == TSDB_DATA_TYPE_JSON) {
char* tagJson = parseTagDatatoJson(tagData);
char* tagJson = NULL;
parseTagDatatoJson(tagData, &tagJson);
tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE);
memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson));
varDataSetLen(tagVarChar, strlen(tagJson));

4
source/libs/executor/src/tfill.c
View File

@ -617,7 +617,7 @@ void taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* po
SET_TYPED_DATA(point->val, outputType, r);
}
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) {
void taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) {
int32_t remain = taosNumOfRemainRows(pFillInfo);
int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity);
@ -635,8 +635,6 @@ int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t ca
", current : % d, total : % d, %s",
pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey,
pFillInfo->numOfCurrent, pFillInfo->numOfTotal, pFillInfo->id);
return numOfRes;
}
int64_t getFillInfoStart(struct SFillInfo* pFillInfo) { return pFillInfo->start; }

17
source/libs/executor/src/timesliceoperator.c
View File

@ -233,6 +233,11 @@ static bool isGroupKeyFunc(SExprInfo* pExprInfo) {
return (functionType == FUNCTION_TYPE_GROUP_KEY);
}
static bool isSelectGroupConstValueFunc(SExprInfo* pExprInfo) {
int32_t functionType = pExprInfo->pExpr->_function.functionType;
return (functionType == FUNCTION_TYPE_GROUP_CONST_VALUE);
}
static bool getIgoreNullRes(SExprSupp* pExprSup) {
for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[i];
@ -296,7 +301,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false);
continue;
} else if (!isInterpFunc(pExprInfo)) {
if (isGroupKeyFunc(pExprInfo)) {
if (isGroupKeyFunc(pExprInfo) || isSelectGroupConstValueFunc(pExprInfo)) {
if (pSrcBlock != NULL) {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot);
@ -308,7 +313,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
char* v = colDataGetData(pSrc, index);
colDataSetVal(pDst, pResBlock->info.rows, v, false);
} else {
} else if(!isSelectGroupConstValueFunc(pExprInfo)){
// use stored group key
SGroupKeys* pkey = pSliceInfo->pPrevGroupKey;
if (pkey->isNull == false) {
@ -316,6 +321,14 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
} else {
colDataSetNULL(pDst, rows);
}
} else {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
if (pkey->isNull == false) {
colDataSetVal(pDst, rows, pkey->pData, false);
} else {
colDataSetNULL(pDst, rows);
}
}
}
continue;

52
source/libs/executor/src/timewindowoperator.c
View File

@ -30,7 +30,7 @@
typedef struct SSessionAggOperatorInfo {
SOptrBasicInfo binfo;
SAggSupporter aggSup;
SExprSupp scalarSupp; // supporter for perform scalar function
SExprSupp scalarSupp; // supporter for perform scalar function
SGroupResInfo groupResInfo;
SWindowRowsSup winSup;
bool reptScan; // next round scan
@ -518,8 +518,8 @@ static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataB
// duplicated ts row does not involve in the interpolation of end value for current time window
int32_t x = endRowIndex;
while(x > 0) {
if (tsCols[x] == tsCols[x-1]) {
while (x > 0) {
if (tsCols[x] == tsCols[x - 1]) {
x -= 1;
} else {
endRowIndex = x;
@ -528,7 +528,8 @@ static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataB
}
TSKEY endKey = (pInfo->binfo.inputTsOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
bool interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, nextRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
bool interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, nextRowIndex, pBlock->pDataBlock, tsCols,
endKey, win);
if (interp) {
setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
}
@ -1203,6 +1204,8 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -1223,8 +1226,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState,
&pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1274,7 +1277,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
goto _error;
}
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, pInfo);
if (pInfo->timeWindowInterpo) {
pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
@ -1421,7 +1425,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
if (pInfo->scalarSupp.pExprInfo != NULL) {
SExprSupp* pExprSup = &pInfo->scalarSupp;
projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
}
}
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);
@ -1457,6 +1461,8 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
// todo make this as an non-blocking operator
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -1487,7 +1493,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
pInfo->binfo.inputTsOrder = pStateNode->window.node.inputTsOrder;
pInfo->binfo.outputTsOrder = pStateNode->window.node.outputTsOrder;
int32_t code = filterInitFromNode((SNode*)pStateNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
code = filterInitFromNode((SNode*)pStateNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1511,7 +1517,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
pInfo->twAggSup =
(STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->tsSlotId = tsSlotId;
@ -1548,13 +1555,15 @@ void destroySWindowOperatorInfo(void* param) {
cleanupAggSup(&pInfo->aggSup);
cleanupExprSupp(&pInfo->scalarSupp);
cleanupGroupResInfo(&pInfo->groupResInfo);
taosMemoryFreeClear(param);
}
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -1571,7 +1580,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
SSDataBlock* pResBlock = createDataBlockFromDescNode(pSessionNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
@ -1582,7 +1591,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
pInfo->gap = pSessionNode->gap;
initResultRowInfo(&pInfo->binfo.resultRowInfo);
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
pInfo->binfo.pRes = pResBlock;
@ -1830,6 +1840,8 @@ static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (miaInfo == NULL || pOperator == NULL) {
@ -1852,7 +1864,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
SExprSupp* pSup = &pOperator->exprSupp;
pSup->hasWindowOrGroup = true;
int32_t code = filterInitFromNode((SNode*)pNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
code = filterInitFromNode((SNode*)pNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1879,7 +1891,8 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
SSDataBlock* pResBlock = createDataBlockFromDescNode(pNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&iaInfo->binfo, pResBlock);
initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
code = initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
QUERY_CHECK_CODE(code, lino, _error);
iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, iaInfo);
if (iaInfo->timeWindowInterpo) {
@ -2126,6 +2139,8 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SMergeIntervalAggOperatorInfo* pMergeIntervalInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pMergeIntervalInfo == NULL || pOperator == NULL) {
@ -2158,15 +2173,16 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
initResultSizeInfo(&pOperator->resultInfo, 4096);
int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
SSDataBlock* pResBlock = createDataBlockFromDescNode(pIntervalPhyNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&pIntervalInfo->binfo, pResBlock);
initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win);
code = initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win);
QUERY_CHECK_CODE(code, lino, _error);
pIntervalInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, num, pIntervalInfo);
if (pIntervalInfo->timeWindowInterpo) {

Some files were not shown because too many files have changed in this diff Show More