xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli

This commit is contained in:
Minghao Li 2022-03-23 18:06:26 +08:00
parent d5188f14f9 0646a099b2
commit e12d24d220
205 changed files with 10726 additions and 7036 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -88,6 +88,7 @@ tests/examples/JDBC/JDBCDemo/.classpath
tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.project
tests/examples/JDBC/JDBCDemo/.settings/ tests/examples/JDBC/JDBCDemo/.settings/
source/libs/parser/inc/sql.* source/libs/parser/inc/sql.*
tests/script/tmqResult.txt
# Emacs # Emacs
# -*- mode: gitignore; -*- # -*- mode: gitignore; -*-

21
contrib/CMakeLists.txt
View File

@ -83,11 +83,6 @@ if(${BUILD_WITH_NURAFT})
cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_NURAFT}) endif(${BUILD_WITH_NURAFT})
# iconv
if(${BUILD_WITH_ICONV})
cat("${CMAKE_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_ICONV})
# download dependencies # download dependencies
configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt") configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@ -213,11 +208,9 @@ endif(${BUILD_WITH_TRAFT})
# LIBUV # LIBUV
if(${BUILD_WITH_UV}) if(${BUILD_WITH_UV})
add_compile_options(-Wno-sign-compare) if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
if (${TD_WINDOWS}) MESSAGE("Windows need set no-sign-compare")
file(READ "libuv/include/uv.h" CONTENTS) add_compile_options(-Wno-sign-compare)
string(REGEX REPLACE "/([\r]*)\nstruct uv_tcp_s {" "/\\1\ntypedef BOOL (PASCAL *LPFN_CONNECTEX) (SOCKET s, const struct sockaddr* name, int namelen, PVOID lpSendBuffer, DWORD dwSendDataLength,LPDWORD lpdwBytesSent, LPOVERLAPPED lpOverlapped);\\1\nstruct uv_tcp_s {" CONTENTS_NEW "${CONTENTS}")
file(WRITE "libuv/include/uv.h" "${CONTENTS_NEW}")
endif () endif ()
add_subdirectory(libuv) add_subdirectory(libuv)
endif(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV})
@ -250,15 +243,7 @@ if(${BUILD_WITH_SQLITE})
endif(${BUILD_WITH_SQLITE}) endif(${BUILD_WITH_SQLITE})
# pthread # pthread
if(${BUILD_PTHREAD})
add_definitions(-DPTW32_STATIC_LIB)
add_subdirectory(pthread)
endif(${BUILD_PTHREAD})
# iconv
if(${BUILD_WITH_ICONV})
add_subdirectory(iconv)
endif(${BUILD_WITH_ICONV})
# ================================================================================================ # ================================================================================================
# Build test # Build test

30
example/CMakeLists.txt
View File

@ -1,12 +1,30 @@
aux_source_directory(src TMQ_DEMO_SRC) add_executable(tmq "")
add_executable(tstream "")
add_executable(tmq ${TMQ_DEMO_SRC}) target_sources(tmq
target_link_libraries( PRIVATE
tmq taos "src/tmq.c"
) )
target_include_directories(
tmq target_sources(tstream
PRIVATE
"src/tstream.c"
)
target_link_libraries(tmq
taos
)
target_link_libraries(tstream
taos
)
target_include_directories(tmq
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_include_directories(tstream
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq)
SET_TARGET_PROPERTIES(tstream PROPERTIES OUTPUT_NAME tstream)

107
example/src/tstream.c Normal file
View File

@ -0,0 +1,107 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <assert.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include "taos.h"
int32_t init_env() {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) {
return -1;
}
TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");
if (taos_errno(pRes) != 0) {
printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)");
if (taos_errno(pRes) != 0) {
printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");
if (taos_errno(pRes) != 0) {
printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
return 0;
}
int32_t create_stream() {
printf("create stream\n");
TAOS_RES* pRes;
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) {
return -1;
}
pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
/*const char* sql = "select min(k), max(k), sum(k) from tu1";*/
const char* sql = "select min(k), max(k), sum(k) from st1";
/*const char* sql = "select sum(k) from tu1 interval(10m)";*/
pRes = tmq_create_stream(pConn, "stream1", "out1", sql);
if (taos_errno(pRes) != 0) {
printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
taos_close(pConn);
return 0;
}
int main(int argc, char* argv[]) {
int code;
if (argc > 1) {
printf("env init\n");
code = init_env();
}
create_stream();
#if 0
tmq_t* tmq = build_consumer();
tmq_list_t* topic_list = build_topic_list();
/*perf_loop(tmq, topic_list);*/
/*basic_consume_loop(tmq, topic_list);*/
sync_consume_loop(tmq, topic_list);
#endif
}

12
include/client/taos.h
View File

@ -85,11 +85,7 @@ typedef struct taosField {
int32_t bytes; int32_t bytes;
} TAOS_FIELD; } TAOS_FIELD;
#ifdef _TD_GO_DLL_
#define DLL_EXPORT __declspec(dllexport)
#else
#define DLL_EXPORT #define DLL_EXPORT
#endif
typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
@ -218,7 +214,6 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, v
DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT tmq_list_t *tmq_list_new();
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message); DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message);
DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
@ -262,7 +257,12 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message); DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message);
/* ---------------------- OTHER ---------------------------- */ /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
/* -------------------------------- OTHER -------------------------------- */
typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);

13
include/common/taosdef.h
View File

@ -30,12 +30,13 @@ typedef int64_t tb_uid_t;
#define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX))
typedef enum { typedef enum {
TSDB_SUPER_TABLE = 1, // super table TSDB_SUPER_TABLE = 1, // super table
TSDB_CHILD_TABLE = 2, // table created from super table TSDB_CHILD_TABLE = 2, // table created from super table
TSDB_NORMAL_TABLE = 3, // ordinary table TSDB_NORMAL_TABLE = 3, // ordinary table
TSDB_STREAM_TABLE = 4, // table created by stream processing TSDB_STREAM_TABLE = 4, // table created from stream computing
TSDB_TEMP_TABLE = 5, // temp table created by nest query TSDB_TEMP_TABLE = 5, // temp table created by nest query
TSDB_TABLE_MAX = 6 TSDB_SYSTEM_TABLE = 6,
TSDB_TABLE_MAX = 7
} ETableType; } ETableType;
typedef enum { typedef enum {

74
include/common/tcommon.h
View File

@ -60,19 +60,12 @@ typedef struct SDataBlockInfo {
int16_t numOfCols; int16_t numOfCols;
int16_t hasVarCol; int16_t hasVarCol;
union {int64_t uid; int64_t blockId;}; union {int64_t uid; int64_t blockId;};
int64_t groupId; // no need to serialize
} SDataBlockInfo; } SDataBlockInfo;
//typedef struct SConstantItem {
// SColumnInfo info;
// int32_t startRow; // run-length-encoding to save the space for multiple rows
// int32_t endRow;
// SVariant value;
//} SConstantItem;
// info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList);
typedef struct SSDataBlock { typedef struct SSDataBlock {
SColumnDataAgg *pBlockAgg; SColumnDataAgg* pBlockAgg;
SArray *pDataBlock; // SArray<SColumnInfoData> SArray* pDataBlock; // SArray<SColumnInfoData>
SDataBlockInfo info; SDataBlockInfo info;
} SSDataBlock; } SSDataBlock;
@ -98,23 +91,40 @@ void* blockDataDestroy(SSDataBlock* pBlock);
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { int32_t tEncodeDataBlocks(void** buf, const SArray* blocks);
if (pBlock == NULL) { void* tDecodeDataBlocks(const void* buf, SArray* blocks);
return;
static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
// WARNING: do not use info.numOfCols,
// sometimes info.numOfCols != array size
int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfOutput; ++i) {
SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
tfree(pColInfoData->varmeta.offset);
} else {
tfree(pColInfoData->nullbitmap);
}
tfree(pColInfoData->pData);
} }
blockDataDestroy(pBlock);
taosArrayDestroy(pBlock->pDataBlock);
tfree(pBlock->pBlockAgg);
} }
static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); }
static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
int32_t tlen = 0; int32_t tlen = 0;
int32_t sz = 0; int32_t sz = 0;
tlen += taosEncodeFixedI64(buf, pRsp->consumerId); // tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics); tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics);
if (pRsp->numOfTopics == 0) return tlen; if (pRsp->numOfTopics == 0) return tlen;
tlen += tEncodeSSchemaWrapper(buf, pRsp->schemas); tlen += tEncodeSSchemaWrapper(buf, pRsp->schema);
if (pRsp->pBlockData) { if (pRsp->pBlockData) {
sz = taosArrayGetSize(pRsp->pBlockData); sz = taosArrayGetSize(pRsp->pBlockData);
} }
@ -128,15 +138,15 @@ static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp
static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
int32_t sz; int32_t sz;
buf = taosDecodeFixedI64(buf, &pRsp->consumerId); // buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics); buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics);
if (pRsp->numOfTopics == 0) return buf; if (pRsp->numOfTopics == 0) return buf;
pRsp->schemas = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); pRsp->schema = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper));
if (pRsp->schemas == NULL) return NULL; if (pRsp->schema == NULL) return NULL;
buf = tDecodeSSchemaWrapper(buf, pRsp->schemas); buf = tDecodeSSchemaWrapper(buf, pRsp->schema);
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock));
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
@ -148,13 +158,13 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
} }
static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
if (pRsp->schemas) { if (pRsp->schema) {
if (pRsp->schemas->nCols) { if (pRsp->schema->nCols) {
tfree(pRsp->schemas->pSchema); tfree(pRsp->schema->pSchema);
} }
free(pRsp->schemas); free(pRsp->schema);
} }
taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock); taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner);
pRsp->pBlockData = NULL; pRsp->pBlockData = NULL;
} }
@ -166,10 +176,8 @@ typedef struct SColumn {
int64_t dataBlockId; int64_t dataBlockId;
}; };
union { int16_t colId;
int16_t colId; int16_t slotId;
int16_t slotId;
};
char name[TSDB_COL_NAME_LEN]; char name[TSDB_COL_NAME_LEN];
int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string)
@ -196,7 +204,7 @@ typedef struct SGroupbyExpr {
typedef struct SFunctParam { typedef struct SFunctParam {
int32_t type; int32_t type;
SColumn *pCol; SColumn* pCol;
SVariant param; SVariant param;
} SFunctParam; } SFunctParam;
@ -214,12 +222,12 @@ typedef struct SResSchame {
typedef struct SExprBasicInfo { typedef struct SExprBasicInfo {
SResSchema resSchema; SResSchema resSchema;
int16_t numOfParams; // argument value of each function int16_t numOfParams; // argument value of each function
SFunctParam *pParam; SFunctParam* pParam;
} SExprBasicInfo; } SExprBasicInfo;
typedef struct SExprInfo { typedef struct SExprInfo {
struct SExprBasicInfo base; struct SExprBasicInfo base;
struct tExprNode *pExpr; struct tExprNode* pExpr;
} SExprInfo; } SExprInfo;
typedef struct SStateWindow { typedef struct SStateWindow {

17
include/common/tdatablock.h
View File

@ -102,8 +102,8 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
: ((p1_)->pData + ((r_) * (p1_)->info.bytes))) : ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2);
uint32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows);
int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
@ -112,16 +112,15 @@ void colDataTrim(SColumnInfoData* pColumnInfoData);
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock); size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock); size_t blockDataGetNumOfRows(const SSDataBlock* pBlock);
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize);
int32_t pageSize);
SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount);
int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf); int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf);
SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount);
size_t blockDataGetSize(const SSDataBlock* pBlock); size_t blockDataGetSize(const SSDataBlock* pBlock);
size_t blockDataGetRowSize(const SSDataBlock* pBlock); size_t blockDataGetRowSize(SSDataBlock* pBlock);
double blockDataGetSerialRowSize(const SSDataBlock* pBlock); double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock); size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
@ -132,7 +131,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
void blockDataClearup(SSDataBlock* pDataBlock); void blockDataCleanup(SSDataBlock* pDataBlock);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock);
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
void* blockDataDestroy(SSDataBlock* pBlock); void* blockDataDestroy(SSDataBlock* pBlock);

150
include/common/tmsg.h
View File

@ -23,6 +23,7 @@
#include "tencode.h" #include "tencode.h"
#include "thash.h" #include "thash.h"
#include "tlist.h" #include "tlist.h"
#include "tname.h"
#include "trow.h" #include "trow.h"
#include "tuuid.h" #include "tuuid.h"
@ -471,6 +472,10 @@ typedef struct {
int32_t code; int32_t code;
} SQueryTableRsp; } SQueryTableRsp;
int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp);
int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp);
typedef struct { typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
int32_t numOfVgroups; int32_t numOfVgroups;
@ -863,6 +868,7 @@ void tFreeSShowRsp(SShowRsp* pRsp);
typedef struct { typedef struct {
int32_t type; int32_t type;
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN];
int64_t showId; int64_t showId;
int8_t free; int8_t free;
} SRetrieveTableReq; } SRetrieveTableReq;
@ -880,6 +886,17 @@ typedef struct {
char data[]; char data[];
} SRetrieveTableRsp; } SRetrieveTableRsp;
typedef struct {
int64_t handle;
int64_t useconds;
int8_t completed; // all results are returned to client
int8_t precision;
int8_t compressed;
int32_t compLen;
int32_t numOfRows;
char data[];
} SRetrieveMetaTableRsp;
typedef struct { typedef struct {
char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port
int32_t port; int32_t port;
@ -1282,7 +1299,7 @@ static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) {
if (pRebSub == NULL) { if (pRebSub == NULL) {
goto _err; goto _err;
} }
pRebSub->key = key; pRebSub->key = strdup(key);
pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t)); pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t));
if (pRebSub->lostConsumers == NULL) { if (pRebSub->lostConsumers == NULL) {
goto _err; goto _err;
@ -1345,33 +1362,54 @@ typedef struct {
int64_t tuid; int64_t tuid;
} SDDropTopicReq; } SDDropTopicReq;
typedef struct {
float xFilesFactor;
int8_t delayUnit;
int8_t nFuncIds;
int32_t* pFuncIds;
int64_t delay;
} SRSmaParam;
typedef struct SVCreateTbReq { typedef struct SVCreateTbReq {
int64_t ver; // use a general definition int64_t ver; // use a general definition
char* dbFName;
char* name; char* name;
uint32_t ttl; uint32_t ttl;
uint32_t keep; uint32_t keep;
uint8_t type; union {
uint8_t info;
struct {
uint8_t rollup : 1; // 1 means rollup sma
uint8_t type : 7;
};
};
union { union {
struct { struct {
tb_uid_t suid; tb_uid_t suid;
uint32_t nCols; uint32_t nCols;
SSchema* pSchema; SSchema* pSchema;
uint32_t nTagCols; uint32_t nTagCols;
SSchema* pTagSchema; SSchema* pTagSchema;
col_id_t nBSmaCols;
col_id_t* pBSmaCols;
SRSmaParam* pRSmaParam;
} stbCfg; } stbCfg;
struct { struct {
tb_uid_t suid; tb_uid_t suid;
SKVRow pTag; SKVRow pTag;
} ctbCfg; } ctbCfg;
struct { struct {
uint32_t nCols; uint32_t nCols;
SSchema* pSchema; SSchema* pSchema;
col_id_t nBSmaCols;
col_id_t* pBSmaCols;
SRSmaParam* pRSmaParam;
} ntbCfg; } ntbCfg;
}; };
} SVCreateTbReq, SVUpdateTbReq; } SVCreateTbReq, SVUpdateTbReq;
typedef struct { typedef struct {
int tmp; // TODO: to avoid compile error int32_t code;
} SVCreateTbRsp, SVUpdateTbRsp; } SVCreateTbRsp, SVUpdateTbRsp;
int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
@ -1382,13 +1420,16 @@ typedef struct {
SArray* pArray; SArray* pArray;
} SVCreateTbBatchReq; } SVCreateTbBatchReq;
typedef struct {
int tmp; // TODO: to avoid compile error
} SVCreateTbBatchRsp;
int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq);
void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq);
typedef struct {
SArray* rspList; // SArray<SVCreateTbRsp>
} SVCreateTbBatchRsp;
int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
typedef struct { typedef struct {
int64_t ver; int64_t ver;
char* name; char* name;
@ -2116,25 +2157,16 @@ typedef struct {
int8_t mqMsgType; int8_t mqMsgType;
int32_t code; int32_t code;
int32_t epoch; int32_t epoch;
int64_t consumerId;
} SMqRspHead; } SMqRspHead;
typedef struct {
int64_t consumerId;
SSchemaWrapper* schemas;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
int32_t numOfTopics;
SArray* pBlockData; // SArray<SSDataBlock>
} SMqPollRsp;
// one req for one vg+topic
typedef struct { typedef struct {
SMsgHead head; SMsgHead head;
int64_t consumerId; int64_t consumerId;
int64_t blockingTime; int64_t blockingTime;
int32_t epoch; int32_t epoch;
int8_t withSchema;
char cgroup[TSDB_CGROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
int64_t currentOffset; int64_t currentOffset;
@ -2153,19 +2185,22 @@ typedef struct {
} SMqSubTopicEp; } SMqSubTopicEp;
typedef struct { typedef struct {
int64_t consumerId; SMqRspHead head;
char cgroup[TSDB_CGROUP_LEN]; int64_t reqOffset;
SArray* topics; // SArray<SMqSubTopicEp> int64_t rspOffset;
} SMqCMGetSubEpRsp; int32_t skipLogNum;
// TODO: replace with topic name
int32_t numOfTopics;
// TODO: remove from msg
SSchemaWrapper* schema;
SArray* pBlockData; // SArray<SSDataBlock>
} SMqPollRsp;
typedef struct { typedef struct {
SMqRspHead head; SMqRspHead head;
union { char cgroup[TSDB_CGROUP_LEN];
SMqPollRsp consumeRsp; SArray* topics; // SArray<SMqSubTopicEp>
SMqCMGetSubEpRsp getEpRsp; } SMqCMGetSubEpRsp;
};
void* extra;
} SMqMsgWrapper;
typedef struct { typedef struct {
int32_t curBlock; int32_t curBlock;
@ -2173,11 +2208,13 @@ typedef struct {
void** uData; void** uData;
} SMqRowIter; } SMqRowIter;
struct tmq_message_t_v1 { struct tmq_message_t {
SMqPollRsp rsp; SMqPollRsp msg;
void* vg;
SMqRowIter iter; SMqRowIter iter;
}; };
#if 0
struct tmq_message_t { struct tmq_message_t {
SMqRspHead head; SMqRspHead head;
union { union {
@ -2189,6 +2226,7 @@ struct tmq_message_t {
int32_t curRow; int32_t curRow;
void** uData; void** uData;
}; };
#endif
static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
@ -2241,8 +2279,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pRsp->consumerId); // tlen += taosEncodeString(buf, pRsp->cgroup);
tlen += taosEncodeString(buf, pRsp->cgroup);
int32_t sz = taosArrayGetSize(pRsp->topics); int32_t sz = taosArrayGetSize(pRsp->topics);
tlen += taosEncodeFixedI32(buf, sz); tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
@ -2253,8 +2290,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu
} }
static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) {
buf = taosDecodeFixedI64(buf, &pRsp->consumerId); // buf = taosDecodeStringTo(buf, pRsp->cgroup);
buf = taosDecodeStringTo(buf, pRsp->cgroup);
int32_t sz; int32_t sz;
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp));
@ -2274,22 +2310,30 @@ enum {
STREAM_TASK_STATUS__STOP, STREAM_TASK_STATUS__STOP,
}; };
enum {
STREAM_NEXT_OP_DST__VND = 1,
STREAM_NEXT_OP_DST__SND,
};
typedef struct { typedef struct {
void* inputHandle; void* inputHandle;
void** executor; void* executor;
} SStreamTaskParRunner; } SStreamRunner;
typedef struct { typedef struct {
int64_t streamId; int64_t streamId;
int32_t taskId; int32_t taskId;
int32_t level; int32_t level;
int8_t status; int8_t status;
int8_t pipeEnd; int8_t pipeSource;
int8_t parallel; int8_t pipeSink;
int8_t numOfRunners;
int8_t parallelizable;
int8_t nextOpDst; // vnode or snode
SEpSet NextOpEp; SEpSet NextOpEp;
char* qmsg; char* qmsg;
// not applied to encoder and decoder // not applied to encoder and decoder
SStreamTaskParRunner runner; SStreamRunner runner[8];
// void* executor; // void* executor;
// void* stateStore; // void* stateStore;
// storage handle // storage handle
@ -2301,6 +2345,8 @@ static FORCE_INLINE SStreamTask* streamTaskNew(int64_t streamId, int32_t level)
return NULL; return NULL;
} }
pTask->taskId = tGenIdPI32(); pTask->taskId = tGenIdPI32();
pTask->streamId = streamId;
pTask->level = level;
pTask->status = STREAM_TASK_STATUS__RUNNING; pTask->status = STREAM_TASK_STATUS__RUNNING;
pTask->qmsg = NULL; pTask->qmsg = NULL;
return pTask; return pTask;
@ -2321,7 +2367,7 @@ typedef struct {
typedef struct { typedef struct {
SStreamExecMsgHead head; SStreamExecMsgHead head;
// TODO: other info needed by task SArray* data; // SArray<SSDataBlock>
} SStreamTaskExecReq; } SStreamTaskExecReq;
typedef struct { typedef struct {
@ -2330,14 +2376,6 @@ typedef struct {
#pragma pack(pop) #pragma pack(pop)
struct SRpcMsg;
struct SEpSet;
struct SMgmtWrapper;
typedef int32_t (*PutToQueueFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq);
typedef int32_t (*SendReqFp)(struct SMgmtWrapper* pWrapper, struct SEpSet* epSet, struct SRpcMsg* pReq);
typedef int32_t (*SendMnodeReqFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq);
typedef void (*SendRspFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pRsp);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

55
include/common/tmsgcb.h Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_MSG_CB_H_
#define _TD_COMMON_MSG_CB_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SRpcMsg SRpcMsg;
typedef struct SEpSet SEpSet;
typedef struct SMgmtWrapper SMgmtWrapper;
typedef enum { QUERY_QUEUE, FETCH_QUEUE, WRITE_QUEUE, APPLY_QUEUE, SYNC_QUEUE, QUEUE_MAX } EQueueType;
typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, SEpSet* epSet, SRpcMsg* pReq);
typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, SRpcMsg* pRsp);
typedef struct {
SMgmtWrapper* pWrapper;
PutToQueueFp queueFps[QUEUE_MAX];
GetQueueSizeFp qsizeFp;
SendReqFp sendReqFp;
SendMnodeReqFp sendMnodeReqFp;
SendRspFp sendRspFp;
} SMsgCb;
int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq);
int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq);
void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp);
#ifdef __cplusplus
}
#endif
#endif /*_TD_COMMON_MSG_CB_H_*/

1
include/common/tmsgdef.h
View File

@ -191,6 +191,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp)
TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)

4
include/common/tname.h
View File

@ -17,7 +17,6 @@
#define _TD_COMMON_NAME_H_ #define _TD_COMMON_NAME_H_
#include "tdef.h" #include "tdef.h"
#include "tmsg.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -61,7 +60,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type);
int32_t tNameSetAcctId(SName* dst, int32_t acctId); int32_t tNameSetAcctId(SName* dst, int32_t acctId);
SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); bool tNameDBNameEqual(SName* left, SName* right);
#ifdef __cplusplus #ifdef __cplusplus
} }

11
include/common/trow.h
View File

@ -166,6 +166,7 @@ typedef struct {
#define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_HEAD_LEN (sizeof(STSRow))
#define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t))
#define TD_ROW_INFO(r) ((r)->info)
#define TD_ROW_TYPE(r) ((r)->type) #define TD_ROW_TYPE(r) ((r)->type)
#define TD_ROW_DELETE(r) ((r)->del) #define TD_ROW_DELETE(r) ((r)->del)
#define TD_ROW_ENDIAN(r) ((r)->endian) #define TD_ROW_ENDIAN(r) ((r)->endian)
@ -180,6 +181,7 @@ typedef struct {
// (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.
#define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN) #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN)
#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i))
#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) #define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t))
#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) #define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1)
#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) #define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v))
@ -473,6 +475,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
return terrno; return terrno;
} }
TD_ROW_SET_INFO(pBuilder->pBuf, 0);
TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType);
uint32_t len = 0; uint32_t len = 0;
@ -968,10 +971,14 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in
#endif #endif
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) {
if (TD_COL_ROWS_NORM(pCol)) {
pVal->valType = TD_VTYPE_NORM;
} else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) {
return terrno; return terrno;
} }
if (TD_COL_ROWS_NORM(pCol) || tdValTypeIsNorm(pVal->valType)) {
if (tdValTypeIsNorm(pVal->valType)) {
if (IS_VAR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
} else { } else {

251
include/common/ttokendef.h
View File

@ -51,130 +51,133 @@
#define TK_USER 33 #define TK_USER 33
#define TK_PRIVILEGE 34 #define TK_PRIVILEGE 34
#define TK_DROP 35 #define TK_DROP 35
#define TK_SHOW 36 #define TK_DNODE 36
#define TK_DNODE 37 #define TK_PORT 37
#define TK_PORT 38 #define TK_NK_INTEGER 38
#define TK_NK_INTEGER 39 #define TK_DNODES 39
#define TK_DNODES 40 #define TK_NK_IPTOKEN 40
#define TK_NK_IPTOKEN 41 #define TK_LOCAL 41
#define TK_LOCAL 42 #define TK_QNODE 42
#define TK_QNODE 43 #define TK_ON 43
#define TK_ON 44 #define TK_DATABASE 44
#define TK_QNODES 45 #define TK_USE 45
#define TK_DATABASE 46 #define TK_IF 46
#define TK_DATABASES 47 #define TK_NOT 47
#define TK_USE 48 #define TK_EXISTS 48
#define TK_IF 49 #define TK_BLOCKS 49
#define TK_NOT 50 #define TK_CACHE 50
#define TK_EXISTS 51 #define TK_CACHELAST 51
#define TK_BLOCKS 52 #define TK_COMP 52
#define TK_CACHE 53 #define TK_DAYS 53
#define TK_CACHELAST 54 #define TK_FSYNC 54
#define TK_COMP 55 #define TK_MAXROWS 55
#define TK_DAYS 56 #define TK_MINROWS 56
#define TK_FSYNC 57 #define TK_KEEP 57
#define TK_MAXROWS 58 #define TK_PRECISION 58
#define TK_MINROWS 59 #define TK_QUORUM 59
#define TK_KEEP 60 #define TK_REPLICA 60
#define TK_PRECISION 61 #define TK_TTL 61
#define TK_QUORUM 62 #define TK_WAL 62
#define TK_REPLICA 63 #define TK_VGROUPS 63
#define TK_TTL 64 #define TK_SINGLE_STABLE 64
#define TK_WAL 65 #define TK_STREAM_MODE 65
#define TK_VGROUPS 66 #define TK_RETENTIONS 66
#define TK_SINGLE_STABLE 67 #define TK_FILE_FACTOR 67
#define TK_STREAM_MODE 68 #define TK_NK_FLOAT 68
#define TK_RETENTIONS 69 #define TK_TABLE 69
#define TK_FILE_FACTOR 70 #define TK_NK_LP 70
#define TK_NK_FLOAT 71 #define TK_NK_RP 71
#define TK_TABLE 72 #define TK_STABLE 72
#define TK_NK_LP 73 #define TK_ADD 73
#define TK_NK_RP 74 #define TK_COLUMN 74
#define TK_STABLE 75 #define TK_MODIFY 75
#define TK_TABLES 76 #define TK_RENAME 76
#define TK_STABLES 77 #define TK_TAG 77
#define TK_ADD 78 #define TK_SET 78
#define TK_COLUMN 79 #define TK_NK_EQ 79
#define TK_MODIFY 80 #define TK_USING 80
#define TK_RENAME 81 #define TK_TAGS 81
#define TK_TAG 82 #define TK_NK_DOT 82
#define TK_SET 83 #define TK_NK_COMMA 83
#define TK_NK_EQ 84 #define TK_COMMENT 84
#define TK_USING 85 #define TK_BOOL 85
#define TK_TAGS 86 #define TK_TINYINT 86
#define TK_NK_DOT 87 #define TK_SMALLINT 87
#define TK_NK_COMMA 88 #define TK_INT 88
#define TK_COMMENT 89 #define TK_INTEGER 89
#define TK_BOOL 90 #define TK_BIGINT 90
#define TK_TINYINT 91 #define TK_FLOAT 91
#define TK_SMALLINT 92 #define TK_DOUBLE 92
#define TK_INT 93 #define TK_BINARY 93
#define TK_INTEGER 94 #define TK_TIMESTAMP 94
#define TK_BIGINT 95 #define TK_NCHAR 95
#define TK_FLOAT 96 #define TK_UNSIGNED 96
#define TK_DOUBLE 97 #define TK_JSON 97
#define TK_BINARY 98 #define TK_VARCHAR 98
#define TK_TIMESTAMP 99 #define TK_MEDIUMBLOB 99
#define TK_NCHAR 100 #define TK_BLOB 100
#define TK_UNSIGNED 101 #define TK_VARBINARY 101
#define TK_JSON 102 #define TK_DECIMAL 102
#define TK_VARCHAR 103 #define TK_SMA 103
#define TK_MEDIUMBLOB 104 #define TK_ROLLUP 104
#define TK_BLOB 105 #define TK_SHOW 105
#define TK_VARBINARY 106 #define TK_DATABASES 106
#define TK_DECIMAL 107 #define TK_TABLES 107
#define TK_SMA 108 #define TK_STABLES 108
#define TK_ROLLUP 109 #define TK_MNODES 109
#define TK_INDEX 110 #define TK_MODULES 110
#define TK_FULLTEXT 111 #define TK_QNODES 111
#define TK_FUNCTION 112 #define TK_FUNCTIONS 112
#define TK_INTERVAL 113 #define TK_INDEXES 113
#define TK_TOPIC 114 #define TK_FROM 114
#define TK_AS 115 #define TK_LIKE 115
#define TK_MNODES 116 #define TK_INDEX 116
#define TK_NK_BOOL 117 #define TK_FULLTEXT 117
#define TK_NK_VARIABLE 118 #define TK_FUNCTION 118
#define TK_BETWEEN 119 #define TK_INTERVAL 119
#define TK_IS 120 #define TK_TOPIC 120
#define TK_NULL 121 #define TK_AS 121
#define TK_NK_LT 122 #define TK_NK_BOOL 122
#define TK_NK_GT 123 #define TK_NK_VARIABLE 123
#define TK_NK_LE 124 #define TK_BETWEEN 124
#define TK_NK_GE 125 #define TK_IS 125
#define TK_NK_NE 126 #define TK_NULL 126
#define TK_LIKE 127 #define TK_NK_LT 127
#define TK_MATCH 128 #define TK_NK_GT 128
#define TK_NMATCH 129 #define TK_NK_LE 129
#define TK_IN 130 #define TK_NK_GE 130
#define TK_FROM 131 #define TK_NK_NE 131
#define TK_JOIN 132 #define TK_MATCH 132
#define TK_INNER 133 #define TK_NMATCH 133
#define TK_SELECT 134 #define TK_IN 134
#define TK_DISTINCT 135 #define TK_JOIN 135
#define TK_WHERE 136 #define TK_INNER 136
#define TK_PARTITION 137 #define TK_SELECT 137
#define TK_BY 138 #define TK_DISTINCT 138
#define TK_SESSION 139 #define TK_WHERE 139
#define TK_STATE_WINDOW 140 #define TK_PARTITION 140
#define TK_SLIDING 141 #define TK_BY 141
#define TK_FILL 142 #define TK_SESSION 142
#define TK_VALUE 143 #define TK_STATE_WINDOW 143
#define TK_NONE 144 #define TK_SLIDING 144
#define TK_PREV 145 #define TK_FILL 145
#define TK_LINEAR 146 #define TK_VALUE 146
#define TK_NEXT 147 #define TK_NONE 147
#define TK_GROUP 148 #define TK_PREV 148
#define TK_HAVING 149 #define TK_LINEAR 149
#define TK_ORDER 150 #define TK_NEXT 150
#define TK_SLIMIT 151 #define TK_GROUP 151
#define TK_SOFFSET 152 #define TK_HAVING 152
#define TK_LIMIT 153 #define TK_ORDER 153
#define TK_OFFSET 154 #define TK_SLIMIT 154
#define TK_ASC 155 #define TK_SOFFSET 155
#define TK_DESC 156 #define TK_LIMIT 156
#define TK_NULLS 157 #define TK_OFFSET 157
#define TK_FIRST 158 #define TK_ASC 158
#define TK_LAST 159 #define TK_DESC 159
#define TK_NULLS 160
#define TK_FIRST 161
#define TK_LAST 162
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301

2
include/common/ttypes.h
View File

@ -46,7 +46,7 @@ typedef struct {
#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))

12
include/dnode/bnode/bnode.h
View File

@ -16,24 +16,20 @@
#ifndef _TD_BNODE_H_ #ifndef _TD_BNODE_H_
#define _TD_BNODE_H_ #define _TD_BNODE_H_
#include "tmsgcb.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
/* ------------------------ TYPES EXPOSED ------------------------ */ /* ------------------------ TYPES EXPOSED ------------------------ */
typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SBnode SBnode;
typedef struct SBnode SBnode;
typedef struct { typedef struct {
} SBnodeLoad; } SBnodeLoad;
typedef struct { typedef struct {
int32_t dnodeId; SMsgCb msgCb;
int64_t clusterId;
SMgmtWrapper *pWrapper;
SendReqFp sendReqFp;
SendMnodeReqFp sendMnodeReqFp;
SendRspFp sendRspFp;
} SBnodeOpt; } SBnodeOpt;
/* ------------------------ SBnode ------------------------ */ /* ------------------------ SBnode ------------------------ */

21
include/dnode/mnode/mnode.h
View File

@ -17,27 +17,22 @@
#define _TD_MND_H_ #define _TD_MND_H_
#include "monitor.h" #include "monitor.h"
#include "tmsgcb.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
/* ------------------------ TYPES EXPOSED ------------------------ */ /* ------------------------ TYPES EXPOSED ------------------------ */
typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SMnode SMnode;
typedef struct SMnode SMnode;
typedef struct { typedef struct {
int32_t dnodeId; int32_t dnodeId;
int64_t clusterId; int64_t clusterId;
int8_t replica; int8_t replica;
int8_t selfIndex; int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA]; SReplica replicas[TSDB_MAX_REPLICA];
SMgmtWrapper *pWrapper; SMsgCb msgCb;
PutToQueueFp putToWriteQFp;
PutToQueueFp putToReadQFp;
SendReqFp sendReqFp;
SendMnodeReqFp sendMnodeReqFp;
SendRspFp sendRspFp;
} SMnodeOpt; } SMnodeOpt;
/* ------------------------ SMnode ------------------------ */ /* ------------------------ SMnode ------------------------ */

17
include/dnode/qnode/qnode.h
View File

@ -16,13 +16,14 @@
#ifndef _TD_QNODE_H_ #ifndef _TD_QNODE_H_
#define _TD_QNODE_H_ #define _TD_QNODE_H_
#include "tmsgcb.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
/* ------------------------ TYPES EXPOSED ------------------------ */ /* ------------------------ TYPES EXPOSED ------------------------ */
typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SQnode SQnode;
typedef struct SQnode SQnode;
typedef struct { typedef struct {
int64_t numOfStartTask; int64_t numOfStartTask;
@ -36,12 +37,7 @@ typedef struct {
} SQnodeLoad; } SQnodeLoad;
typedef struct { typedef struct {
int32_t dnodeId; SMsgCb msgCb;
int64_t clusterId;
SMgmtWrapper *pWrapper;
SendReqFp sendReqFp;
SendMnodeReqFp sendMnodeReqFp;
SendRspFp sendRspFp;
} SQnodeOpt; } SQnodeOpt;
/* ------------------------ SQnode ------------------------ */ /* ------------------------ SQnode ------------------------ */
@ -74,10 +70,9 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad);
* *
* @param pQnode The qnode object. * @param pQnode The qnode object.
* @param pMsg The request message * @param pMsg The request message
* @param pRsp The response message
* @return int32_t 0 for success, -1 for failure
*/ */
int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg);
int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

12
include/dnode/snode/snode.h
View File

@ -16,7 +16,7 @@
#ifndef _TD_SNODE_H_ #ifndef _TD_SNODE_H_
#define _TD_SNODE_H_ #define _TD_SNODE_H_
#include "tcommon.h" #include "tmsgcb.h"
#include "tmsg.h" #include "tmsg.h"
#include "trpc.h" #include "trpc.h"
@ -25,20 +25,14 @@ extern "C" {
#endif #endif
/* ------------------------ TYPES EXPOSED ------------------------ */ /* ------------------------ TYPES EXPOSED ------------------------ */
typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SSnode SSnode;
typedef struct SSnode SSnode;
typedef struct { typedef struct {
int32_t reserved; int32_t reserved;
} SSnodeLoad; } SSnodeLoad;
typedef struct { typedef struct {
int32_t dnodeId; SMsgCb msgCb;
int64_t clusterId;
SMgmtWrapper *pWrapper;
SendReqFp sendReqFp;
SendMnodeReqFp sendMnodeReqFp;
SendRspFp sendRspFp;
} SSnodeOpt; } SSnodeOpt;
/* ------------------------ SSnode ------------------------ */ /* ------------------------ SSnode ------------------------ */

31
include/libs/catalog/catalog.h
View File

@ -103,16 +103,17 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name) * @param pDBName (input, full db name)
* @param forceUpdate (input, force update db vgroup info from mnode)
* @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
* @return error code * @return error code
*/ */
int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList); int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, SArray** pVgroupList);
int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo); int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo);
int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId);
int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName);
int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid);
/** /**
@ -120,7 +121,7 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId,
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name)
* @param pTableMeta(output, table meta data, NEED to free it by calller) * @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code * @return error code
*/ */
@ -131,7 +132,7 @@ int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSe
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name)
* @param pTableMeta(output, table meta data, NEED to free it by calller) * @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code * @return error code
*/ */
@ -140,28 +141,38 @@ int32_t catalogGetSTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpS
int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg);
/**
* Force refresh DB's local cached vgroup info.
* @param pCtg (input, got with catalogGetHandle)
* @param pTrans (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
* @param dbFName (input, db full name)
* @return error code
*/
int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName);
/** /**
* Force refresh a table's local cached meta data. * Force refresh a table's local cached meta data.
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name)
* @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure)
* @return error code * @return error code
*/ */
int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable);
/** /**
* Force refresh a table's local cached meta data and get the new one. * Force refresh a table's local cached meta data and get the new one.
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name)
* @param pTableMeta(output, table meta data, NEED to free it by calller) * @param pTableMeta(output, table meta data, NEED to free it by calller)
* @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure)
* @return error code * @return error code
*/ */
int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable);
@ -170,7 +181,7 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg);
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name)
* @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
* @return error code * @return error code
*/ */
@ -181,7 +192,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void *pTransporter, const
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name)
* @param vgInfo (output, vgroup info) * @param vgInfo (output, vgroup info)
* @return error code * @return error code
*/ */

56
include/libs/executor/executor.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif #endif
#include "tcommon.h" #include "tcommon.h"
#include "query.h"
typedef void* qTaskInfo_t; typedef void* qTaskInfo_t;
typedef void* DataSinkHandle; typedef void* DataSinkHandle;
@ -30,23 +31,28 @@ struct SSubplan;
typedef struct SReadHandle { typedef struct SReadHandle {
void* reader; void* reader;
void* meta; void* meta;
void* config;
} SReadHandle; } SReadHandle;
/** #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1
* Create the exec task for streaming mode #define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2
* @param pMsg
* @param streamReadHandle /**
* @return * Create the exec task for streaming mode
*/ * @param pMsg
qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle); * @param streamReadHandle
* @return
*/
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle);
/** /**
* Set the input data block for the stream scan. * Set the input data block for the stream scan.
* @param tinfo * @param tinfo
* @param input * @param input
* @param type
* @return * @return
*/ */
int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type);
/** /**
* Update the table id list, add or remove. * Update the table id list, add or remove.
@ -58,16 +64,17 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input);
*/ */
int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd); int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd);
/** /**
* Create the exec task object according to task json * Create the exec task object according to task json
* @param readHandle * @param readHandle
* @param vgId * @param vgId
* @param pTaskInfoMsg * @param pTaskInfoMsg
* @param pTaskInfo * @param pTaskInfo
* @param qId * @param qId
* @return * @return
*/ */
int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan,
qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
/** /**
* The main task execution function, including query on both table and multiple tables, * The main task execution function, including query on both table and multiple tables,
@ -77,7 +84,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
* @param handle * @param handle
* @return * @return
*/ */
int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds); int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds);
/** /**
* Retrieve the produced results information, if current query is not paused or completed, * Retrieve the produced results information, if current query is not paused or completed,
@ -140,7 +147,8 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t
* @param numOfIndex * @param numOfIndex
* @return * @return
*/ */
//int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); // int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex*
// groupByIndex, int32_t numOfIndex);
/** /**
* Update the table id list of a given query. * Update the table id list of a given query.
@ -163,19 +171,19 @@ void* qOpenTaskMgmt(int32_t vgId);
* broadcast the close information and wait for all query stop. * broadcast the close information and wait for all query stop.
* @param pExecutor * @param pExecutor
*/ */
void qTaskMgmtNotifyClosing(void* pExecutor); void qTaskMgmtNotifyClosing(void* pExecutor);
/** /**
* Re-open the query handle management module when opening the vnode again. * Re-open the query handle management module when opening the vnode again.
* @param pExecutor * @param pExecutor
*/ */
void qQueryMgmtReOpen(void *pExecutor); void qQueryMgmtReOpen(void* pExecutor);
/** /**
* Close query mgmt and clean up resources. * Close query mgmt and clean up resources.
* @param pExecutor * @param pExecutor
*/ */
void qCleanupTaskMgmt(void* pExecutor); void qCleanupTaskMgmt(void* pExecutor);
/** /**
* Add the query into the query mgmt object * Add the query into the query mgmt object
@ -184,7 +192,7 @@ void qCleanupTaskMgmt(void* pExecutor);
* @param qInfo * @param qInfo
* @return * @return
*/ */
void** qRegisterTask(void* pMgmt, uint64_t qId, void *qInfo); void** qRegisterTask(void* pMgmt, uint64_t qId, void* qInfo);
/** /**
* acquire the query handle according to the key from query mgmt object. * acquire the query handle according to the key from query mgmt object.

2
include/libs/function/function.h
View File

@ -163,7 +163,7 @@ typedef struct SInputColumnInfoData {
typedef struct SqlFunctionCtx { typedef struct SqlFunctionCtx {
SInputColumnInfoData input; SInputColumnInfoData input;
SResultDataInfo resDataInfo; SResultDataInfo resDataInfo;
uint32_t order; // asc|desc uint32_t order; // data block scanner order: asc|desc
//////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////
int32_t startRow; // start row index int32_t startRow; // start row index
int32_t size; // handled processed row number int32_t size; // handled processed row number

3
include/libs/nodes/cmdnodes.h
View File

@ -179,7 +179,8 @@ typedef struct SAlterDnodeStmt {
typedef struct SShowStmt { typedef struct SShowStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; SNode* pDbName; // SValueNode
SNode* pTbNamePattern; // SValueNode
} SShowStmt; } SShowStmt;
typedef enum EIndexType { typedef enum EIndexType {

20
include/libs/nodes/nodes.h
View File

@ -78,7 +78,6 @@ typedef enum ENodeType {
QUERY_NODE_CREATE_DATABASE_STMT, QUERY_NODE_CREATE_DATABASE_STMT,
QUERY_NODE_DROP_DATABASE_STMT, QUERY_NODE_DROP_DATABASE_STMT,
QUERY_NODE_ALTER_DATABASE_STMT, QUERY_NODE_ALTER_DATABASE_STMT,
QUERY_NODE_SHOW_DATABASES_STMT, // temp
QUERY_NODE_CREATE_TABLE_STMT, QUERY_NODE_CREATE_TABLE_STMT,
QUERY_NODE_CREATE_SUBTABLE_CLAUSE, QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
QUERY_NODE_CREATE_MULTI_TABLE_STMT, QUERY_NODE_CREATE_MULTI_TABLE_STMT,
@ -86,20 +85,13 @@ typedef enum ENodeType {
QUERY_NODE_DROP_TABLE_STMT, QUERY_NODE_DROP_TABLE_STMT,
QUERY_NODE_DROP_SUPER_TABLE_STMT, QUERY_NODE_DROP_SUPER_TABLE_STMT,
QUERY_NODE_ALTER_TABLE_STMT, QUERY_NODE_ALTER_TABLE_STMT,
QUERY_NODE_SHOW_TABLES_STMT, // temp
QUERY_NODE_SHOW_STABLES_STMT,
QUERY_NODE_CREATE_USER_STMT, QUERY_NODE_CREATE_USER_STMT,
QUERY_NODE_ALTER_USER_STMT, QUERY_NODE_ALTER_USER_STMT,
QUERY_NODE_DROP_USER_STMT, QUERY_NODE_DROP_USER_STMT,
QUERY_NODE_SHOW_USERS_STMT,
QUERY_NODE_USE_DATABASE_STMT, QUERY_NODE_USE_DATABASE_STMT,
QUERY_NODE_CREATE_DNODE_STMT, QUERY_NODE_CREATE_DNODE_STMT,
QUERY_NODE_DROP_DNODE_STMT, QUERY_NODE_DROP_DNODE_STMT,
QUERY_NODE_ALTER_DNODE_STMT, QUERY_NODE_ALTER_DNODE_STMT,
QUERY_NODE_SHOW_DNODES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT,
QUERY_NODE_SHOW_MNODES_STMT,
QUERY_NODE_SHOW_QNODES_STMT,
QUERY_NODE_CREATE_INDEX_STMT, QUERY_NODE_CREATE_INDEX_STMT,
QUERY_NODE_DROP_INDEX_STMT, QUERY_NODE_DROP_INDEX_STMT,
QUERY_NODE_CREATE_QNODE_STMT, QUERY_NODE_CREATE_QNODE_STMT,
@ -107,6 +99,18 @@ typedef enum ENodeType {
QUERY_NODE_CREATE_TOPIC_STMT, QUERY_NODE_CREATE_TOPIC_STMT,
QUERY_NODE_DROP_TOPIC_STMT, QUERY_NODE_DROP_TOPIC_STMT,
QUERY_NODE_ALTER_LOCAL_STMT, QUERY_NODE_ALTER_LOCAL_STMT,
QUERY_NODE_SHOW_DATABASES_STMT,
QUERY_NODE_SHOW_TABLES_STMT,
QUERY_NODE_SHOW_STABLES_STMT,
QUERY_NODE_SHOW_USERS_STMT,
QUERY_NODE_SHOW_DNODES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT,
QUERY_NODE_SHOW_MNODES_STMT,
QUERY_NODE_SHOW_MODULES_STMT,
QUERY_NODE_SHOW_QNODES_STMT,
QUERY_NODE_SHOW_FUNCTIONS_STMT,
QUERY_NODE_SHOW_INDEXES_STMT,
QUERY_NODE_SHOW_STREAMS_STMT,
// logic plan node // logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN, QUERY_NODE_LOGIC_PLAN_SCAN,

10
include/libs/nodes/plannodes.h
View File

@ -35,8 +35,7 @@ typedef struct SLogicNode {
typedef enum EScanType { typedef enum EScanType {
SCAN_TYPE_TAG, SCAN_TYPE_TAG,
SCAN_TYPE_TABLE, SCAN_TYPE_TABLE,
SCAN_TYPE_STABLE, SCAN_TYPE_SYSTEM_TABLE,
SCAN_TYPE_TOPIC,
SCAN_TYPE_STREAM SCAN_TYPE_STREAM
} EScanType; } EScanType;
@ -165,10 +164,14 @@ typedef struct SScanPhysiNode {
SName tableName; SName tableName;
} SScanPhysiNode; } SScanPhysiNode;
typedef SScanPhysiNode SSystemTableScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode;
typedef SScanPhysiNode SStreamScanPhysiNode; typedef SScanPhysiNode SStreamScanPhysiNode;
typedef struct SSystemTableScanPhysiNode {
SScanPhysiNode scan;
SEpSet mgmtEpSet;
} SSystemTableScanPhysiNode;
typedef struct STableScanPhysiNode { typedef struct STableScanPhysiNode {
SScanPhysiNode scan; SScanPhysiNode scan;
uint8_t scanFlag; // denotes reversed scan of data or not uint8_t scanFlag; // denotes reversed scan of data or not
@ -244,6 +247,7 @@ typedef struct SSubplan {
ESubplanType subplanType; ESubplanType subplanType;
int32_t msgType; // message type for subplan, used to denote the send message type to vnode. int32_t msgType; // message type for subplan, used to denote the send message type to vnode.
int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner.
char dbFName[TSDB_DB_FNAME_LEN];
SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node
SQueryNodeStat execNodeStat; // only for scan subplan SQueryNodeStat execNodeStat; // only for scan subplan
SNodeList* pChildren; // the datasource subplan,from which to fetch the result SNodeList* pChildren; // the datasource subplan,from which to fetch the result

1
include/libs/nodes/querynodes.h
View File

@ -130,6 +130,7 @@ typedef struct SRealTableNode {
STableNode table; // QUERY_NODE_REAL_TABLE STableNode table; // QUERY_NODE_REAL_TABLE
struct STableMeta* pMeta; struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList; SVgroupsInfo* pVgroupList;
char useDbName[TSDB_DB_NAME_LEN];
} SRealTableNode; } SRealTableNode;
typedef struct STempTableNode { typedef struct STempTableNode {

3
include/libs/parser/parser.h
View File

@ -52,7 +52,8 @@ typedef struct SQuery {
SSchema* pResSchema; SSchema* pResSchema;
SCmdMsgInfo* pCmdMsg; SCmdMsgInfo* pCmdMsg;
int32_t msgType; int32_t msgType;
bool streamQuery; SArray* pDbList;
SArray* pTableList;
} SQuery; } SQuery;
int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);

1
include/libs/planner/planner.h
View File

@ -25,6 +25,7 @@ extern "C" {
typedef struct SPlanContext { typedef struct SPlanContext {
uint64_t queryId; uint64_t queryId;
int32_t acctId; int32_t acctId;
SEpSet mgmtEpSet;
SNode* pAstRoot; SNode* pAstRoot;
bool topicQuery; bool topicQuery;
bool streamQuery; bool streamQuery;

20
include/libs/qcom/query.h
View File

@ -25,7 +25,7 @@ extern "C" {
#include "tlog.h" #include "tlog.h"
#include "tmsg.h" #include "tmsg.h"
enum { typedef enum {
JOB_TASK_STATUS_NULL = 0, JOB_TASK_STATUS_NULL = 0,
JOB_TASK_STATUS_NOT_START = 1, JOB_TASK_STATUS_NOT_START = 1,
JOB_TASK_STATUS_EXECUTING, JOB_TASK_STATUS_EXECUTING,
@ -35,12 +35,12 @@ enum {
JOB_TASK_STATUS_CANCELLING, JOB_TASK_STATUS_CANCELLING,
JOB_TASK_STATUS_CANCELLED, JOB_TASK_STATUS_CANCELLED,
JOB_TASK_STATUS_DROPPING, JOB_TASK_STATUS_DROPPING,
}; } EJobTaskType;
enum { typedef enum {
TASK_TYPE_PERSISTENT = 1, TASK_TYPE_PERSISTENT = 1,
TASK_TYPE_TEMP, TASK_TYPE_TEMP,
}; } ETaskType;
typedef struct STableComInfo { typedef struct STableComInfo {
uint8_t numOfTags; // the number of tags in schema uint8_t numOfTags; // the number of tags in schema
@ -169,6 +169,9 @@ const SSchema* tGetTbnameColumnSchema();
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
char *jobTaskStatusStr(int32_t status);
SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name);
extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen);
extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize);
@ -178,6 +181,15 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
#define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
#define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
#define NEED_SCHEDULER_RETRY_ERROR(_code) ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
#define REQUEST_MAX_TRY_TIMES 5
#define qFatal(...) \ #define qFatal(...) \
do { \ do { \
if (qDebugFlag & DEBUG_FATAL) { \ if (qDebugFlag & DEBUG_FATAL) { \

7
include/libs/qworker/qworker.h
View File

@ -20,6 +20,7 @@
extern "C" { extern "C" {
#endif #endif
#include "tmsgcb.h"
#include "trpc.h" #include "trpc.h"
@ -48,11 +49,7 @@ typedef struct {
uint64_t numOfErrors; uint64_t numOfErrors;
} SQWorkerStat; } SQWorkerStat;
typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *); int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb);
typedef int32_t (*sendReqFp)(void *, struct SEpSet *, struct SRpcMsg *);
int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj,
putReqToQueryQFp fp1, sendReqFp fp2);
int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);

8
include/libs/scheduler/scheduler.h
View File

@ -52,10 +52,10 @@ typedef struct SQueryProfileSummary {
} SQueryProfileSummary; } SQueryProfileSummary;
typedef struct SQueryResult { typedef struct SQueryResult {
int32_t code; int32_t code;
uint64_t numOfRows; uint64_t numOfRows;
int32_t msgSize; int32_t msgSize;
char *msg; char *msg;
} SQueryResult; } SQueryResult;
typedef struct STaskInfo { typedef struct STaskInfo {

417
include/os/osAtomic.h
View File

@ -20,337 +20,96 @@
extern "C" { extern "C" {
#endif #endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) // If the error is in a third-party library, place this header file under the third-party library header file.
#define atomic_load_8(ptr) (*(char volatile*)(ptr)) // When you want to use this feature, you should find or add the same function in the following section.
#define atomic_load_16(ptr) (*(short volatile*)(ptr)) #ifndef ALLOW_FORBID_FUNC
#define atomic_load_32(ptr) (*(long volatile*)(ptr)) #define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID
#define atomic_load_64(ptr) (*(__int64 volatile*)(ptr)) #define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID
#define atomic_load_ptr(ptr) (*(void* volatile*)(ptr)) #define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID
#define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID
#define atomic_store_8(ptr, val) ((*(char volatile*)(ptr)) = (char)(val)) #define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID
#define atomic_store_16(ptr, val) ((*(short volatile*)(ptr)) = (short)(val)) #define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID
#define atomic_store_32(ptr, val) ((*(long volatile*)(ptr)) = (long)(val)) #define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID
#define atomic_store_64(ptr, val) ((*(__int64 volatile*)(ptr)) = (__int64)(val)) #define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID
#define atomic_store_ptr(ptr, val) ((*(void* volatile*)(ptr)) = (void*)(val)) #define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID
#define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID
#define atomic_exchange_8(ptr, val) _InterlockedExchange8((char volatile*)(ptr), (char)(val)) #define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID
#define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val)) #define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID
#define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val)) #define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID
#define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val)) #define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID
#ifdef _WIN64
#define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val))
#else
#define atomic_exchange_ptr(ptr, val) _InlineInterlockedExchangePointer((void* volatile*)(ptr), (void*)(val))
#endif
#ifdef _TD_GO_DLL_
#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
#else
#define atomic_val_compare_exchange_8(ptr, oldval, newval) _InterlockedCompareExchange8((char volatile*)(ptr), (char)(newval), (char)(oldval))
#endif
#define atomic_val_compare_exchange_16(ptr, oldval, newval) _InterlockedCompareExchange16((short volatile*)(ptr), (short)(newval), (short)(oldval))
#define atomic_val_compare_exchange_32(ptr, oldval, newval) _InterlockedCompareExchange((long volatile*)(ptr), (long)(newval), (long)(oldval))
#define atomic_val_compare_exchange_64(ptr, oldval, newval) _InterlockedCompareExchange64((__int64 volatile*)(ptr), (__int64)(newval), (__int64)(oldval))
#define atomic_val_compare_exchange_ptr(ptr, oldval, newval) _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval))
char interlocked_add_fetch_8(char volatile *ptr, char val);
short interlocked_add_fetch_16(short volatile *ptr, short val);
long interlocked_add_fetch_32(long volatile *ptr, long val);
__int64 interlocked_add_fetch_64(__int64 volatile *ptr, __int64 val);
char interlocked_and_fetch_8(char volatile* ptr, char val);
short interlocked_and_fetch_16(short volatile* ptr, short val);
long interlocked_and_fetch_32(long volatile* ptr, long val);
__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val);
__int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val);
char interlocked_or_fetch_8(char volatile* ptr, char val);
short interlocked_or_fetch_16(short volatile* ptr, short val);
long interlocked_or_fetch_32(long volatile* ptr, long val);
__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val);
char interlocked_xor_fetch_8(char volatile* ptr, char val);
short interlocked_xor_fetch_16(short volatile* ptr, short val);
long interlocked_xor_fetch_32(long volatile* ptr, long val);
__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val);
__int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val);
#define atomic_add_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), (char)(val))
#define atomic_add_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), (short)(val))
#define atomic_add_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), (long)(val))
#define atomic_add_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), (__int64)(val))
#ifdef _TD_GO_DLL_
#define atomic_fetch_add_8 __sync_fetch_and_ad
#define atomic_fetch_add_16 __sync_fetch_and_add
#else
#define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val))
#define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val))
#endif
#define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val))
#define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val))
#define atomic_fetch_add_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), (long)(val))
#define atomic_fetch_add_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), (__int64)(val))
#define atomic_sub_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), -(char)(val))
#define atomic_sub_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), -(short)(val))
#define atomic_sub_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), -(long)(val))
#define atomic_sub_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), -(__int64)(val))
#define atomic_fetch_sub_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), -(char)(val))
#define atomic_fetch_sub_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), -(short)(val))
#define atomic_fetch_sub_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), -(long)(val))
#define atomic_fetch_sub_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), -(__int64)(val))
#define atomic_and_fetch_8(ptr, val) interlocked_and_fetch_8((char volatile*)(ptr), (char)(val))
#define atomic_and_fetch_16(ptr, val) interlocked_and_fetch_16((short volatile*)(ptr), (short)(val))
#define atomic_and_fetch_32(ptr, val) interlocked_and_fetch_32((long volatile*)(ptr), (long)(val))
#define atomic_and_fetch_64(ptr, val) interlocked_and_fetch_64((__int64 volatile*)(ptr), (__int64)(val))
#define atomic_fetch_and_8(ptr, val) _InterlockedAnd8((char volatile*)(ptr), (char)(val))
#define atomic_fetch_and_16(ptr, val) _InterlockedAnd16((short volatile*)(ptr), (short)(val))
#define atomic_fetch_and_32(ptr, val) _InterlockedAnd((long volatile*)(ptr), (long)(val))
#define atomic_fetch_and_64(ptr, val) interlocked_fetch_and_64((__int64 volatile*)(ptr), (__int64)(val))
#define atomic_or_fetch_8(ptr, val) interlocked_or_fetch_8((char volatile*)(ptr), (char)(val))
#define atomic_or_fetch_16(ptr, val) interlocked_or_fetch_16((short volatile*)(ptr), (short)(val))
#define atomic_or_fetch_32(ptr, val) interlocked_or_fetch_32((long volatile*)(ptr), (long)(val))
#define atomic_or_fetch_64(ptr, val) interlocked_or_fetch_64((__int64 volatile*)(ptr), (__int64)(val))
#define atomic_fetch_or_8(ptr, val) _InterlockedOr8((char volatile*)(ptr), (char)(val))
#define atomic_fetch_or_16(ptr, val) _InterlockedOr16((short volatile*)(ptr), (short)(val))
#define atomic_fetch_or_32(ptr, val) _InterlockedOr((long volatile*)(ptr), (long)(val))
#define atomic_fetch_or_64(ptr, val) interlocked_fetch_or_64((__int64 volatile*)(ptr), (__int64)(val))
#define atomic_xor_fetch_8(ptr, val) interlocked_xor_fetch_8((char volatile*)(ptr), (char)(val))
#define atomic_xor_fetch_16(ptr, val) interlocked_xor_fetch_16((short volatile*)(ptr), (short)(val))
#define atomic_xor_fetch_32(ptr, val) interlocked_xor_fetch_32((long volatile*)(ptr), (long)(val))
#define atomic_xor_fetch_64(ptr, val) interlocked_xor_fetch_64((__int64 volatile*)(ptr), (__int64)(val))
#define atomic_fetch_xor_8(ptr, val) _InterlockedXor8((char volatile*)(ptr), (char)(val))
#define atomic_fetch_xor_16(ptr, val) _InterlockedXor16((short volatile*)(ptr), (short)(val))
#define atomic_fetch_xor_32(ptr, val) _InterlockedXor((long volatile*)(ptr), (long)(val))
#define atomic_fetch_xor_64(ptr, val) interlocked_fetch_xor_64((__int64 volatile*)(ptr), (__int64)(val))
#ifdef _WIN64
#define atomic_add_fetch_ptr atomic_add_fetch_64
#define atomic_fetch_add_ptr atomic_fetch_add_64
#define atomic_sub_fetch_ptr atomic_sub_fetch_64
#define atomic_fetch_sub_ptr atomic_fetch_sub_64
#define atomic_and_fetch_ptr atomic_and_fetch_64
#define atomic_fetch_and_ptr atomic_fetch_and_64
#define atomic_or_fetch_ptr atomic_or_fetch_64
#define atomic_fetch_or_ptr atomic_fetch_or_64
#define atomic_xor_fetch_ptr atomic_xor_fetch_64
#define atomic_fetch_xor_ptr atomic_fetch_xor_64
#else
#define atomic_add_fetch_ptr atomic_add_fetch_32
#define atomic_fetch_add_ptr atomic_fetch_add_32
#define atomic_sub_fetch_ptr atomic_sub_fetch_32
#define atomic_fetch_sub_ptr atomic_fetch_sub_32
#define atomic_and_fetch_ptr atomic_and_fetch_32
#define atomic_fetch_and_ptr atomic_fetch_and_32
#define atomic_or_fetch_ptr atomic_or_fetch_32
#define atomic_fetch_or_ptr atomic_fetch_or_32
#define atomic_xor_fetch_ptr atomic_xor_fetch_32
#define atomic_fetch_xor_ptr atomic_fetch_xor_32
#endif
#elif defined(_TD_NINGSI_60)
/*
* type __sync_fetch_and_add (type *ptr, type value);
* type __sync_fetch_and_sub (type *ptr, type value);
* type __sync_fetch_and_or (type *ptr, type value);
* type __sync_fetch_and_and (type *ptr, type value);
* type __sync_fetch_and_xor (type *ptr, type value);
* type __sync_fetch_and_nand (type *ptr, type value);
* type __sync_add_and_fetch (type *ptr, type value);
* type __sync_sub_and_fetch (type *ptr, type value);
* type __sync_or_and_fetch (type *ptr, type value);
* type __sync_and_and_fetch (type *ptr, type value);
* type __sync_xor_and_fetch (type *ptr, type value);
* type __sync_nand_and_fetch (type *ptr, type value);
*
* bool __sync_bool_compare_and_swap (type*ptr, type oldval, type newval, ...)
* type __sync_val_compare_and_swap (type *ptr, type oldval, ?type newval, ...)
* */
#define atomic_load_8(ptr) __sync_fetch_and_add((ptr), 0)
#define atomic_load_16(ptr) __sync_fetch_and_add((ptr), 0)
#define atomic_load_32(ptr) __sync_fetch_and_add((ptr), 0)
#define atomic_load_64(ptr) __sync_fetch_and_add((ptr), 0)
#define atomic_load_ptr(ptr) __sync_fetch_and_add((ptr), 0)
#define atomic_store_8(ptr, val) (*(ptr)=(val))
#define atomic_store_16(ptr, val) (*(ptr)=(val))
#define atomic_store_32(ptr, val) (*(ptr)=(val))
#define atomic_store_64(ptr, val) (*(ptr)=(val))
#define atomic_store_ptr(ptr, val) (*(ptr)=(val))
int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val );
int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val );
int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val );
int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val );
void* atomic_exchange_ptr_impl( void **ptr, void *val );
#define atomic_exchange_8(ptr, val) atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val)
#define atomic_exchange_16(ptr, val) atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val)
#define atomic_exchange_32(ptr, val) atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val)
#define atomic_exchange_64(ptr, val) atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val)
#define atomic_exchange_ptr(ptr, val) atomic_exchange_ptr_impl((void **)ptr, (void*)val)
#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap
#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap
#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap
#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
#define atomic_add_fetch_8(ptr, val) __sync_add_and_fetch((ptr), (val))
#define atomic_add_fetch_16(ptr, val) __sync_add_and_fetch((ptr), (val))
#define atomic_add_fetch_32(ptr, val) __sync_add_and_fetch((ptr), (val))
#define atomic_add_fetch_64(ptr, val) __sync_add_and_fetch((ptr), (val))
#define atomic_add_fetch_ptr(ptr, val) __sync_add_and_fetch((ptr), (val))
#define atomic_fetch_add_8(ptr, val) __sync_fetch_and_add((ptr), (val))
#define atomic_fetch_add_16(ptr, val) __sync_fetch_and_add((ptr), (val))
#define atomic_fetch_add_32(ptr, val) __sync_fetch_and_add((ptr), (val))
#define atomic_fetch_add_64(ptr, val) __sync_fetch_and_add((ptr), (val))
#define atomic_fetch_add_ptr(ptr, val) __sync_fetch_and_add((ptr), (val))
#define atomic_sub_fetch_8(ptr, val) __sync_sub_and_fetch((ptr), (val))
#define atomic_sub_fetch_16(ptr, val) __sync_sub_and_fetch((ptr), (val))
#define atomic_sub_fetch_32(ptr, val) __sync_sub_and_fetch((ptr), (val))
#define atomic_sub_fetch_64(ptr, val) __sync_sub_and_fetch((ptr), (val))
#define atomic_sub_fetch_ptr(ptr, val) __sync_sub_and_fetch((ptr), (val))
#define atomic_fetch_sub_8(ptr, val) __sync_fetch_and_sub((ptr), (val))
#define atomic_fetch_sub_16(ptr, val) __sync_fetch_and_sub((ptr), (val))
#define atomic_fetch_sub_32(ptr, val) __sync_fetch_and_sub((ptr), (val))
#define atomic_fetch_sub_64(ptr, val) __sync_fetch_and_sub((ptr), (val))
#define atomic_fetch_sub_ptr(ptr, val) __sync_fetch_and_sub((ptr), (val))
#define atomic_and_fetch_8(ptr, val) __sync_and_and_fetch((ptr), (val))
#define atomic_and_fetch_16(ptr, val) __sync_and_and_fetch((ptr), (val))
#define atomic_and_fetch_32(ptr, val) __sync_and_and_fetch((ptr), (val))
#define atomic_and_fetch_64(ptr, val) __sync_and_and_fetch((ptr), (val))
#define atomic_and_fetch_ptr(ptr, val) __sync_and_and_fetch((ptr), (val))
#define atomic_fetch_and_8(ptr, val) __sync_fetch_and_and((ptr), (val))
#define atomic_fetch_and_16(ptr, val) __sync_fetch_and_and((ptr), (val))
#define atomic_fetch_and_32(ptr, val) __sync_fetch_and_and((ptr), (val))
#define atomic_fetch_and_64(ptr, val) __sync_fetch_and_and((ptr), (val))
#define atomic_fetch_and_ptr(ptr, val) __sync_fetch_and_and((ptr), (val))
#define atomic_or_fetch_8(ptr, val) __sync_or_and_fetch((ptr), (val))
#define atomic_or_fetch_16(ptr, val) __sync_or_and_fetch((ptr), (val))
#define atomic_or_fetch_32(ptr, val) __sync_or_and_fetch((ptr), (val))
#define atomic_or_fetch_64(ptr, val) __sync_or_and_fetch((ptr), (val))
#define atomic_or_fetch_ptr(ptr, val) __sync_or_and_fetch((ptr), (val))
#define atomic_fetch_or_8(ptr, val) __sync_fetch_and_or((ptr), (val))
#define atomic_fetch_or_16(ptr, val) __sync_fetch_and_or((ptr), (val))
#define atomic_fetch_or_32(ptr, val) __sync_fetch_and_or((ptr), (val))
#define atomic_fetch_or_64(ptr, val) __sync_fetch_and_or((ptr), (val))
#define atomic_fetch_or_ptr(ptr, val) __sync_fetch_and_or((ptr), (val))
#define atomic_xor_fetch_8(ptr, val) __sync_xor_and_fetch((ptr), (val))
#define atomic_xor_fetch_16(ptr, val) __sync_xor_and_fetch((ptr), (val))
#define atomic_xor_fetch_32(ptr, val) __sync_xor_and_fetch((ptr), (val))
#define atomic_xor_fetch_64(ptr, val) __sync_xor_and_fetch((ptr), (val))
#define atomic_xor_fetch_ptr(ptr, val) __sync_xor_and_fetch((ptr), (val))
#define atomic_fetch_xor_8(ptr, val) __sync_fetch_and_xor((ptr), (val))
#define atomic_fetch_xor_16(ptr, val) __sync_fetch_and_xor((ptr), (val))
#define atomic_fetch_xor_32(ptr, val) __sync_fetch_and_xor((ptr), (val))
#define atomic_fetch_xor_64(ptr, val) __sync_fetch_and_xor((ptr), (val))
#define atomic_fetch_xor_ptr(ptr, val) __sync_fetch_and_xor((ptr), (val))
#else
#define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
#define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
#define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
#define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
#define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
#define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap
#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap
#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap
#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
#define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
#define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
#endif #endif
int8_t atomic_load_8(int8_t volatile *ptr);
int16_t atomic_load_16(int16_t volatile *ptr);
int32_t atomic_load_32(int32_t volatile *ptr);
int64_t atomic_load_64(int64_t volatile *ptr);
void* atomic_load_ptr(void *ptr);
void atomic_store_8(int8_t volatile *ptr, int8_t val);
void atomic_store_16(int16_t volatile *ptr, int16_t val);
void atomic_store_32(int32_t volatile *ptr, int32_t val);
void atomic_store_64(int64_t volatile *ptr, int64_t val);
void atomic_store_ptr(void *ptr, void *val);
int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val);
void* atomic_exchange_ptr(void *ptr, void *val);
int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval);
int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval);
int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval);
void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval);
int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_add_fetch_ptr(void *ptr, int32_t val);
int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_add_ptr(void *ptr, int32_t val);
int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_sub_fetch_ptr(void *ptr, int32_t val);
int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_sub_ptr(void *ptr, int32_t val);
int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_and_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_and_ptr(void *ptr, void *val);
int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_or_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_or_ptr(void *ptr, void *val);
int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_xor_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_xor_ptr(void *ptr, void *val);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
include/os/osDir.h
View File

@ -17,6 +17,7 @@
#define _TD_OS_DIR_H_ #define _TD_OS_DIR_H_
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define opendir OPENDIR_FUNC_TAOS_FORBID #define opendir OPENDIR_FUNC_TAOS_FORBID
#define readdir READDIR_FUNC_TAOS_FORBID #define readdir READDIR_FUNC_TAOS_FORBID

1
include/os/osFile.h
View File

@ -23,6 +23,7 @@ extern "C" {
#include "osSocket.h" #include "osSocket.h"
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following sectio
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID #define open OPEN_FUNC_TAOS_FORBID
#define fopen FOPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID

1
include/os/osLocale.h
View File

@ -23,6 +23,7 @@ extern "C" {
#endif #endif
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define setlocale SETLOCALE_FUNC_TAOS_FORBID #define setlocale SETLOCALE_FUNC_TAOS_FORBID
#endif #endif

1
include/os/osRand.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif #endif
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define rand RAND_FUNC_TAOS_FORBID #define rand RAND_FUNC_TAOS_FORBID
#define srand SRAND_FUNC_TAOS_FORBID #define srand SRAND_FUNC_TAOS_FORBID

1
include/os/osSleep.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif #endif
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define Sleep SLEEP_FUNC_TAOS_FORBID #define Sleep SLEEP_FUNC_TAOS_FORBID
#define sleep SLEEP_FUNC_TAOS_FORBID #define sleep SLEEP_FUNC_TAOS_FORBID

4
include/os/osSocket.h
View File

@ -17,6 +17,7 @@
#define _TD_OS_SOCKET_H_ #define _TD_OS_SOCKET_H_
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define socket SOCKET_FUNC_TAOS_FORBID #define socket SOCKET_FUNC_TAOS_FORBID
#define bind BIND_FUNC_TAOS_FORBID #define bind BIND_FUNC_TAOS_FORBID
@ -52,9 +53,6 @@ extern "C" {
#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32))
#define htobe64 htonll #define htobe64 htonll
#if defined(_TD_GO_DLL_)
uint64_t htonll(uint64_t val);
#endif
#endif #endif
#if defined(_TD_DARWIN_64) #if defined(_TD_DARWIN_64)

1
include/os/osString.h
View File

@ -24,6 +24,7 @@ typedef wchar_t TdWchar;
typedef int32_t TdUcs4; typedef int32_t TdUcs4;
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID #define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID
#define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID #define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID

2
include/os/osSystem.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif #endif
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define popen POPEN_FUNC_TAOS_FORBID #define popen POPEN_FUNC_TAOS_FORBID
#define pclose PCLOSE_FUNC_TAOS_FORBID #define pclose PCLOSE_FUNC_TAOS_FORBID
@ -28,7 +29,6 @@ extern "C" {
#define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID
#endif #endif
int32_t taosSystem(const char *cmd, char *buf, int32_t bufSize);
void* taosLoadDll(const char* filename); void* taosLoadDll(const char* filename);
void* taosLoadSym(void* handle, char* name); void* taosLoadSym(void* handle, char* name);
void taosCloseDll(void* handle); void taosCloseDll(void* handle);

1
include/os/osThread.h
View File

@ -37,6 +37,7 @@ typedef pthread_condattr_t TdThreadCondAttr;
#define taosThreadCleanupPop pthread_cleanup_pop #define taosThreadCleanupPop pthread_cleanup_pop
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define pthread_t PTHREAD_T_TYPE_TAOS_FORBID #define pthread_t PTHREAD_T_TYPE_TAOS_FORBID
#define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID #define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID

7
include/os/osTime.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif #endif
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define strptime STRPTIME_FUNC_TAOS_FORBID #define strptime STRPTIME_FUNC_TAOS_FORBID
#define gettimeofday GETTIMEOFDAY_FUNC_TAOS_FORBID #define gettimeofday GETTIMEOFDAY_FUNC_TAOS_FORBID
@ -33,11 +34,7 @@ extern "C" {
#define CLOCK_REALTIME 0 #define CLOCK_REALTIME 0
#ifdef _TD_GO_DLL_ #define MILLISECOND_PER_SECOND (1000i64)
#define MILLISECOND_PER_SECOND (1000LL)
#else
#define MILLISECOND_PER_SECOND (1000i64)
#endif
#else #else
#define MILLISECOND_PER_SECOND ((int64_t)1000L) #define MILLISECOND_PER_SECOND ((int64_t)1000L)
#endif #endif

1
include/os/osTimer.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif #endif
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define timer_create TIMER_CREATE_FUNC_TAOS_FORBID #define timer_create TIMER_CREATE_FUNC_TAOS_FORBID
#define timer_settime TIMER_SETTIME_FUNC_TAOS_FORBID #define timer_settime TIMER_SETTIME_FUNC_TAOS_FORBID

1
include/os/osTimezone.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif #endif
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define tzset TZSET_FUNC_TAOS_FORBID #define tzset TZSET_FUNC_TAOS_FORBID
#endif #endif

10
include/util/taoserror.h
View File

@ -309,6 +309,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) #define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513)
#define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) #define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514)
#define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) #define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515)
#define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0516)
// tsdb // tsdb
#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600)
@ -334,8 +335,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614)
#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615)
#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616)
#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0617) #define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x0617)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0618) #define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0618)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0619)
// query // query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)
@ -437,10 +439,12 @@ int32_t* taosGetErrno();
#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) #define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404)
#define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2405) #define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2405)
#define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2406) #define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2406)
#define TSDB_CODE_CTG_VG_META_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x2407)
//scheduler //scheduler&qworker
#define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) #define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501)
#define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) #define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502)
#define TSDB_CODE_QW_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x2503)
//parser //parser
#define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600) #define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600)

2
include/util/tdef.h
View File

@ -99,7 +99,7 @@ extern const int32_t TYPE_BYTES[15];
#define TSDB_INS_TABLE_MNODES "mnodes" #define TSDB_INS_TABLE_MNODES "mnodes"
#define TSDB_INS_TABLE_MODULES "modules" #define TSDB_INS_TABLE_MODULES "modules"
#define TSDB_INS_TABLE_QNODES "qnodes" #define TSDB_INS_TABLE_QNODES "qnodes"
#define TSDB_INS_TABLE_USER_DATABASE "user_database" #define TSDB_INS_TABLE_USER_DATABASES "user_databases"
#define TSDB_INS_TABLE_USER_FUNCTIONS "user_functions" #define TSDB_INS_TABLE_USER_FUNCTIONS "user_functions"
#define TSDB_INS_TABLE_USER_INDEXES "user_indexes" #define TSDB_INS_TABLE_USER_INDEXES "user_indexes"
#define TSDB_INS_TABLE_USER_STABLES "user_stables" #define TSDB_INS_TABLE_USER_STABLES "user_stables"

2
include/util/thash.h
View File

@ -86,7 +86,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj);
* @param size * @param size
* @return * @return
*/ */
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size); int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size);
/** /**
* return the payload data with the specified key * return the payload data with the specified key

13
include/util/tqueue.h
View File

@ -42,8 +42,14 @@ shall be used to set up the protection.
typedef struct STaosQueue STaosQueue; typedef struct STaosQueue STaosQueue;
typedef struct STaosQset STaosQset; typedef struct STaosQset STaosQset;
typedef struct STaosQall STaosQall; typedef struct STaosQall STaosQall;
typedef void (*FItem)(void *ahandle, void *pItem); typedef struct {
typedef void (*FItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); void *ahandle;
int32_t workerId;
int32_t threadNum;
} SQueueInfo;
typedef void (*FItem)(SQueueInfo *pInfo, void *pItem);
typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems);
STaosQueue *taosOpenQueue(); STaosQueue *taosOpenQueue();
void taosCloseQueue(STaosQueue *queue); void taosCloseQueue(STaosQueue *queue);
@ -70,10 +76,7 @@ int32_t taosGetQueueNumber(STaosQset *qset);
int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp); int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp);
int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp);
int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId);
void taosResetQsetThread(STaosQset *qset, void *pItem); void taosResetQsetThread(STaosQset *qset, void *pItem);
int32_t taosGetQueueItemsNumber(STaosQueue *queue); int32_t taosGetQueueItemsNumber(STaosQueue *queue);
int32_t taosGetQsetItemsNumber(STaosQset *qset); int32_t taosGetQsetItemsNumber(STaosQset *qset);

66
include/util/tworker.h
View File

@ -27,33 +27,34 @@ typedef struct SWWorkerPool SWWorkerPool;
typedef struct SQWorker { typedef struct SQWorker {
int32_t id; // worker ID int32_t id; // worker ID
TdThread thread; // thread TdThread thread; // thread
SQWorkerPool *pool; SQWorkerPool *pool;
} SQWorker, SFWorker; } SQWorker;
typedef struct SQWorkerPool { typedef struct SQWorkerPool {
int32_t max; // max number of workers int32_t max; // max number of workers
int32_t min; // min number of workers int32_t min; // min number of workers
int32_t num; // current number of workers int32_t num; // current number of workers
STaosQset *qset; STaosQset *qset;
const char *name; const char *name;
SQWorker *workers; SQWorker *workers;
TdThreadMutex mutex; TdThreadMutex mutex;
} SQWorkerPool, SFWorkerPool; } SQWorkerPool;
typedef struct SWWorker { typedef struct SWWorker {
int32_t id; // worker id int32_t id; // worker id
TdThread thread; // thread TdThread thread; // thread
STaosQall *qall; STaosQall *qall;
STaosQset *qset; // queue set STaosQset *qset; // queue set
SWWorkerPool *pool; SWWorkerPool *pool;
} SWWorker; } SWWorker;
typedef struct SWWorkerPool { typedef struct SWWorkerPool {
int32_t max; // max number of workers int32_t max; // max number of workers
int32_t nextId; // from 0 to max-1, cyclic int32_t num;
const char *name; int32_t nextId; // from 0 to max-1, cyclic
SWWorker *workers; const char *name;
SWWorker *workers;
TdThreadMutex mutex; TdThreadMutex mutex;
} SWWorkerPool; } SWWorkerPool;
@ -62,16 +63,43 @@ void tQWorkerCleanup(SQWorkerPool *pool);
STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp); STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp);
void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue); void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue);
int32_t tFWorkerInit(SFWorkerPool *pool);
void tFWorkerCleanup(SFWorkerPool *pool);
STaosQueue *tFWorkerAllocQueue(SFWorkerPool *pool, void *ahandle, FItem fp);
void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue);
int32_t tWWorkerInit(SWWorkerPool *pool); int32_t tWWorkerInit(SWWorkerPool *pool);
void tWWorkerCleanup(SWWorkerPool *pool); void tWWorkerCleanup(SWWorkerPool *pool);
STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp); STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp);
void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue); void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue);
typedef struct {
const char *name;
int32_t minNum;
int32_t maxNum;
FItem fp;
void *param;
} SSingleWorkerCfg;
typedef struct {
const char *name;
STaosQueue *queue;
SQWorkerPool pool;
} SSingleWorker;
typedef struct {
const char *name;
int32_t maxNum;
FItems fp;
void *param;
} SMultiWorkerCfg;
typedef struct {
const char *name;
STaosQueue *queue;
SWWorkerPool pool;
} SMultiWorker;
int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg);
void tSingleWorkerCleanup(SSingleWorker *pWorker);
int32_t tMultiWorkerInit(SMultiWorker *pWorker, const SMultiWorkerCfg *pCfg);
void tMultiWorkerCleanup(SMultiWorker *pWorker);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

38
source/client/inc/clientInt.h
View File

@ -37,9 +37,8 @@ extern "C" {
#define CHECK_CODE_GOTO(expr, label) \ #define CHECK_CODE_GOTO(expr, label) \
do { \ do { \
int32_t code = expr; \ code = expr; \
if (TSDB_CODE_SUCCESS != code) { \ if (TSDB_CODE_SUCCESS != code) { \
terrno = code; \
goto label; \ goto label; \
} \ } \
} while (0) } while (0)
@ -142,18 +141,27 @@ typedef struct STscObj {
SAppInstInfo* pAppInfo; SAppInstInfo* pAppInfo;
} STscObj; } STscObj;
typedef struct SResultColumn {
union {
char* nullbitmap; // bitmap, one bit for each item in the list
int32_t* offset;
};
char* pData;
} SResultColumn;
typedef struct SReqResultInfo { typedef struct SReqResultInfo {
const char* pRspMsg; const char* pRspMsg;
const char* pData; const char* pData;
TAOS_FIELD* fields; TAOS_FIELD* fields;
uint32_t numOfCols; uint32_t numOfCols;
int32_t* length; int32_t* length;
TAOS_ROW row; TAOS_ROW row;
char** pCol; SResultColumn* pCol;
uint32_t numOfRows; uint32_t numOfRows;
uint64_t totalRows; uint64_t totalRows;
uint32_t current; uint32_t current;
bool completed; bool completed;
int32_t payloadLen;
} SReqResultInfo; } SReqResultInfo;
typedef struct SShowReqInfo { typedef struct SShowReqInfo {
@ -186,6 +194,8 @@ typedef struct SRequestObj {
char* msgBuf; char* msgBuf;
void* pInfo; // sql parse info, generated by parser module void* pInfo; // sql parse info, generated by parser module
int32_t code; int32_t code;
SArray* dbList;
SArray* tableList;
SQueryExecMetric metric; SQueryExecMetric metric;
SRequestSendRecvBody body; SRequestSendRecvBody body;
} SRequestObj; } SRequestObj;
@ -226,7 +236,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
void* doFetchRow(SRequestObj* pRequest); void* doFetchRow(SRequestObj* pRequest);
void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);

6
source/client/src/clientEnv.c
View File

@ -48,8 +48,8 @@ static void registerRequest(SRequestObj *pRequest) {
if (pTscObj->pAppInfo) { if (pTscObj->pAppInfo) {
SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary; SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary;
int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1); int32_t total = atomic_add_fetch_64(&pSummary->totalRequests, 1);
int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1); int32_t currentInst = atomic_add_fetch_64(&pSummary->currentRequests, 1);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
@ -62,7 +62,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
STscObj * pTscObj = pRequest->pTscObj; STscObj * pTscObj = pRequest->pTscObj;
SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary;
int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); int32_t currentInst = atomic_sub_fetch_64(&pActivity->currentRequests, 1);
int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
int64_t duration = taosGetTimestampMs() - pRequest->metric.start; int64_t duration = taosGetTimestampMs() - pRequest->metric.start;

229
source/client/src/clientImpl.c
View File

@ -12,7 +12,7 @@
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody);
static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); static int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
static bool stringLengthCheck(const char* str, size_t maxsize) { static bool stringLengthCheck(const char* str, size_t maxsize) {
if (str == NULL) { if (str == NULL) {
@ -159,8 +159,12 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
} }
code = qParseQuerySql(&cxt, pQuery); code = qParseQuerySql(&cxt, pQuery);
if (TSDB_CODE_SUCCESS == code && ((*pQuery)->haveResultSet)) { if (TSDB_CODE_SUCCESS == code) {
setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); if ((*pQuery)->haveResultSet) {
setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
}
TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*);
TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*);
} }
return code; return code;
@ -191,8 +195,17 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) { int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) {
pRequest->type = pQuery->msgType; pRequest->type = pQuery->msgType;
SPlanContext cxt = { .queryId = pRequest->requestId, .pAstRoot = pQuery->pRoot, .acctId = pRequest->pTscObj->acctId }; SPlanContext cxt = {
return qCreateQueryPlan(&cxt, pPlan, pNodeList); .queryId = pRequest->requestId,
.acctId = pRequest->pTscObj->acctId,
.mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
.pAstRoot = pQuery->pRoot
};
int32_t code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
if (code != 0) {
return code;
}
return code;
} }
void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) { void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) {
@ -219,6 +232,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
} }
pRequest->code = code; pRequest->code = code;
terrno = code;
return pRequest->code; return pRequest->code;
} }
@ -231,9 +245,96 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
} }
pRequest->code = res.code; pRequest->code = res.code;
terrno = res.code;
return pRequest->code; return pRequest->code;
} }
SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
SRequestObj* pRequest = NULL;
SQuery* pQuery = NULL;
int32_t code = 0;
SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return);
if (pQuery->directRpc) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
} else {
CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList), _return);
CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return);
}
_return:
taosArrayDestroy(pNodeList);
qDestroyQuery(pQuery);
if (NULL != pRequest && TSDB_CODE_SUCCESS != code) {
pRequest->code = terrno;
}
return pRequest;
}
int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
SCatalog *pCatalog = NULL;
int32_t code = 0;
int32_t dbNum = taosArrayGetSize(pRequest->dbList);
int32_t tblNum = taosArrayGetSize(pRequest->tableList);
if (dbNum <= 0 && tblNum <= 0) {
return TSDB_CODE_QRY_APP_ERROR;
}
code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SEpSet epset = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
for (int32_t i = 0; i < dbNum; ++i) {
char *dbFName = taosArrayGet(pRequest->dbList, i);
code = catalogRefreshDBVgInfo(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, dbFName);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
for (int32_t i = 0; i < tblNum; ++i) {
SName *tableName = taosArrayGet(pRequest->tableList, i);
code = catalogRefreshTableMeta(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, tableName, -1);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
return code;
}
SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
SRequestObj* pRequest = NULL;
int32_t retryNum = 0;
int32_t code = 0;
while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
pRequest = execQueryImpl(pTscObj, sql, sqlLen);
if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
break;
}
code = refreshMeta(pTscObj, pRequest);
if (code) {
pRequest->code = code;
break;
}
}
return pRequest;
}
TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
STscObj* pTscObj = (STscObj*)taos; STscObj* pTscObj = (STscObj*)taos;
if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
@ -242,30 +343,7 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
return NULL; return NULL;
} }
SRequestObj* pRequest = NULL; return execQuery(pTscObj, sql, sqlLen);
SQuery* pQuery = NULL;
SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
terrno = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return);
if (pQuery->directRpc) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
} else {
CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList), _return);
CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return);
pRequest->code = terrno;
}
_return:
taosArrayDestroy(pNodeList);
qDestroyQuery(pQuery);
if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) {
pRequest->code = terrno;
}
return pRequest;
} }
int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
@ -384,7 +462,7 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
tfree(pMsgBody); tfree(pMsgBody);
} }
bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) {
return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP; return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP || msgType == TDMT_VND_QUERY_HEARTBEAT_RSP;
} }
void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle; SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle;
@ -395,7 +473,6 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
assert(pRequest->self == pSendInfo->requestObjRefId); assert(pRequest->self == pSendInfo->requestObjRefId);
pRequest->metric.rsp = taosGetTimestampMs(); pRequest->metric.rsp = taosGetTimestampMs();
pRequest->code = pMsg->code;
STscObj* pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
if (pEpSet) { if (pEpSet) {
@ -479,13 +556,16 @@ void* doFetchRow(SRequestObj* pRequest) {
} }
SReqResultInfo* pResInfo = &pRequest->body.resInfo; SReqResultInfo* pResInfo = &pRequest->body.resInfo;
int32_t code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData); pRequest->code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData);
if (code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
pRequest->code = code; return NULL;
}
pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData);
if (pRequest->code != TSDB_CODE_SUCCESS) {
return NULL; return NULL;
} }
setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData);
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId);
@ -552,10 +632,23 @@ void* doFetchRow(SRequestObj* pRequest) {
_return: _return:
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current; SResultColumn* pCol = &pResultInfo->pCol[i];
if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
pResultInfo->length[i] = varDataLen(pResultInfo->row[i]); if (pCol->offset[pResultInfo->current] != -1) {
pResultInfo->row[i] = varDataVal(pResultInfo->row[i]); char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData;
pResultInfo->length[i] = varDataLen(pStart);
pResultInfo->row[i] = varDataVal(pStart);
} else {
pResultInfo->row[i] = NULL;
}
} else {
if (!colDataIsNull_f(pCol->nullbitmap, pResultInfo->current)) {
pResultInfo->row[i] = pResultInfo->pCol[i].pData + pResultInfo->fields[i].bytes * pResultInfo->current;
} else {
pResultInfo->row[i] = NULL;
}
} }
} }
@ -563,30 +656,52 @@ _return:
return pResultInfo->row; return pResultInfo->row;
} }
static void doPrepareResPtr(SReqResultInfo* pResInfo) { static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
if (pResInfo->row == NULL) { if (pResInfo->row == NULL) {
pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES);
pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); pResInfo->pCol = calloc(pResInfo->numOfCols, sizeof(SResultColumn));
pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t));
} }
if (pResInfo->row == NULL || pResInfo->pCol == NULL || pResInfo->length == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} else {
return TSDB_CODE_SUCCESS;
}
} }
void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) {
assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL);
if (numOfRows == 0) { if (numOfRows == 0) {
return; return TSDB_CODE_SUCCESS;
} }
// todo check for the failure of malloc int32_t code = doPrepareResPtr(pResultInfo);
doPrepareResPtr(pResultInfo); if (code != TSDB_CODE_SUCCESS) {
return code;
}
int32_t offset = 0; int32_t* colLength = (int32_t*)pResultInfo->pData;
char* pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
colLength[i] = htonl(colLength[i]);
if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
pResultInfo->pCol[i].offset = (int32_t*)pStart;
pStart += numOfRows * sizeof(int32_t);
} else {
pResultInfo->pCol[i].nullbitmap = pStart;
pStart += BitmapLen(pResultInfo->numOfRows);
}
pResultInfo->pCol[i].pData = pStart;
pResultInfo->length[i] = pResultInfo->fields[i].bytes; pResultInfo->length[i] = pResultInfo->fields[i].bytes;
pResultInfo->row[i] = (char*)(pResultInfo->pData + offset * pResultInfo->numOfRows); pResultInfo->row[i] = pResultInfo->pCol[i].pData;
pResultInfo->pCol[i] = pResultInfo->row[i];
offset += pResultInfo->fields[i].bytes; pStart += colLength[i];
} }
return TSDB_CODE_SUCCESS;
} }
char* getDbOfConnection(STscObj* pObj) { char* getDbOfConnection(STscObj* pObj) {
@ -608,15 +723,17 @@ void setConnectionDB(STscObj* pTscObj, const char* db) {
taosThreadMutexUnlock(&pTscObj->mutex); taosThreadMutexUnlock(&pTscObj->mutex);
} }
void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {
assert(pResultInfo != NULL && pRsp != NULL); assert(pResultInfo != NULL && pRsp != NULL);
pResultInfo->pRspMsg = (const char*)pRsp; pResultInfo->pRspMsg = (const char*)pRsp;
pResultInfo->pData = (void*)pRsp->data; pResultInfo->pData = (void*)pRsp->data;
pResultInfo->numOfRows = htonl(pRsp->numOfRows); pResultInfo->numOfRows = htonl(pRsp->numOfRows);
pResultInfo->current = 0; pResultInfo->current = 0;
pResultInfo->completed = (pRsp->completed == 1); pResultInfo->completed = (pRsp->completed == 1);
pResultInfo->payloadLen = htonl(pRsp->compLen);
// TODO handle the compressed case
pResultInfo->totalRows += pResultInfo->numOfRows; pResultInfo->totalRows += pResultInfo->numOfRows;
setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows);
} }

4
source/client/src/clientMsgHandler.c
View File

@ -238,7 +238,11 @@ int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
// todo rsp with the vnode id list // todo rsp with the vnode id list
SRequestObj* pRequest = param; SRequestObj* pRequest = param;
free(pMsg->pData); free(pMsg->pData);
if (code != TSDB_CODE_SUCCESS) {
setErrno(pRequest, code);
}
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
return code;
} }
int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {

92
source/client/src/tmq.c
View File

@ -471,8 +471,8 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
} }
sqlLen = strlen(sql); sqlLen = strlen(sql);
if (strlen(streamName) >= TSDB_TABLE_NAME_LEN) { if (strlen(tbName) >= TSDB_TABLE_NAME_LEN) {
tscError("stream name too long, max length:%d", TSDB_TABLE_NAME_LEN - 1); tscError("output tb name too long, max length:%d", TSDB_TABLE_NAME_LEN - 1);
terrno = TSDB_CODE_TSC_INVALID_INPUT; terrno = TSDB_CODE_TSC_INVALID_INPUT;
goto _return; goto _return;
} }
@ -485,6 +485,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
tscDebug("start to create stream: %s", streamName); tscDebug("start to create stream: %s", streamName);
int32_t code = 0;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode), _return); CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode), _return);
@ -571,6 +572,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
tscDebug("start to create topic: %s", topicName); tscDebug("start to create topic: %s", topicName);
int32_t code = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return); CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return);
@ -681,7 +683,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return 0; if (tmq_message == NULL) return 0;
SMqPollRsp* pRsp = &tmq_message->consumeRsp; SMqPollRsp* pRsp = &tmq_message->msg;
return pRsp->skipLogNum; return pRsp->skipLogNum;
} }
@ -690,15 +692,15 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
static bool noPrintSchema; static bool noPrintSchema;
char pBuf[128]; char pBuf[128];
SMqPollRsp* pRsp = &tmq_message->consumeRsp; SMqPollRsp* pRsp = &tmq_message->msg;
int32_t colNum = pRsp->schemas->nCols; int32_t colNum = pRsp->schema->nCols;
if (!noPrintSchema) { if (!noPrintSchema) {
printf("|"); printf("|");
for (int32_t i = 0; i < colNum; i++) { for (int32_t i = 0; i < colNum; i++) {
if (i == 0) if (i == 0)
printf(" %25s |", pRsp->schemas->pSchema[i].name); printf(" %25s |", pRsp->schema->pSchema[i].name);
else else
printf(" %15s |", pRsp->schemas->pSchema[i].name); printf(" %15s |", pRsp->schema->pSchema[i].name);
} }
printf("\n"); printf("\n");
printf("===============================================\n"); printf("===============================================\n");
@ -735,7 +737,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqClientVg* pVg = pParam->pVg; SMqClientVg* pVg = pParam->pVg;
tmq_t* tmq = pParam->tmq; tmq_t* tmq = pParam->tmq;
if (code != 0) { if (code != 0) {
printf("msg discard\n"); printf("msg discard %x\n", code);
goto WRITE_QUEUE_FAIL; goto WRITE_QUEUE_FAIL;
} }
@ -778,19 +780,19 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
goto WRITE_QUEUE_FAIL; goto WRITE_QUEUE_FAIL;
} }
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg);
pRsp->curBlock = 0; pRsp->iter.curBlock = 0;
pRsp->curRow = 0; pRsp->iter.curRow = 0;
// TODO: alloc mem // TODO: alloc mem
/*pRsp->*/ /*pRsp->*/
/*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
if (pRsp->consumeRsp.numOfTopics == 0) { if (pRsp->msg.numOfTopics == 0) {
/*printf("no data\n");*/ /*printf("no data\n");*/
taosFreeQitem(pRsp); taosFreeQitem(pRsp);
goto WRITE_QUEUE_FAIL; goto WRITE_QUEUE_FAIL;
} }
pRsp->extra = pParam->pVg; pRsp->vg = pParam->pVg;
taosWriteQitem(tmq->mqueue, pRsp); taosWriteQitem(tmq->mqueue, pRsp);
atomic_add_fetch_32(&tmq->readyRequest, 1); atomic_add_fetch_32(&tmq->readyRequest, 1);
tsem_post(&tmq->rspSem); tsem_post(&tmq->rspSem);
@ -860,14 +862,14 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
} }
tDeleteSMqCMGetSubEpRsp(&rsp); tDeleteSMqCMGetSubEpRsp(&rsp);
} else { } else {
tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp));
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1; code = -1;
goto END; goto END;
} }
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp); tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp);
taosWriteQitem(tmq->mqueue, pRsp); taosWriteQitem(tmq->mqueue, pRsp);
tsem_post(&tmq->rspSem); tsem_post(&tmq->rspSem);
@ -983,6 +985,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
return pReq; return pReq;
} }
#if 0
tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
tmq_message_t* msg = NULL; tmq_message_t* msg = NULL;
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
@ -1050,6 +1053,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
} }
return NULL; return NULL;
} }
#endif
int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
/*printf("call poll\n");*/ /*printf("call poll\n");*/
@ -1111,11 +1115,12 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
} }
// return // return
int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) { int32_t tmqHandleRes(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) { if (rspHead->mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
/*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/ /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) { if (rspHead->epoch > atomic_load_32(&tmq->epoch)) {
tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp); SMqCMGetSubEpRsp* rspMsg = (SMqCMGetSubEpRsp*)rspHead;
tmqUpdateEp(tmq, rspHead->epoch, rspMsg);
tmqClearUnhandleMsg(tmq); tmqClearUnhandleMsg(tmq);
*pReset = true; *pReset = true;
} else { } else {
@ -1129,21 +1134,22 @@ int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) {
tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
while (1) { while (1) {
tmq_message_t* rspMsg = NULL; SMqRspHead* rspHead = NULL;
taosGetQitem(tmq->qall, (void**)&rspMsg); taosGetQitem(tmq->qall, (void**)&rspHead);
if (rspMsg == NULL) { if (rspHead == NULL) {
taosReadAllQitems(tmq->mqueue, tmq->qall); taosReadAllQitems(tmq->mqueue, tmq->qall);
taosGetQitem(tmq->qall, (void**)&rspMsg); taosGetQitem(tmq->qall, (void**)&rspHead);
if (rspMsg == NULL) return NULL; if (rspHead == NULL) return NULL;
} }
if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { if (rspHead->mqMsgType == TMQ_MSG_TYPE__POLL_RSP) {
tmq_message_t* rspMsg = (tmq_message_t*)rspHead;
atomic_sub_fetch_32(&tmq->readyRequest, 1); atomic_sub_fetch_32(&tmq->readyRequest, 1);
/*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/
if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) { if (rspMsg->msg.head.epoch == atomic_load_32(&tmq->epoch)) {
/*printf("epoch match\n");*/ /*printf("epoch match\n");*/
SMqClientVg* pVg = rspMsg->extra; SMqClientVg* pVg = rspMsg->vg;
pVg->currentOffset = rspMsg->consumeRsp.rspOffset; pVg->currentOffset = rspMsg->msg.rspOffset;
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
return rspMsg; return rspMsg;
} else { } else {
@ -1153,8 +1159,8 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
} else { } else {
/*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/ /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
bool reset = false; bool reset = false;
tmqHandleRes(tmq, rspMsg, &reset); tmqHandleRes(tmq, rspHead, &reset);
taosFreeQitem(rspMsg); taosFreeQitem(rspHead);
if (pollIfReset && reset) { if (pollIfReset && reset) {
printf("reset and repoll\n"); printf("reset and repoll\n");
tmqPollImpl(tmq, blockingTime); tmqPollImpl(tmq, blockingTime);
@ -1163,6 +1169,7 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
} }
} }
#if 0
tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) {
tmq_message_t* rspMsg = NULL; tmq_message_t* rspMsg = NULL;
int64_t startTime = taosGetTimestampMs(); int64_t startTime = taosGetTimestampMs();
@ -1185,6 +1192,7 @@ tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) {
return NULL; return NULL;
} }
} }
#endif
tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
tmq_message_t* rspMsg; tmq_message_t* rspMsg;
@ -1350,7 +1358,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
void tmq_message_destroy(tmq_message_t* tmq_message) { void tmq_message_destroy(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return; if (tmq_message == NULL) return;
SMqPollRsp* pRsp = &tmq_message->consumeRsp; SMqPollRsp* pRsp = &tmq_message->msg;
tDeleteSMqConsumeRsp(pRsp); tDeleteSMqConsumeRsp(pRsp);
/*free(tmq_message);*/ /*free(tmq_message);*/
taosFreeQitem(tmq_message); taosFreeQitem(tmq_message);
@ -1366,24 +1374,24 @@ const char* tmq_err2str(tmq_resp_err_t err) {
} }
TAOS_ROW tmq_get_row(tmq_message_t* message) { TAOS_ROW tmq_get_row(tmq_message_t* message) {
SMqPollRsp* rsp = &message->consumeRsp; SMqPollRsp* rsp = &message->msg;
while (1) { while (1) {
if (message->curBlock < taosArrayGetSize(rsp->pBlockData)) { if (message->iter.curBlock < taosArrayGetSize(rsp->pBlockData)) {
SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->curBlock); SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->iter.curBlock);
if (message->curRow < pBlock->info.rows) { if (message->iter.curRow < pBlock->info.rows) {
for (int i = 0; i < pBlock->info.numOfCols; i++) { for (int i = 0; i < pBlock->info.numOfCols; i++) {
SColumnInfoData* pData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pData = taosArrayGet(pBlock->pDataBlock, i);
if (colDataIsNull_s(pData, message->curRow)) if (colDataIsNull_s(pData, message->iter.curRow))
message->uData[i] = NULL; message->iter.uData[i] = NULL;
else { else {
message->uData[i] = colDataGetData(pData, message->curRow); message->iter.uData[i] = colDataGetData(pData, message->iter.curRow);
} }
} }
message->curRow++; message->iter.curRow++;
return message->uData; return message->iter.uData;
} else { } else {
message->curBlock++; message->iter.curBlock++;
message->curRow = 0; message->iter.curRow = 0;
continue; continue;
} }
} }

38
source/client/test/clientTests.cpp
View File

@ -271,6 +271,8 @@ TEST(testCase, create_stable_Test) {
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "use abc1");
pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("error in create stable, reason:%s\n", taos_errstr(pRes)); printf("error in create stable, reason:%s\n", taos_errstr(pRes));
@ -283,17 +285,17 @@ TEST(testCase, create_stable_Test) {
ASSERT_EQ(numOfFields, 0); ASSERT_EQ(numOfFields, 0);
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); // pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); // printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
} // }
//
pRes = taos_query(pConn, "use abc1"); // pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes); // taos_free_result(pRes);
pRes = taos_query(pConn, "drop stable `123_$^)`"); // pRes = taos_query(pConn, "drop stable `123_$^)`");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); // printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes));
} // }
taos_close(pConn); taos_close(pConn);
} }
@ -333,7 +335,7 @@ TEST(testCase, create_ctable_Test) {
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table tu using sts tags('2021-10-10 1:1:1');"); pRes = taos_query(pConn, "create table tu using st1 tags('2021-10-10 1:1:1');");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes));
} }
@ -483,7 +485,9 @@ TEST(testCase, show_table_Test) {
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "show abc1.tables"); taos_query(pConn, "use abc1");
pRes = taos_query(pConn, "show tables");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); printf("failed to show tables, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes); taos_free_result(pRes);
@ -656,13 +660,7 @@ TEST(testCase, agg_query_tables) {
TAOS_RES* pRes = taos_query(pConn, "use abc1"); TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table tx using st1 tags(111111111111111)"); pRes = taos_query(pConn, "select count(*), sum(k),min(k),max(k) from tu");
if (taos_errno(pRes) != 0) {
printf("failed to create table, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "select count(*) from t_x_19");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes); taos_free_result(pRes);

468
source/common/src/tdatablock.c
View File

@ -18,14 +18,14 @@
#include "tcompare.h" #include "tcompare.h"
#include "tglobal.h" #include "tglobal.h"
int32_t taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = 0; pEp->port = 0;
strcpy(pEp->fqdn, ep); strcpy(pEp->fqdn, ep);
char *temp = strchr(pEp->fqdn, ':'); char* temp = strchr(pEp->fqdn, ':');
if (temp) { if (temp) {
*temp = 0; *temp = 0;
pEp->port = atoi(temp+1); pEp->port = atoi(temp + 1);
} }
if (pEp->port == 0) { if (pEp->port == 0) {
@ -36,7 +36,7 @@ int32_t taosGetFqdnPortFromEp(const char *ep, SEp* pEp) {
return 0; return 0;
} }
void addEpIntoEpSet(SEpSet *pEpSet, const char* fqdn, uint16_t port) { void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) {
if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) { if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) {
return; return;
} }
@ -47,26 +47,25 @@ void addEpIntoEpSet(SEpSet *pEpSet, const char* fqdn, uint16_t port) {
pEpSet->numOfEps += 1; pEpSet->numOfEps += 1;
} }
bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) { bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) {
if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) {
return false; return false;
} }
for (int32_t i = 0; i < s1->numOfEps; i++) { for (int32_t i = 0; i < s1->numOfEps; i++) {
if (s1->eps[i].port != s2->eps[i].port if (s1->eps[i].port != s2->eps[i].port || strncmp(s1->eps[i].fqdn, s2->eps[i].fqdn, TSDB_FQDN_LEN) != 0)
|| strncmp(s1->eps[i].fqdn, s2->eps[i].fqdn, TSDB_FQDN_LEN) != 0)
return false; return false;
} }
return true; return true;
} }
void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet) { void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet) {
taosCorBeginWrite(&pEpSet->version); taosCorBeginWrite(&pEpSet->version);
pEpSet->epSet = *pNewEpSet; pEpSet->epSet = *pNewEpSet;
taosCorEndWrite(&pEpSet->version); taosCorEndWrite(&pEpSet->version);
} }
SEpSet getEpSet_s(SCorEpSet *pEpSet) { SEpSet getEpSet_s(SCorEpSet* pEpSet) {
SEpSet ep = {0}; SEpSet ep = {0};
taosCorBeginRead(&pEpSet->version); taosCorBeginRead(&pEpSet->version);
ep = pEpSet->epSet; ep = pEpSet->epSet;
@ -75,7 +74,6 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) {
return ep; return ep;
} }
int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {
ASSERT(pColumnInfoData != NULL); ASSERT(pColumnInfoData != NULL);
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
@ -95,7 +93,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
if (isNull) { if (isNull) {
// There is a placehold for each NULL value of binary or nchar type. // There is a placehold for each NULL value of binary or nchar type.
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type.
} else { } else {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow);
} }
@ -113,7 +111,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
newSize = 8; newSize = 8;
} }
while(newSize < pAttr->length + varDataTLen(pData)) { while (newSize < pAttr->length + varDataTLen(pData)) {
newSize = newSize * 1.5; newSize = newSize * 1.5;
} }
@ -133,19 +131,40 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
pColumnInfoData->varmeta.length += varDataTLen(pData); pColumnInfoData->varmeta.length += varDataTLen(pData);
} else { } else {
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
switch(type) { switch (type) {
case TSDB_DATA_TYPE_BOOL: {*(bool*) p = *(bool*) pData;break;} case TSDB_DATA_TYPE_BOOL: {
*(bool*)p = *(bool*)pData;
break;
}
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT: {*(int8_t*) p = *(int8_t*) pData;break;} case TSDB_DATA_TYPE_UTINYINT: {
*(int8_t*)p = *(int8_t*)pData;
break;
}
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_USMALLINT: {*(int16_t*) p = *(int16_t*) pData;break;} case TSDB_DATA_TYPE_USMALLINT: {
*(int16_t*)p = *(int16_t*)pData;
break;
}
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT: {*(int32_t*) p = *(int32_t*) pData;break;} case TSDB_DATA_TYPE_UINT: {
*(int32_t*)p = *(int32_t*)pData;
break;
}
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT: {*(int64_t*) p = *(int64_t*) pData;break;} case TSDB_DATA_TYPE_UBIGINT: {
case TSDB_DATA_TYPE_FLOAT: {*(float*) p = *(float*) pData;break;} *(int64_t*)p = *(int64_t*)pData;
case TSDB_DATA_TYPE_DOUBLE: {*(double*) p = *(double*) pData;break;} break;
}
case TSDB_DATA_TYPE_FLOAT: {
*(float*)p = *(float*)pData;
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
*(double*)p = *(double*)pData;
break;
}
default: default:
assert(0); assert(0);
} }
@ -154,7 +173,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
return 0; return 0;
} }
static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource, int32_t numOfRow2) { static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource,
int32_t numOfRow2) {
uint32_t total = numOfRow1 + numOfRow2; uint32_t total = numOfRow1 + numOfRow2;
if (BitmapLen(numOfRow1) < BitmapLen(total)) { if (BitmapLen(numOfRow1) < BitmapLen(total)) {
@ -188,7 +208,8 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
} }
} }
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2) { int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource,
uint32_t numOfRow2) {
ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
if (numOfRow2 == 0) { if (numOfRow2 == 0) {
@ -202,8 +223,8 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
// TODO // TODO
} }
pColumnInfoData->varmeta.offset = (int32_t*) p; pColumnInfoData->varmeta.offset = (int32_t*)p;
for(int32_t i = 0; i < numOfRow2; ++i) { for (int32_t i = 0; i < numOfRow2; ++i) {
pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length; pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length;
} }
@ -239,14 +260,62 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
return numOfRow1 + numOfRow2; return numOfRow1 + numOfRow2;
} }
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) {
ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
if (numOfRows == 0) {
return numOfRows;
}
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
// Handle the bitmap
char* p = realloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumnInfoData->varmeta.offset = (int32_t*) p;
memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows);
if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) {
char* tmp = realloc(pColumnInfoData->pData, pSource->varmeta.length);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumnInfoData->pData = tmp;
pColumnInfoData->varmeta.allocLen = pSource->varmeta.length;
}
memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
pColumnInfoData->varmeta.length = pSource->varmeta.length;
} else {
char* tmp = realloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows));
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumnInfoData->nullbitmap = tmp;
memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows));
int32_t newSize = numOfRows * pColumnInfoData->info.bytes;
tmp = realloc(pColumnInfoData->pData, newSize);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumnInfoData->pData = tmp;
memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
}
return 0;
}
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) {
ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock)); ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock));
return pBlock->info.numOfCols; return pBlock->info.numOfCols;
} }
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; }
return pBlock->info.rows;
}
int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) {
if (pDataBlock == NULL || pDataBlock->info.rows <= 0) { if (pDataBlock == NULL || pDataBlock->info.rows <= 0) {
@ -263,8 +332,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) {
} }
ASSERT(pColInfoData->nullbitmap == NULL); ASSERT(pColInfoData->nullbitmap == NULL);
pDataBlock->info.window.skey = *(TSKEY*) colDataGetData(pColInfoData, 0); pDataBlock->info.window.skey = *(TSKEY*)colDataGetData(pColInfoData, 0);
pDataBlock->info.window.ekey = *(TSKEY*) colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); pDataBlock->info.window.ekey = *(TSKEY*)colDataGetData(pColInfoData, (pDataBlock->info.rows - 1));
return 0; return 0;
} }
@ -272,7 +341,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols);
int32_t numOfCols = pSrc->info.numOfCols; int32_t numOfCols = pSrc->info.numOfCols;
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
@ -280,7 +349,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows);
int32_t newSize = oldLen + newLen; int32_t newSize = oldLen + newLen;
char* tmp = realloc(pCol2->pData, newSize); char* tmp = realloc(pCol2->pData, newSize);
if (tmp != NULL) { if (tmp != NULL) {
pCol2->pData = tmp; pCol2->pData = tmp;
colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows); colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows);
@ -296,10 +365,10 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
size_t blockDataGetSize(const SSDataBlock* pBlock) { size_t blockDataGetSize(const SSDataBlock* pBlock) {
assert(pBlock != NULL); assert(pBlock != NULL);
size_t total = 0; size_t total = 0;
int32_t numOfCols = pBlock->info.numOfCols; int32_t numOfCols = pBlock->info.numOfCols;
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
total += colDataGetLength(pColInfoData, pBlock->info.rows); total += colDataGetLength(pColInfoData, pBlock->info.rows);
@ -315,7 +384,8 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) {
// the number of tuples can be fit in one page. // the number of tuples can be fit in one page.
// Actual data rows pluses the corresponding meta data must fit in one memory buffer of the given page size. // Actual data rows pluses the corresponding meta data must fit in one memory buffer of the given page size.
int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize) { int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex,
int32_t pageSize) {
ASSERT(pBlock != NULL && stopIndex != NULL); ASSERT(pBlock != NULL && stopIndex != NULL);
int32_t numOfCols = pBlock->info.numOfCols; int32_t numOfCols = pBlock->info.numOfCols;
@ -323,13 +393,13 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
int32_t bitmapChar = 1; int32_t bitmapChar = 1;
size_t headerSize = sizeof(int32_t); size_t headerSize = sizeof(int32_t);
size_t colHeaderSize = sizeof(int32_t) * numOfCols; size_t colHeaderSize = sizeof(int32_t) * numOfCols;
size_t payloadSize = pageSize - (headerSize + colHeaderSize); size_t payloadSize = pageSize - (headerSize + colHeaderSize);
// TODO speedup by checking if the whole page can fit in firstly. // TODO speedup by checking if the whole page can fit in firstly.
if (!hasVarCol) { if (!hasVarCol) {
size_t rowSize = blockDataGetRowSize(pBlock); size_t rowSize = blockDataGetRowSize(pBlock);
int32_t capacity = (payloadSize / (rowSize * 8 + bitmapChar * numOfCols)) * 8; int32_t capacity = (payloadSize / (rowSize * 8 + bitmapChar * numOfCols)) * 8;
*stopIndex = startIndex + capacity; *stopIndex = startIndex + capacity;
@ -342,7 +412,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
// iterate the rows that can be fit in this buffer page // iterate the rows that can be fit in this buffer page
int32_t size = (headerSize + colHeaderSize); int32_t size = (headerSize + colHeaderSize);
for(int32_t j = startIndex; j < numOfRows; ++j) { for (int32_t j = startIndex; j < numOfRows; ++j) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
@ -359,7 +429,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
size += pColInfoData->info.bytes; size += pColInfoData->info.bytes;
if (((j - startIndex) & 0x07) == 0) { if (((j - startIndex) & 0x07) == 0) {
size += 1; // the space for null bitmap size += 1; // the space for null bitmap
} }
} }
} }
@ -393,8 +463,8 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
pDst->info.rows = 0; pDst->info.rows = 0;
pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo = {0};
SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i);
colInfo.info = pSrcCol->info; colInfo.info = pSrcCol->info;
@ -414,7 +484,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) {
bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg); bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg);
char* p = colDataGetData(pColData, j); char* p = colDataGetData(pColData, j);
colDataAppend(pDstCol, j - startIndex, p, isNull); colDataAppend(pDstCol, j - startIndex, p, isNull);
@ -427,11 +497,11 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
/** /**
* *
* +------------------+---------------+--------------------+ * +------------------+---------------------------------------------+
* |the number of rows| column length | column #1 | * |the number of rows| column #1 |
* | (4 bytes) | (4 bytes) |--------------------+ * | (4 bytes) |------------+-----------------------+--------+
* | | | null bitmap| values| * | | null bitmap| column length(4bytes) | values |
* +------------------+---------------+--------------------+ * +------------------+------------+-----------------------+--------+
* @param buf * @param buf
* @param pBlock * @param pBlock
* @return * @return
@ -440,14 +510,14 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
ASSERT(pBlock != NULL); ASSERT(pBlock != NULL);
// write the number of rows // write the number of rows
*(uint32_t*) buf = pBlock->info.rows; *(uint32_t*)buf = pBlock->info.rows;
int32_t numOfCols = pBlock->info.numOfCols; int32_t numOfCols = pBlock->info.numOfCols;
int32_t numOfRows = pBlock->info.rows; int32_t numOfRows = pBlock->info.rows;
char* pStart = buf + sizeof(uint32_t); char* pStart = buf + sizeof(uint32_t);
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (IS_VAR_DATA_TYPE(pCol->info.type)) {
memcpy(pStart, pCol->varmeta.offset, numOfRows * sizeof(int32_t)); memcpy(pStart, pCol->varmeta.offset, numOfRows * sizeof(int32_t));
@ -459,7 +529,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
uint32_t dataSize = colDataGetLength(pCol, numOfRows); uint32_t dataSize = colDataGetLength(pCol, numOfRows);
*(int32_t*) pStart = dataSize; *(int32_t*)pStart = dataSize;
pStart += sizeof(int32_t); pStart += sizeof(int32_t);
memcpy(pStart, pCol->pData, dataSize); memcpy(pStart, pCol->pData, dataSize);
@ -470,12 +540,12 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
} }
int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
pBlock->info.rows = *(int32_t*) buf; pBlock->info.rows = *(int32_t*)buf;
int32_t numOfCols = pBlock->info.numOfCols; int32_t numOfCols = pBlock->info.numOfCols;
const char* pStart = buf + sizeof(uint32_t); const char* pStart = buf + sizeof(uint32_t);
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
size_t metaSize = pBlock->info.rows * sizeof(int32_t); size_t metaSize = pBlock->info.rows * sizeof(int32_t);
@ -487,7 +557,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
pStart += BitmapLen(pBlock->info.rows); pStart += BitmapLen(pBlock->info.rows);
} }
int32_t colLength = *(int32_t*) pStart; int32_t colLength = *(int32_t*)pStart;
pStart += sizeof(int32_t); pStart += sizeof(int32_t);
if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (IS_VAR_DATA_TYPE(pCol->info.type)) {
@ -512,17 +582,21 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
size_t blockDataGetRowSize(const SSDataBlock* pBlock) { size_t blockDataGetRowSize(SSDataBlock* pBlock) {
ASSERT(pBlock != NULL); ASSERT(pBlock != NULL);
size_t rowSize = 0; if (pBlock->info.rowSize == 0) {
size_t rowSize = 0;
size_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = pBlock->info.numOfCols;
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
rowSize += pColInfo->info.bytes; rowSize += pColInfo->info.bytes;
} }
return rowSize; pBlock->info.rowSize = rowSize;
}
return pBlock->info.rowSize;
} }
/** /**
@ -537,10 +611,10 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) { SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) {
SSchema* pSchema = calloc(pBlock->info.numOfCols, sizeof(SSchema)); SSchema* pSchema = calloc(pBlock->info.numOfCols, sizeof(SSchema));
for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
pSchema[i].bytes = pColInfoData->info.bytes; pSchema[i].bytes = pColInfoData->info.bytes;
pSchema[i].type = pColInfoData->info.type; pSchema[i].type = pColInfoData->info.type;
pSchema[i].colId = pColInfoData->info.colId; pSchema[i].colId = pColInfoData->info.colId;
} }
@ -556,14 +630,14 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
double rowSize = 0; double rowSize = 0;
size_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = pBlock->info.numOfCols;
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
rowSize += pColInfo->info.bytes; rowSize += pColInfo->info.bytes;
if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
rowSize += sizeof(int32_t); rowSize += sizeof(int32_t);
} else { } else {
rowSize += 1/8.0; rowSize += 1/8.0; // one bit for each record
} }
} }
@ -571,56 +645,56 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
} }
typedef struct SSDataBlockSortHelper { typedef struct SSDataBlockSortHelper {
SArray *orderInfo; // SArray<SBlockOrderInfo> SArray* orderInfo; // SArray<SBlockOrderInfo>
SSDataBlock *pDataBlock; SSDataBlock* pDataBlock;
bool nullFirst; bool nullFirst;
} SSDataBlockSortHelper; } SSDataBlockSortHelper;
int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*) param; const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param;
SSDataBlock* pDataBlock = pHelper->pDataBlock; SSDataBlock* pDataBlock = pHelper->pDataBlock;
int32_t left = *(int32_t*) p1; int32_t left = *(int32_t*)p1;
int32_t right = *(int32_t*) p2; int32_t right = *(int32_t*)p2;
SArray* pInfo = pHelper->orderInfo; SArray* pInfo = pHelper->orderInfo;
for(int32_t i = 0; i < pInfo->size; ++i) { for (int32_t i = 0; i < pInfo->size; ++i) {
SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i);
SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); SColumnInfoData* pColInfoData = pOrder->pColData; // TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex);
if (pColInfoData->hasNull) { if (pColInfoData->hasNull) {
bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg);
bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg);
if (leftNull && rightNull) { if (leftNull && rightNull) {
continue; // continue to next slot continue; // continue to next slot
} }
if (rightNull) { if (rightNull) {
return pHelper->nullFirst? 1:-1; return pHelper->nullFirst ? 1 : -1;
} }
if (leftNull) { if (leftNull) {
return pHelper->nullFirst? -1:1; return pHelper->nullFirst ? -1 : 1;
} }
} }
void* left1 = colDataGetData(pColInfoData, left); void* left1 = colDataGetData(pColInfoData, left);
void* right1 = colDataGetData(pColInfoData, right); void* right1 = colDataGetData(pColInfoData, right);
switch(pColInfoData->info.type) { switch (pColInfoData->info.type) {
case TSDB_DATA_TYPE_INT: { case TSDB_DATA_TYPE_INT: {
int32_t leftx = *(int32_t*) left1; int32_t leftx = *(int32_t*)left1;
int32_t rightx = *(int32_t*) right1; int32_t rightx = *(int32_t*)right1;
if (leftx == rightx) { if (leftx == rightx) {
break; break;
} else { } else {
if (pOrder->order == TSDB_ORDER_ASC) { if (pOrder->order == TSDB_ORDER_ASC) {
return (leftx < rightx)? -1:1; return (leftx < rightx) ? -1 : 1;
} else { } else {
return (leftx < rightx)? 1:-1; return (leftx < rightx) ? 1 : -1;
} }
} }
} }
@ -632,7 +706,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
return 0; return 0;
} }
static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, int32_t tupleIndex) { static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock,
int32_t tupleIndex) {
int32_t code = 0; int32_t code = 0;
int32_t numOfCols = pSrcBlock->info.numOfCols; int32_t numOfCols = pSrcBlock->info.numOfCols;
@ -666,17 +741,17 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
} }
} }
#else #else
for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pDst = &pCols[i];
SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pSrc->info.type)) { if (IS_VAR_DATA_TYPE(pSrc->info.type)) {
memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length); memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length);
pDst->varmeta.length = pSrc->varmeta.length; pDst->varmeta.length = pSrc->varmeta.length;
for(int32_t j = 0; j < pDataBlock->info.rows; ++j) { for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]]; pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]];
} }
} else { } else {
switch (pSrc->info.type) { switch (pSrc->info.type) {
case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UINT:
@ -749,7 +824,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) {
return NULL; return NULL;
} }
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i);
pCols[i].info = pColInfoData->info; pCols[i].info = pColInfoData->info;
@ -771,7 +846,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) {
static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) {
int32_t numOfCols = pDataBlock->info.numOfCols; int32_t numOfCols = pDataBlock->info.numOfCols;
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i);
pColInfoData->info = pCols[i].info; pColInfoData->info = pCols[i].info;
@ -796,31 +871,39 @@ static int32_t* createTupleIndex(size_t rows) {
return NULL; return NULL;
} }
for(int32_t i = 0; i < rows; ++i) { for (int32_t i = 0; i < rows; ++i) {
index[i] = i; index[i] = i;
} }
return index; return index;
} }
static void destroyTupleIndex(int32_t* index) { static void destroyTupleIndex(int32_t* index) { tfree(index); }
tfree(index);
}
static __compar_fn_t getComparFn(int32_t type, int32_t order) { static __compar_fn_t getComparFn(int32_t type, int32_t order) {
switch(type) { switch (type) {
case TSDB_DATA_TYPE_TINYINT: return order == TSDB_ORDER_ASC? compareInt8Val:compareInt8ValDesc; case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT: return order == TSDB_ORDER_ASC? compareInt16Val:compareInt16ValDesc; return order == TSDB_ORDER_ASC ? compareInt8Val : compareInt8ValDesc;
case TSDB_DATA_TYPE_INT: return order == TSDB_ORDER_ASC? compareInt32Val:compareInt32ValDesc; case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_BIGINT: return order == TSDB_ORDER_ASC? compareInt64Val:compareInt64ValDesc; return order == TSDB_ORDER_ASC ? compareInt16Val : compareInt16ValDesc;
case TSDB_DATA_TYPE_FLOAT: return order == TSDB_ORDER_ASC? compareFloatVal:compareFloatValDesc; case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_DOUBLE: return order == TSDB_ORDER_ASC? compareDoubleVal:compareDoubleValDesc; return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc;
case TSDB_DATA_TYPE_UTINYINT: return order == TSDB_ORDER_ASC? compareUint8Val:compareUint8ValDesc; case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_USMALLINT:return order == TSDB_ORDER_ASC? compareUint16Val:compareUint16ValDesc; return order == TSDB_ORDER_ASC ? compareInt64Val : compareInt64ValDesc;
case TSDB_DATA_TYPE_UINT: return order == TSDB_ORDER_ASC? compareUint32Val:compareUint32ValDesc; case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_UBIGINT: return order == TSDB_ORDER_ASC? compareUint64Val:compareUint64ValDesc; return order == TSDB_ORDER_ASC ? compareFloatVal : compareFloatValDesc;
case TSDB_DATA_TYPE_DOUBLE:
return order == TSDB_ORDER_ASC ? compareDoubleVal : compareDoubleValDesc;
case TSDB_DATA_TYPE_UTINYINT:
return order == TSDB_ORDER_ASC ? compareUint8Val : compareUint8ValDesc;
case TSDB_DATA_TYPE_USMALLINT:
return order == TSDB_ORDER_ASC ? compareUint16Val : compareUint16ValDesc;
case TSDB_DATA_TYPE_UINT:
return order == TSDB_ORDER_ASC ? compareUint32Val : compareUint32ValDesc;
case TSDB_DATA_TYPE_UBIGINT:
return order == TSDB_ORDER_ASC ? compareUint64Val : compareUint64ValDesc;
default: default:
return order == TSDB_ORDER_ASC? compareInt32Val:compareInt32ValDesc; return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc;
} }
} }
@ -865,10 +948,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { // var data type } else { // var data type
} }
} else if (pDataBlock->info.numOfCols == 2) { } else if (pDataBlock->info.numOfCols == 2) {
} }
} }
@ -881,7 +962,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs
int64_t p0 = taosGetTimestampUs(); int64_t p0 = taosGetTimestampUs();
SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
for(int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) { for (int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) {
struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i); struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i);
pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex); pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex);
} }
@ -909,7 +990,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs
copyBackToBlock(pDataBlock, pCols); copyBackToBlock(pDataBlock, pCols);
int64_t p4 = taosGetTimestampUs(); int64_t p4 = taosGetTimestampUs();
printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1-p0, p2 - p1, p3 - p2, p4-p3, rows); printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1,
p3 - p2, p4 - p3, rows);
destroyTupleIndex(index); destroyTupleIndex(index);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -917,14 +999,18 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs
typedef struct SHelper { typedef struct SHelper {
int32_t index; int32_t index;
union {char *pData; int64_t i64; double d64;}; union {
char* pData;
int64_t i64;
double d64;
};
} SHelper; } SHelper;
SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* pBlock) { SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* pBlock) {
int32_t sortValLengthPerRow = 0; int32_t sortValLengthPerRow = 0;
int32_t numOfCols = taosArrayGetSize(pOrderInfo); int32_t numOfCols = taosArrayGetSize(pOrderInfo);
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex);
pInfo->pColData = pColInfo; pInfo->pColData = pColInfo;
@ -933,19 +1019,20 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock*
size_t len = sortValLengthPerRow * pBlock->info.rows; size_t len = sortValLengthPerRow * pBlock->info.rows;
char* buf = calloc(1, len); char* buf = calloc(1, len);
SHelper* phelper = calloc(numOfRows, sizeof(SHelper)); SHelper* phelper = calloc(numOfRows, sizeof(SHelper));
for(int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
phelper[i].index = i; phelper[i].index = i;
phelper[i].pData = buf + sortValLengthPerRow * i; phelper[i].pData = buf + sortValLengthPerRow * i;
} }
int32_t offset = 0; int32_t offset = 0;
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i);
for(int32_t j = 0; j < numOfRows; ++j) { for (int32_t j = 0; j < numOfRows; ++j) {
phelper[j].i64 = *(int32_t*) pInfo->pColData->pData + pInfo->pColData->info.bytes * j; phelper[j].i64 = *(int32_t*)pInfo->pColData->pData + pInfo->pColData->info.bytes * j;
// memcpy(phelper[j].pData + offset, pInfo->pColData->pData + pInfo->pColData->info.bytes * j, pInfo->pColData->info.bytes); // memcpy(phelper[j].pData + offset, pInfo->pColData->pData + pInfo->pColData->info.bytes * j,
// pInfo->pColData->info.bytes);
} }
offset += pInfo->pColData->info.bytes; offset += pInfo->pColData->info.bytes;
@ -955,70 +1042,68 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock*
} }
int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) {
const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*) param; const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param;
// SSDataBlock* pDataBlock = pHelper->pDataBlock; // SSDataBlock* pDataBlock = pHelper->pDataBlock;
SHelper* left = (SHelper*) p1; SHelper* left = (SHelper*)p1;
SHelper* right = (SHelper*) p2; SHelper* right = (SHelper*)p2;
SArray* pInfo = pHelper->orderInfo; SArray* pInfo = pHelper->orderInfo;
int32_t offset = 0; int32_t offset = 0;
// for(int32_t i = 0; i < pInfo->size; ++i) { // for(int32_t i = 0; i < pInfo->size; ++i) {
// SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0); // SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0);
// SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); // SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex);
// if (pColInfoData->hasNull) { // if (pColInfoData->hasNull) {
// bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); // bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg);
// bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); // bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg);
// if (leftNull && rightNull) { // if (leftNull && rightNull) {
// continue; // continue to next slot // continue; // continue to next slot
// } // }
// //
// if (rightNull) { // if (rightNull) {
// return pHelper->nullFirst? 1:-1; // return pHelper->nullFirst? 1:-1;
// } // }
// //
// if (leftNull) { // if (leftNull) {
// return pHelper->nullFirst? -1:1; // return pHelper->nullFirst? -1:1;
// } // }
// } // }
// void* left1 = colDataGetData(pColInfoData, left); // void* left1 = colDataGetData(pColInfoData, left);
// void* right1 = colDataGetData(pColInfoData, right); // void* right1 = colDataGetData(pColInfoData, right);
// switch(pColInfoData->info.type) { // switch(pColInfoData->info.type) {
// case TSDB_DATA_TYPE_INT: { // case TSDB_DATA_TYPE_INT: {
int32_t leftx = *(int32_t*)left->pData;//*(int32_t*)(left->pData + offset); int32_t leftx = *(int32_t*)left->pData; //*(int32_t*)(left->pData + offset);
int32_t rightx = *(int32_t*)right->pData;//*(int32_t*)(right->pData + offset); int32_t rightx = *(int32_t*)right->pData; //*(int32_t*)(right->pData + offset);
// offset += pColInfoData->info.bytes; // offset += pColInfoData->info.bytes;
if (leftx == rightx) { if (leftx == rightx) {
// break; // break;
return 0; return 0;
} else { } else {
// if (pOrder->order == TSDB_ORDER_ASC) { // if (pOrder->order == TSDB_ORDER_ASC) {
return (leftx < rightx)? -1:1; return (leftx < rightx) ? -1 : 1;
// } else { // } else {
// return (leftx < rightx)? 1:-1; // return (leftx < rightx)? 1:-1;
// } // }
} }
// } // }
// default: // default:
// assert(0); // assert(0);
// } // }
// } // }
return 0; return 0;
} }
int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { return 0; }
return 0;
}
int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) {
// Allocate the additional buffer. // Allocate the additional buffer.
int64_t p0 = taosGetTimestampUs(); int64_t p0 = taosGetTimestampUs();
SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
@ -1032,7 +1117,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
taosqsort(index, rows, sizeof(SHelper), &helper, dataBlockCompar_rv); taosqsort(index, rows, sizeof(SHelper), &helper, dataBlockCompar_rv);
int64_t p1 = taosGetTimestampUs(); int64_t p1 = taosGetTimestampUs();
SColumnInfoData* pCols = createHelpColInfoData(pDataBlock); SColumnInfoData* pCols = createHelpColInfoData(pDataBlock);
if (pCols == NULL) { if (pCols == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -1052,12 +1137,13 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
copyBackToBlock(pDataBlock, pCols); copyBackToBlock(pDataBlock, pCols);
int64_t p4 = taosGetTimestampUs(); int64_t p4 = taosGetTimestampUs();
printf("sort:%" PRId64 ", create:%" PRId64", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1,
p3 - p2, p4 - p3, rows);
// destroyTupleIndex(index); // destroyTupleIndex(index);
return 0; return 0;
} }
void blockDataClearup(SSDataBlock* pDataBlock) { void blockDataCleanup(SSDataBlock* pDataBlock) {
pDataBlock->info.rows = 0; pDataBlock->info.rows = 0;
if (pDataBlock->info.hasVarCol) { if (pDataBlock->info.hasVarCol) {
@ -1110,8 +1196,8 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
int32_t code = 0; int32_t code = 0;
for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
code = blockDataEnsureColumnCapacity(p, numOfRows); code = blockDataEnsureColumnCapacity(p, numOfRows);
if (code) { if (code) {
@ -1127,21 +1213,8 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
return NULL; return NULL;
} }
int32_t numOfOutput = pBlock->info.numOfCols; blockDestroyInner(pBlock);
for(int32_t i = 0; i < numOfOutput; ++i) { tfree(pBlock);
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
tfree(pColInfoData->varmeta.offset);
} else {
tfree(pColInfoData->nullbitmap);
}
tfree(pColInfoData->pData);
}
taosArrayDestroy(pBlock->pDataBlock);
tfree(pBlock->pBlockAgg);
// tfree(pBlock);
return NULL; return NULL;
} }
@ -1154,7 +1227,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) {
pBlock->info.numOfCols = numOfCols; pBlock->info.numOfCols = numOfCols;
pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
for(int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo = {0};
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
colInfo.info = p->info; colInfo.info = p->info;
@ -1230,4 +1303,27 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
taosArrayPush(pBlock->pDataBlock, &data); taosArrayPush(pBlock->pDataBlock, &data);
} }
return (void*)buf; return (void*)buf;
} }
int32_t tEncodeDataBlocks(void** buf, const SArray* blocks) {
int32_t tlen = 0;
int32_t sz = taosArrayGetSize(blocks);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pBlock = taosArrayGet(blocks, i);
tlen += tEncodeDataBlock(buf, pBlock);
}
return tlen;
}
void* tDecodeDataBlocks(const void* buf, SArray* blocks) {
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock pBlock = {0};
buf = tDecodeDataBlock(buf, &pBlock);
}
return (void*)buf;
}

3
source/common/src/tdataformat.c
View File

@ -421,6 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
return NULL; return NULL;
} }
#if 0
SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints);
if (pRet == NULL) return NULL; if (pRet == NULL) return NULL;
@ -431,6 +432,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
for (int i = 0; i < pDataCols->numOfCols; i++) { for (int i = 0; i < pDataCols->numOfCols; i++) {
pRet->cols[i].type = pDataCols->cols[i].type; pRet->cols[i].type = pDataCols->cols[i].type;
pRet->cols[i].bitmap = pDataCols->cols[i].bitmap;
pRet->cols[i].colId = pDataCols->cols[i].colId; pRet->cols[i].colId = pDataCols->cols[i].colId;
pRet->cols[i].bytes = pDataCols->cols[i].bytes; pRet->cols[i].bytes = pDataCols->cols[i].bytes;
pRet->cols[i].offset = pDataCols->cols[i].offset; pRet->cols[i].offset = pDataCols->cols[i].offset;
@ -453,6 +455,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
return pRet; return pRet;
} }
#endif
void tdResetDataCols(SDataCols *pCols) { void tdResetDataCols(SDataCols *pCols) {
if (pCols != NULL) { if (pCols != NULL) {

5
source/common/src/tglobal.c
View File

@ -132,6 +132,9 @@ bool tsdbForceKeepFile = false;
int32_t tsDiskCfgNum = 0; int32_t tsDiskCfgNum = 0;
SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0};
// stream scheduler
bool tsStreamSchedV = true;
/* /*
* minimum scale for whole system, millisecond by default * minimum scale for whole system, millisecond by default
* for TSDB_TIME_PRECISION_MILLI: 86400000L * for TSDB_TIME_PRECISION_MILLI: 86400000L
@ -585,4 +588,4 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
taosResetLog(); taosResetLog();
cfgDumpCfg(tsCfg, 1, false); cfgDumpCfg(tsCfg, 1, false);
} }
} }

183
source/common/src/tmsg.c
View File

@ -290,7 +290,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->ttl);
tlen += taosEncodeFixedU32(buf, pReq->keep); tlen += taosEncodeFixedU32(buf, pReq->keep);
tlen += taosEncodeFixedU8(buf, pReq->type); tlen += taosEncodeFixedU8(buf, pReq->info);
switch (pReq->type) { switch (pReq->type) {
case TD_SUPER_TABLE: case TD_SUPER_TABLE:
@ -309,6 +309,20 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
} }
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]);
}
if (pReq->rollup && NULL != pReq->stbCfg.pRSmaParam) {
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
tlen += taosEncodeFixedI8(buf, param->delayUnit);
tlen += taosEncodeFixedI8(buf, param->nFuncIds);
for (int8_t i = 0; i < param->nFuncIds; ++i) {
tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
}
tlen += taosEncodeFixedI64(buf, param->delay);
}
break; break;
case TD_CHILD_TABLE: case TD_CHILD_TABLE:
tlen += taosEncodeFixedI64(buf, pReq->ctbCfg.suid); tlen += taosEncodeFixedI64(buf, pReq->ctbCfg.suid);
@ -322,6 +336,20 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
} }
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]);
}
if (pReq->rollup && NULL != pReq->stbCfg.pRSmaParam) {
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
tlen += taosEncodeFixedI8(buf, param->delayUnit);
tlen += taosEncodeFixedI8(buf, param->nFuncIds);
for (int8_t i = 0; i < param->nFuncIds; ++i) {
tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
}
tlen += taosEncodeFixedI64(buf, param->delay);
}
break; break;
default: default:
ASSERT(0); ASSERT(0);
@ -335,7 +363,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeString(buf, &(pReq->name));
buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->ttl));
buf = taosDecodeFixedU32(buf, &(pReq->keep)); buf = taosDecodeFixedU32(buf, &(pReq->keep));
buf = taosDecodeFixedU8(buf, &(pReq->type)); buf = taosDecodeFixedU8(buf, &(pReq->info));
switch (pReq->type) { switch (pReq->type) {
case TD_SUPER_TABLE: case TD_SUPER_TABLE:
@ -356,6 +384,32 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
} }
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
if (pReq->stbCfg.nBSmaCols > 0) {
pReq->stbCfg.pBSmaCols = (col_id_t *)malloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t));
for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i);
}
} else {
pReq->stbCfg.pBSmaCols = NULL;
}
if (pReq->rollup) {
pReq->stbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam));
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
buf = taosDecodeFixedU32(buf, (uint32_t *)&param->xFilesFactor);
buf = taosDecodeFixedI8(buf, &param->delayUnit);
buf = taosDecodeFixedI8(buf, &param->nFuncIds);
if (param->nFuncIds > 0) {
for (int8_t i = 0; i < param->nFuncIds; ++i) {
buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
}
} else {
param->pFuncIds = NULL;
}
buf = taosDecodeFixedI64(buf, &param->delay);
} else {
pReq->stbCfg.pRSmaParam = NULL;
}
break; break;
case TD_CHILD_TABLE: case TD_CHILD_TABLE:
buf = taosDecodeFixedI64(buf, &pReq->ctbCfg.suid); buf = taosDecodeFixedI64(buf, &pReq->ctbCfg.suid);
@ -370,6 +424,32 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
} }
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
if (pReq->stbCfg.nBSmaCols > 0) {
pReq->stbCfg.pBSmaCols = (col_id_t *)malloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t));
for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i);
}
} else {
pReq->stbCfg.pBSmaCols = NULL;
}
if (pReq->rollup) {
pReq->stbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam));
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
buf = taosDecodeFixedU32(buf, (uint32_t *)&param->xFilesFactor);
buf = taosDecodeFixedI8(buf, &param->delayUnit);
buf = taosDecodeFixedI8(buf, &param->nFuncIds);
if (param->nFuncIds > 0) {
for (int8_t i = 0; i < param->nFuncIds; ++i) {
buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
}
} else {
param->pFuncIds = NULL;
}
buf = taosDecodeFixedI64(buf, &param->delay);
} else {
pReq->stbCfg.pRSmaParam = NULL;
}
break; break;
default: default:
ASSERT(0); ASSERT(0);
@ -1733,7 +1813,10 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
if (tStartEncode(&encoder) < 0) return -1; if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; if (tEncodeI64(&encoder, pReq->showId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->type) < 0) return -1;
if (tEncodeI8(&encoder, pReq->free) < 0) return -1; if (tEncodeI8(&encoder, pReq->free) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -1747,8 +1830,10 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
if (tStartDecode(&decoder) < 0) return -1; if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->free) < 0) return -1; if (tDecodeI8(&decoder, &pReq->free) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
tEndDecode(&decoder); tEndDecode(&decoder);
tCoderClear(&decoder); tCoderClear(&decoder);
return 0; return 0;
@ -2619,6 +2704,77 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *
void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); } void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); }
int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->code) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp) {
SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1;
tEndDecode(&decoder);
tCoderClear(&decoder);
return 0;
}
int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (pRsp->rspList) {
int32_t num = taosArrayGetSize(pRsp->rspList);
if (tEncodeI32(&encoder, num) < 0) return -1;
for (int32_t i = 0; i < num; ++i) {
SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
}
} else {
if (tEncodeI32(&encoder, 0) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
SCoder decoder = {0};
int32_t num = 0;
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &num) < 0) return -1;
if (num > 0) {
pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp));
if (NULL == pRsp->rspList) return -1;
for (int32_t i = 0; i < num; ++i) {
SVCreateTbRsp rsp = {0};
if (tDecodeI32(&decoder, &rsp.code) < 0) return -1;
if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1;
}
} else {
pRsp->rspList = NULL;
}
tEndDecode(&decoder);
tCoderClear(&decoder);
return 0;
}
int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) { int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) {
int32_t tlen = 0; int32_t tlen = 0;
@ -2667,7 +2823,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->outputTbName) < 0) return -1; if (tEncodeCStr(&encoder, pReq->outputTbName) < 0) return -1;
if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
if (tEncodeI32(&encoder, astLen) < 0) return -1;
if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
@ -2715,26 +2872,36 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
} }
int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) { int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) {
if (tStartEncode(pEncoder) < 0) return -1; /*if (tStartEncode(pEncoder) < 0) return -1;*/
if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->level) < 0) return -1; if (tEncodeI32(pEncoder, pTask->level) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->pipeSource) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->pipeSink) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->parallelizable) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->nextOpDst) < 0) return -1;
// if (tEncodeI8(pEncoder, pTask->numOfRunners) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pTask->NextOpEp) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->NextOpEp) < 0) return -1;
if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1;
tEndEncode(pEncoder); /*tEndEncode(pEncoder);*/
return pEncoder->pos; return pEncoder->pos;
} }
int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) { int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) {
if (tStartDecode(pDecoder) < 0) return -1; /*if (tStartDecode(pDecoder) < 0) return -1;*/
if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->pipeSource) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->pipeSink) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->parallelizable) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->nextOpDst) < 0) return -1;
// if (tDecodeI8(pDecoder, &pTask->numOfRunners) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &pTask->NextOpEp) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->NextOpEp) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1;
tEndDecode(pDecoder); /*tEndDecode(pDecoder);*/
return 0; return 0;
} }

35
source/common/src/tmsgcb.c Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tmsgcb.h"
int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) {
return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq);
}
int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) {
return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype);
}
int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq) {
return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq);
}
int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq) {
return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq);
}
void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); }

30
source/common/src/tname.c
View File

@ -222,6 +222,27 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId) {
return 0; return 0;
} }
bool tNameDBNameEqual(SName* left, SName* right) {
if (NULL == left) {
if (NULL == right) {
return true;
}
return false;
}
if (NULL == right) {
return false;
}
if (left->acctId != right->acctId) {
return false;
}
return (0 == strcmp(left->dbname, right->dbname));
}
int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { int32_t tNameFromString(SName* dst, const char* str, uint32_t type) {
assert(dst != NULL && str != NULL && strlen(str) > 0); assert(dst != NULL && str != NULL && strlen(str) > 0);
@ -273,13 +294,4 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) {
return 0; return 0;
} }
SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name) {
SSchema s = {0};
s.type = type;
s.bytes = bytes;
s.colId = colId;
tstrncpy(s.name, name, tListLen(s.name));
return s;
}

52
source/common/src/trow.c
View File

@ -78,7 +78,7 @@ static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBit
setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes); setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);
pCol->len += TYPE_BYTES[pCol->type]; pCol->len += TYPE_BYTES[pCol->type];
} }
if(setBitmap) { if (setBitmap) {
tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE); tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE);
} }
} }
@ -118,8 +118,8 @@ int trbWriteCol(SRowBuilder *pRB, void *pData, col_id_t cid) {
#endif #endif
STSRow* tdRowDup(STSRow *row) { STSRow *tdRowDup(STSRow *row) {
STSRow* trow = malloc(TD_ROW_LEN(row)); STSRow *trow = malloc(TD_ROW_LEN(row));
if (trow == NULL) return NULL; if (trow == NULL) return NULL;
tdRowCpy(trow, row); tdRowCpy(trow, row);
@ -176,7 +176,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
SDataCol *pDataCol = &(pCols->cols[0]); SDataCol *pDataCol = &(pCols->cols[0]);
if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints); tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints);
} }
while (dcol < pCols->numOfCols) { while (dcol < pCols->numOfCols) {
@ -378,9 +378,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
} }
} }
STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) {
STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) {
#if 0 #if 0
ASSERT(TD_ROW_KEY(row1) == TD_ROW_KEY(row2)); ASSERT(TD_ROW_KEY(row1) == TD_ROW_KEY(row2));
ASSERT(schemaVersion(pSchema1) == TD_ROW_SVER(row1)); ASSERT(schemaVersion(pSchema1) == TD_ROW_SVER(row1));
@ -473,6 +471,44 @@ STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema
} }
taosArrayDestroy(stashRow); taosArrayDestroy(stashRow);
return buffer; return buffer;
#endif #endif
return NULL; return NULL;
} }
SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints);
if (pRet == NULL) return NULL;
pRet->numOfCols = pDataCols->numOfCols;
pRet->sversion = pDataCols->sversion;
if (keepData) pRet->numOfRows = pDataCols->numOfRows;
for (int i = 0; i < pDataCols->numOfCols; i++) {
pRet->cols[i].type = pDataCols->cols[i].type;
pRet->cols[i].bitmap = pDataCols->cols[i].bitmap;
pRet->cols[i].colId = pDataCols->cols[i].colId;
pRet->cols[i].bytes = pDataCols->cols[i].bytes;
pRet->cols[i].offset = pDataCols->cols[i].offset;
if (keepData) {
if (pDataCols->cols[i].len > 0) {
if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) {
tdFreeDataCols(pRet);
return NULL;
}
pRet->cols[i].len = pDataCols->cols[i].len;
memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints;
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
}
if (!TD_COL_ROWS_NORM(pRet->cols + i)) {
int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints);
memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes);
}
}
}
}
return pRet;
}

2
source/dnode/bnode/inc/bndInt.h
View File

@ -30,7 +30,7 @@ extern "C" {
#endif #endif
typedef struct SBnode { typedef struct SBnode {
SBnodeOpt opt; SMsgCb msgCb;
} SBnode; } SBnode;
#ifdef __cplusplus #ifdef __cplusplus

1
source/dnode/bnode/src/bnode.c
View File

@ -17,6 +17,7 @@
SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) { SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) {
SBnode *pBnode = calloc(1, sizeof(SBnode)); SBnode *pBnode = calloc(1, sizeof(SBnode));
pBnode->msgCb = pOption->msgCb;
return pBnode; return pBnode;
} }

2
source/dnode/mgmt/bnode/inc/bmInt.h
View File

@ -28,7 +28,7 @@ typedef struct SBnodeMgmt {
SDnode *pDnode; SDnode *pDnode;
SMgmtWrapper *pWrapper; SMgmtWrapper *pWrapper;
const char *path; const char *path;
SDnodeWorker writeWorker; SMultiWorker writeWorker;
} SBnodeMgmt; } SBnodeMgmt;
// bmInt.c // bmInt.c

13
source/dnode/mgmt/bnode/src/bmInt.c
View File

@ -19,13 +19,12 @@
static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) {
SDnode *pDnode = pMgmt->pDnode; SMsgCb msgCb = {0};
pOption->pWrapper = pMgmt->pWrapper; msgCb.pWrapper = pMgmt->pWrapper;
pOption->sendReqFp = dndSendReqToDnode; msgCb.sendReqFp = dndSendReqToDnode;
pOption->sendMnodeReqFp = dndSendReqToMnode; msgCb.sendMnodeReqFp = dndSendReqToMnode;
pOption->sendRspFp = dndSendRsp; msgCb.sendRspFp = dndSendRsp;
pOption->dnodeId = pDnode->dnodeId; pOption->msgCb = msgCb;
pOption->clusterId = pDnode->clusterId;
} }
static int32_t bmOpenImp(SBnodeMgmt *pMgmt) { static int32_t bmOpenImp(SBnodeMgmt *pMgmt) {

12
source/dnode/mgmt/bnode/src/bmWorker.c
View File

@ -33,7 +33,8 @@ static void bmSendErrorRsps(SMgmtWrapper *pWrapper, STaosQall *qall, int32_t num
} }
} }
static void bmProcessQueue(SBnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) { static void bmProcessQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SBnodeMgmt *pMgmt = pInfo->ahandle;
SMgmtWrapper *pWrapper = pMgmt->pWrapper; SMgmtWrapper *pWrapper = pMgmt->pWrapper;
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
@ -63,14 +64,15 @@ static void bmProcessQueue(SBnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs
} }
int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) {
SDnodeWorker *pWorker = &pMgmt->writeWorker; SMultiWorker *pWorker = &pMgmt->writeWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
return dndWriteMsgToWorker(pWorker, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
} }
int32_t bmStartWorker(SBnodeMgmt *pMgmt) { int32_t bmStartWorker(SBnodeMgmt *pMgmt) {
if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, bmProcessQueue) != 0) { SMultiWorkerCfg cfg = {.maxNum = 1, .name = "bnode-write", .fp = (FItems)bmProcessQueue, .param = pMgmt};
if (tMultiWorkerInit(&pMgmt->writeWorker, &cfg) != 0) {
dError("failed to start bnode write worker since %s", terrstr()); dError("failed to start bnode write worker since %s", terrstr());
return -1; return -1;
} }
@ -78,4 +80,4 @@ int32_t bmStartWorker(SBnodeMgmt *pMgmt) {
return 0; return 0;
} }
void bmStopWorker(SBnodeMgmt *pMgmt) { dndCleanupWorker(&pMgmt->writeWorker); } void bmStopWorker(SBnodeMgmt *pMgmt) { tMultiWorkerCleanup(&pMgmt->writeWorker); }

25
source/dnode/mgmt/container/inc/dnd.h
View File

@ -74,21 +74,10 @@ typedef int32_t (*CreateNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required); typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required);
typedef struct {
EWorkerType type;
const char *name;
int32_t minNum;
int32_t maxNum;
void *queueFp;
void *param;
STaosQueue *queue;
union {
SQWorkerPool pool;
SWWorkerPool mpool;
};
} SDnodeWorker;
typedef struct SMsgHandle { typedef struct SMsgHandle {
int32_t vgId;
NodeMsgFp vgIdMsgFp;
SMgmtWrapper *pVgIdWrapper; // Handle the case where the same message type is distributed to qnode or vnode
NodeMsgFp msgFp; NodeMsgFp msgFp;
SMgmtWrapper *pWrapper; SMgmtWrapper *pWrapper;
} SMsgHandle; } SMsgHandle;
@ -114,6 +103,7 @@ typedef struct SMgmtWrapper {
void *pMgmt; void *pMgmt;
SDnode *pDnode; SDnode *pDnode;
NodeMsgFp msgFps[TDMT_MAX]; NodeMsgFp msgFps[TDMT_MAX];
int32_t msgVgIds[TDMT_MAX]; // Handle the case where the same message type is distributed to qnode or vnode
SMgmtFp fp; SMgmtFp fp;
} SMgmtWrapper; } SMgmtWrapper;
@ -149,7 +139,7 @@ typedef struct SDnode {
EDndStatus dndGetStatus(SDnode *pDnode); EDndStatus dndGetStatus(SDnode *pDnode);
void dndSetStatus(SDnode *pDnode, EDndStatus stat); void dndSetStatus(SDnode *pDnode, EDndStatus stat);
SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType);
void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp); void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId);
void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc);
void dndSendMonitorReport(SDnode *pDnode); void dndSendMonitorReport(SDnode *pDnode);
@ -157,11 +147,6 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg); int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg);
void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp);
int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
int32_t maxNum, void *queueFp);
void dndCleanupWorker(SDnodeWorker *pWorker);
int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg);
int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);

3
source/dnode/mgmt/container/src/dndInt.c
View File

@ -65,8 +65,9 @@ void dndCleanup() {
dInfo("dnode env is cleaned up"); dInfo("dnode env is cleaned up");
} }
void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp) { void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId) {
pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp;
pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId;
} }
EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; }

54
source/dnode/mgmt/container/src/dndTransport.c
View File

@ -20,13 +20,26 @@
#define INTERNAL_CKEY "_key" #define INTERNAL_CKEY "_key"
#define INTERNAL_SECRET "_pwd" #define INTERNAL_SECRET "_pwd"
static inline void dndProcessQVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, SEpSet *pEpSet) {
SMsgHead *pHead = pMsg->pCont;
int32_t vgId = htonl(pHead->vgId);
SMgmtWrapper *pWrapper = pHandle->pWrapper;
if (vgId == pHandle->vgId && pHandle->pVgIdWrapper != NULL) {
pWrapper = pHandle->pVgIdWrapper;
}
dTrace("msg:%s will be processed by %s, handle:%p app:%p vgId:%d", TMSG_INFO(pMsg->msgType), pWrapper->name,
pMsg->handle, pMsg->ahandle, vgId);
dndProcessRpcMsg(pWrapper, pMsg, pEpSet);
}
static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
SDnode *pDnode = parent; SDnode *pDnode = parent;
STransMgmt *pMgmt = &pDnode->trans; STransMgmt *pMgmt = &pDnode->trans;
tmsg_t msgType = pRsp->msgType; tmsg_t msgType = pRsp->msgType;
if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
// if (pRsp == NULL || pRsp->pCont == NULL) return;
dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle);
rpcFreeCont(pRsp->pCont); rpcFreeCont(pRsp->pCont);
return; return;
@ -34,9 +47,13 @@ static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)];
if (pHandle->msgFp != NULL) { if (pHandle->msgFp != NULL) {
dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), if (pHandle->vgId == 0) {
pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType),
dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code));
dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet);
} else {
dndProcessQVnodeRpcMsg(pHandle, pRsp, pEpSet);
}
} else { } else {
dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle);
rpcFreeCont(pRsp->pCont); rpcFreeCont(pRsp->pCont);
@ -110,9 +127,13 @@ static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)];
if (pHandle->msgFp != NULL) { if (pHandle->msgFp != NULL) {
dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, if (pHandle->vgId == 0) {
pReq->handle, pReq->ahandle); dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name,
dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); pReq->handle, pReq->ahandle);
dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet);
} else {
dndProcessQVnodeRpcMsg(pHandle, pReq, pEpSet);
}
} else { } else {
dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle);
SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle};
@ -245,17 +266,24 @@ int32_t dndInitMsgHandle(SDnode *pDnode) {
for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) {
NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; NodeMsgFp msgFp = pWrapper->msgFps[msgIndex];
int32_t vgId = pWrapper->msgVgIds[msgIndex];
if (msgFp == NULL) continue; if (msgFp == NULL) continue;
SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex];
if (pHandle->msgFp != NULL) { if (pHandle->msgFp != NULL && pHandle->vgId == vgId) {
dError("msg:%s has multiple process nodes, prev node:%s, curr node:%s", tMsgInfo[msgIndex], dError("msg:%s has multiple process nodes, prev node:%s:%d, curr node:%s:%d", tMsgInfo[msgIndex],
pHandle->pWrapper->name, pWrapper->name); pHandle->pWrapper->name, pHandle->pWrapper->msgVgIds[msgIndex], pWrapper->name, vgId);
return -1; return -1;
} else { } else {
dTrace("msg:%s will be processed by %s", tMsgInfo[msgIndex], pWrapper->name); dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId);
pHandle->msgFp = msgFp; if (vgId == 0) {
pHandle->pWrapper = pWrapper; pHandle->msgFp = msgFp;
pHandle->pWrapper = pWrapper;
} else {
pHandle->vgId = vgId;
pHandle->vgIdMsgFp = msgFp;
pHandle->pVgIdWrapper = pWrapper;
}
} }
} }
} }

96
source/dnode/mgmt/container/src/dndWorker.c
View File

@ -1,96 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http:www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndInt.h"
int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
int32_t maxNum, void *queueFp) {
if (pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
}
pWorker->type = type;
pWorker->name = name;
pWorker->minNum = minNum;
pWorker->maxNum = maxNum;
pWorker->queueFp = queueFp;
pWorker->param = param;
if (pWorker->type == DND_WORKER_SINGLE) {
SQWorkerPool *pPool = &pWorker->pool;
pPool->name = name;
pPool->min = minNum;
pPool->max = maxNum;
if (tQWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pWorker->queue = tQWorkerAllocQueue(pPool, param, (FItem)queueFp);
if (pWorker->queue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} else if (pWorker->type == DND_WORKER_MULTI) {
SWWorkerPool *pPool = &pWorker->mpool;
pPool->name = name;
pPool->max = maxNum;
if (tWWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pWorker->queue = tWWorkerAllocQueue(pPool, param, (FItems)queueFp);
if (pWorker->queue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} else {
terrno = TSDB_CODE_INVALID_PARA;
}
return 0;
}
void dndCleanupWorker(SDnodeWorker *pWorker) {
if (pWorker->queue == NULL) return;
while (!taosQueueEmpty(pWorker->queue)) {
taosMsleep(10);
}
if (pWorker->type == DND_WORKER_SINGLE) {
tQWorkerCleanup(&pWorker->pool);
tQWorkerFreeQueue(&pWorker->pool, pWorker->queue);
} else if (pWorker->type == DND_WORKER_MULTI) {
tWWorkerCleanup(&pWorker->mpool);
tWWorkerFreeQueue(&pWorker->mpool, pWorker->queue);
} else {
}
}
int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg) {
if (pWorker == NULL || pWorker->queue == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
}
if (taosWriteQitem(pWorker->queue, pMsg) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}

6
source/dnode/mgmt/dnode/inc/dmInt.h
View File

@ -29,10 +29,10 @@ typedef struct SDnodeMgmt {
SEpSet mnodeEpSet; SEpSet mnodeEpSet;
SHashObj *dnodeHash; SHashObj *dnodeHash;
SArray *dnodeEps; SArray *dnodeEps;
TdThread *threadId; TdThread *threadId;
SRWLatch latch; SRWLatch latch;
SDnodeWorker mgmtWorker; SSingleWorker mgmtWorker;
SDnodeWorker statusWorker; SSingleWorker statusWorker;
const char *path; const char *path;
SDnode *pDnode; SDnode *pDnode;
SMgmtWrapper *pWrapper; SMgmtWrapper *pWrapper;

26
source/dnode/mgmt/dnode/src/dmMsg.c
View File

@ -114,19 +114,19 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
void dmInitMsgHandles(SMgmtWrapper *pWrapper) { void dmInitMsgHandles(SMgmtWrapper *pWrapper) {
// Requests handled by DNODE // Requests handled by DNODE
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg, 0);
// Requests handled by MNODE // Requests handled by MNODE
dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg, 0);
} }

26
source/dnode/mgmt/dnode/src/dmWorker.c
View File

@ -52,10 +52,12 @@ static void *dmThreadRoutine(void *param) {
} }
} }
static void dmProcessQueue(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SDnode *pDnode = pMgmt->pDnode; SDnodeMgmt *pMgmt = pInfo->ahandle;
SDnode *pDnode = pMgmt->pDnode;
SRpcMsg *pRpc = &pMsg->rpcMsg; SRpcMsg *pRpc = &pMsg->rpcMsg;
int32_t code = -1; int32_t code = -1;
dTrace("msg:%p, will be processed in dnode queue", pMsg); dTrace("msg:%p, will be processed in dnode queue", pMsg);
switch (pRpc->msgType) { switch (pRpc->msgType) {
@ -98,13 +100,17 @@ static void dmProcessQueue(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
} }
int32_t dmStartWorker(SDnodeMgmt *pMgmt) { int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
if (dndInitWorker(pMgmt, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dmProcessQueue) != 0) { SSingleWorkerCfg mgmtCfg = {
.minNum = 1, .maxNum = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) {
dError("failed to start dnode mgmt worker since %s", terrstr()); dError("failed to start dnode mgmt worker since %s", terrstr());
return -1; return -1;
} }
if (dndInitWorker(pMgmt, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dmProcessQueue) != 0) { SSingleWorkerCfg statusCfg = {
dError("failed to start dnode mgmt worker since %s", terrstr()); .minNum = 1, .maxNum = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->statusWorker, &statusCfg) != 0) {
dError("failed to start dnode status worker since %s", terrstr());
return -1; return -1;
} }
@ -123,8 +129,8 @@ int32_t dmStartThread(SDnodeMgmt *pMgmt) {
} }
void dmStopWorker(SDnodeMgmt *pMgmt) { void dmStopWorker(SDnodeMgmt *pMgmt) {
dndCleanupWorker(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->mgmtWorker);
dndCleanupWorker(&pMgmt->statusWorker); tSingleWorkerCleanup(&pMgmt->statusWorker);
if (pMgmt->threadId != NULL) { if (pMgmt->threadId != NULL) {
taosDestoryThread(pMgmt->threadId); taosDestoryThread(pMgmt->threadId);
@ -133,11 +139,11 @@ void dmStopWorker(SDnodeMgmt *pMgmt) {
} }
int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
SDnodeWorker *pWorker = &pMgmt->mgmtWorker; SSingleWorker *pWorker = &pMgmt->mgmtWorker;
if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) { if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) {
pWorker = &pMgmt->statusWorker; pWorker = &pMgmt->statusWorker;
} }
dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name); dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name);
return dndWriteMsgToWorker(pWorker, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
} }

6
source/dnode/mgmt/mnode/inc/mmInt.h
View File

@ -28,9 +28,9 @@ typedef struct SMnodeMgmt {
SDnode *pDnode; SDnode *pDnode;
SMgmtWrapper *pWrapper; SMgmtWrapper *pWrapper;
const char *path; const char *path;
SDnodeWorker readWorker; SSingleWorker readWorker;
SDnodeWorker writeWorker; SSingleWorker writeWorker;
SDnodeWorker syncWorker; SSingleWorker syncWorker;
SReplica replicas[TSDB_MAX_REPLICA]; SReplica replicas[TSDB_MAX_REPLICA];
int8_t replica; int8_t replica;
int8_t selfIndex; int8_t selfIndex;

15
source/dnode/mgmt/mnode/src/mmInt.c
View File

@ -39,14 +39,17 @@ static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) {
static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
SDnode *pDnode = pMgmt->pDnode; SDnode *pDnode = pMgmt->pDnode;
pOption->pWrapper = pMgmt->pWrapper;
pOption->putToWriteQFp = mmPutMsgToWriteQueue;
pOption->putToReadQFp = mmPutMsgToReadQueue;
pOption->sendReqFp = dndSendReqToDnode;
pOption->sendMnodeReqFp = dndSendReqToMnode;
pOption->sendRspFp = dndSendRsp;
pOption->dnodeId = pDnode->dnodeId; pOption->dnodeId = pDnode->dnodeId;
pOption->clusterId = pDnode->clusterId; pOption->clusterId = pDnode->clusterId;
SMsgCb msgCb = {0};
msgCb.pWrapper = pMgmt->pWrapper;
msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToReadQueue;
msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue;
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
pOption->msgCb = msgCb;
} }
static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {

142
source/dnode/mgmt/mnode/src/mmMsg.c
View File

@ -75,78 +75,80 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
void mmInitMsgHandles(SMgmtWrapper *pWrapper) { void mmInitMsgHandles(SMgmtWrapper *pWrapper) {
// Requests handled by DNODE // Requests handled by DNODE
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
// Requests handled by MNODE // Requests handled by MNODE
dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg); dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg); dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg); dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg); dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg); dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg); dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg); dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg); dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
// Requests handled by VNODE // Requests handled by VNODE
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)mmProcessWriteMsg);
} }

66
source/dnode/mgmt/mnode/src/mmWorker.c
View File

@ -16,7 +16,9 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mmInt.h" #include "mmInt.h"
static void mmProcessQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SMnodeMgmt *pMgmt = pInfo->ahandle;
dTrace("msg:%p, will be processed in mnode queue", pMsg); dTrace("msg:%p, will be processed in mnode queue", pMsg);
SRpcMsg *pRpc = &pMsg->rpcMsg; SRpcMsg *pRpc = &pMsg->rpcMsg;
int32_t code = -1; int32_t code = -1;
@ -42,34 +44,9 @@ static void mmProcessQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
int32_t mmStartWorker(SMnodeMgmt *pMgmt) { static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SNodeMsg *pMsg) {
if (dndInitWorker(pMgmt, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, mmProcessQueue) != 0) {
dError("failed to start mnode read worker since %s", terrstr());
return -1;
}
if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, mmProcessQueue) != 0) {
dError("failed to start mnode write worker since %s", terrstr());
return -1;
}
if (dndInitWorker(pMgmt, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, mmProcessQueue) != 0) {
dError("failed to start mnode sync worker since %s", terrstr());
return -1;
}
return 0;
}
void mmStopWorker(SMnodeMgmt *pMgmt) {
dndCleanupWorker(&pMgmt->readWorker);
dndCleanupWorker(&pMgmt->writeWorker);
dndCleanupWorker(&pMgmt->syncWorker);
}
static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SNodeMsg *pMsg) {
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
return dndWriteMsgToWorker(pWorker, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
} }
int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
@ -84,16 +61,16 @@ int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg);
} }
static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRpcMsg *pRpc) { static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) {
SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
if (pMsg == NULL) { if (pMsg == NULL) {
return -1; return -1;
} }
dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
pMsg->rpcMsg = *pRpc; pMsg->rpcMsg = *pRpc;
int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg); int32_t code = taosWriteQitem(pWorker->queue, pMsg);
if (code != 0) { if (code != 0) {
dTrace("msg:%p, is freed", pMsg); dTrace("msg:%p, is freed", pMsg);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
@ -112,3 +89,30 @@ int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
SMnodeMgmt *pMgmt = pWrapper->pMgmt; SMnodeMgmt *pMgmt = pWrapper->pMgmt;
return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc); return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc);
} }
int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
SSingleWorkerCfg cfg = {.minNum = 0, .maxNum = 1, .name = "mnode-read", .fp = (FItem)mmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->readWorker, &cfg) != 0) {
dError("failed to start mnode-read worker since %s", terrstr());
return -1;
}
if (tSingleWorkerInit(&pMgmt->writeWorker, &cfg) != 0) {
dError("failed to start mnode-write worker since %s", terrstr());
return -1;
}
if (tSingleWorkerInit(&pMgmt->syncWorker, &cfg) != 0) {
dError("failed to start mnode sync-worker since %s", terrstr());
return -1;
}
return 0;
}
void mmStopWorker(SMnodeMgmt *pMgmt) {
tSingleWorkerCleanup(&pMgmt->readWorker);
tSingleWorkerCleanup(&pMgmt->writeWorker);
tSingleWorkerCleanup(&pMgmt->syncWorker);
}

8
source/dnode/mgmt/qnode/inc/qmInt.h
View File

@ -28,8 +28,8 @@ typedef struct SQnodeMgmt {
SDnode *pDnode; SDnode *pDnode;
SMgmtWrapper *pWrapper; SMgmtWrapper *pWrapper;
const char *path; const char *path;
SDnodeWorker queryWorker; SSingleWorker queryWorker;
SDnodeWorker fetchWorker; SSingleWorker fetchWorker;
} SQnodeMgmt; } SQnodeMgmt;
// qmInt.c // qmInt.c
@ -42,6 +42,10 @@ int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
// qmWorker.c // qmWorker.c
int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t qmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t qmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype);
int32_t qmStartWorker(SQnodeMgmt *pMgmt); int32_t qmStartWorker(SQnodeMgmt *pMgmt);
void qmStopWorker(SQnodeMgmt *pMgmt); void qmStopWorker(SQnodeMgmt *pMgmt);
int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg);

16
source/dnode/mgmt/qnode/src/qmInt.c
View File

@ -19,13 +19,15 @@
static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) {
SDnode *pDnode = pMgmt->pDnode; SMsgCb msgCb = {0};
pOption->pWrapper = pMgmt->pWrapper; msgCb.pWrapper = pMgmt->pWrapper;
pOption->sendReqFp = dndSendReqToDnode; msgCb.queueFps[QUERY_QUEUE] = qmPutMsgToQueryQueue;
pOption->sendMnodeReqFp = dndSendReqToMnode; msgCb.queueFps[FETCH_QUEUE] = qmPutMsgToFetchQueue;
pOption->sendRspFp = dndSendRsp; msgCb.qsizeFp = qmGetQueueSize;
pOption->dnodeId = pDnode->dnodeId; msgCb.sendReqFp = dndSendReqToDnode;
pOption->clusterId = pDnode->clusterId; msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
pOption->msgCb = msgCb;
} }
static int32_t qmOpenImp(SQnodeMgmt *pMgmt) { static int32_t qmOpenImp(SQnodeMgmt *pMgmt) {

14
source/dnode/mgmt/qnode/src/qmMsg.c
View File

@ -54,4 +54,16 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
} }
} }
void qmInitMsgHandles(SMgmtWrapper *pWrapper) {} void qmInitMsgHandles(SMgmtWrapper *pWrapper) {
// Requests handled by VNODE
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)qmProcessQueryMsg, 1);
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)qmProcessQueryMsg, 1);
dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)qmProcessFetchMsg, 1);
dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)qmProcessFetchMsg, 1);
dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)qmProcessFetchMsg, 1);
dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)qmProcessFetchMsg, 1);
dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)qmProcessFetchMsg, 1);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)qmProcessFetchMsg, 1);
dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)qmProcessFetchMsg, 1);
}

132
source/dnode/mgmt/qnode/src/qmWorker.c
View File

@ -16,51 +16,119 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "qmInt.h" #include "qmInt.h"
static void qmProcessQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
dTrace("msg:%p, will be processed in qnode queue", pMsg); SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
SRpcMsg *pRsp = NULL; dndSendRsp(pWrapper, &rsp);
SRpcMsg *pRpc = &pMsg->rpcMsg; }
int32_t code = qndProcessMsg(pMgmt->pQnode, pRpc, &pRsp);
if (pRpc->msgType & 1u) { static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
if (pRsp != NULL) { SQnodeMgmt *pMgmt = pInfo->ahandle;
pRsp->ahandle = pRpc->ahandle;
dndSendRsp(pMgmt->pWrapper, pRsp); dTrace("msg:%p, will be processed in qnode-query queue", pMsg);
free(pRsp); int32_t code = qndProcessQueryMsg(pMgmt->pQnode, &pMsg->rpcMsg);
} else { if (code != 0) {
if (code != 0) code = terrno; qmSendRsp(pMgmt->pWrapper, pMsg, code);
SRpcMsg rpcRsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
dndSendRsp(pMgmt->pWrapper, &rpcRsp);
}
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
rpcFreeCont(pRpc->pCont); rpcFreeCont(pMsg->rpcMsg.pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { static void qmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SDnodeWorker *pWorker = &pMgmt->queryWorker; SQnodeMgmt *pMgmt = pInfo->ahandle;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, will be processed in qnode-fetch queue", pMsg);
return dndWriteMsgToWorker(pWorker, pMsg); int32_t code = qndProcessFetchMsg(pMgmt->pQnode, &pMsg->rpcMsg);
if (code != 0) {
qmSendRsp(pMgmt->pWrapper, pMsg, code);
}
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
rpcFreeCont(pMsg->rpcMsg.pCont);
taosFreeQitem(pMsg);
} }
int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { static int32_t qmPutMsgToWorker(SSingleWorker *pWorker, SNodeMsg *pMsg) {
SDnodeWorker *pWorker = &pMgmt->fetchWorker; dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
return taosWriteQitem(pWorker->queue, pMsg);
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
return dndWriteMsgToWorker(pWorker, pMsg);
} }
int32_t qmStartWorker(SQnodeMgmt *pMgmt) { int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { return qmPutMsgToWorker(&pMgmt->queryWorker, pMsg); }
if (dndInitWorker(pMgmt, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, qmProcessQueue) != 0) {
dError("failed to start qnode query worker since %s", terrstr()); int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { return qmPutMsgToWorker(&pMgmt->fetchWorker, pMsg); }
static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) {
SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
if (pMsg == NULL) {
return -1; return -1;
} }
if (dndInitWorker(pMgmt, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, qmProcessQueue) != 0) { dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
dError("failed to start qnode fetch worker since %s", terrstr()); pMsg->rpcMsg = *pRpc;
int32_t code = taosWriteQitem(pWorker->queue, pMsg);
if (code != 0) {
dTrace("msg:%p, is freed", pMsg);
taosFreeQitem(pMsg);
rpcFreeCont(pRpc->pCont);
}
return code;
}
int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
SQnodeMgmt *pMgmt = pWrapper->pMgmt;
return qmPutRpcMsgToWorker(pMgmt, &pMgmt->queryWorker, pRpc);
}
int32_t qmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
SQnodeMgmt *pMgmt = pWrapper->pMgmt;
return qmPutRpcMsgToWorker(pMgmt, &pMgmt->fetchWorker, pRpc);
}
int32_t qmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) {
int32_t size = -1;
SQnodeMgmt *pMgmt = pWrapper->pMgmt;
switch (qtype) {
case QUERY_QUEUE:
size = taosQueueSize(pMgmt->queryWorker.queue);
break;
case FETCH_QUEUE:
size = taosQueueSize(pMgmt->fetchWorker.queue);
break;
default:
break;
}
return size;
}
int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
int32_t maxFetchThreads = 4;
int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores);
int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1);
int32_t maxQueryThreads = minQueryThreads;
SSingleWorkerCfg queryCfg = {.minNum = minQueryThreads,
.maxNum = maxQueryThreads,
.name = "qnode-query",
.fp = (FItem)qmProcessQueryQueue,
.param = pMgmt};
if (tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg) != 0) {
dError("failed to start qnode-query worker since %s", terrstr());
return -1;
}
SSingleWorkerCfg fetchCfg = {.minNum = minFetchThreads,
.maxNum = maxFetchThreads,
.name = "qnode-fetch",
.fp = (FItem)qmProcessFetchQueue,
.param = pMgmt};
if (tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg) != 0) {
dError("failed to start qnode-fetch worker since %s", terrstr());
return -1; return -1;
} }
@ -68,6 +136,6 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
} }
void qmStopWorker(SQnodeMgmt *pMgmt) { void qmStopWorker(SQnodeMgmt *pMgmt) {
dndCleanupWorker(&pMgmt->queryWorker); tSingleWorkerCleanup(&pMgmt->queryWorker);
dndCleanupWorker(&pMgmt->fetchWorker); tSingleWorkerCleanup(&pMgmt->fetchWorker);
} }

4
source/dnode/mgmt/snode/inc/smInt.h
View File

@ -30,8 +30,8 @@ typedef struct SSnodeMgmt {
const char *path; const char *path;
SRWLatch latch; SRWLatch latch;
int8_t uniqueWorkerInUse; int8_t uniqueWorkerInUse;
SArray *uniqueWorkers; // SArray<SDnodeWorker*> SArray *uniqueWorkers; // SArray<SMultiWorker*>
SDnodeWorker sharedWorker; SSingleWorker sharedWorker;
} SSnodeMgmt; } SSnodeMgmt;
// smInt.c // smInt.c

13
source/dnode/mgmt/snode/src/smInt.c
View File

@ -19,13 +19,12 @@
static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) { static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) {
SDnode *pDnode = pMgmt->pDnode; SMsgCb msgCb = {0};
pOption->pWrapper = pMgmt->pWrapper; msgCb.pWrapper = pMgmt->pWrapper;
pOption->sendReqFp = dndSendReqToDnode; msgCb.sendReqFp = dndSendReqToDnode;
pOption->sendMnodeReqFp = dndSendReqToMnode; msgCb.sendMnodeReqFp = dndSendReqToMnode;
pOption->sendRspFp = dndSendRsp; msgCb.sendRspFp = dndSendRsp;
pOption->dnodeId = pDnode->dnodeId; pOption->msgCb = msgCb;
pOption->clusterId = pDnode->clusterId;
} }
static int32_t smOpenImp(SSnodeMgmt *pMgmt) { static int32_t smOpenImp(SSnodeMgmt *pMgmt) {

4
source/dnode/mgmt/snode/src/smMsg.c
View File

@ -56,6 +56,6 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
void smInitMsgHandles(SMgmtWrapper *pWrapper) { void smInitMsgHandles(SMgmtWrapper *pWrapper) {
// Requests handled by SNODE // Requests handled by SNODE
dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg); dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg, 0);
} }

48
source/dnode/mgmt/snode/src/smWorker.c
View File

@ -16,7 +16,9 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "smInt.h" #include "smInt.h"
static void smProcessUniqueQueue(SSnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) { static void smProcessUniqueQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SSnodeMgmt *pMgmt = pInfo->ahandle;
for (int32_t i = 0; i < numOfMsgs; i++) { for (int32_t i = 0; i < numOfMsgs; i++) {
SNodeMsg *pMsg = NULL; SNodeMsg *pMsg = NULL;
taosGetQitem(qall, (void **)&pMsg); taosGetQitem(qall, (void **)&pMsg);
@ -30,7 +32,9 @@ static void smProcessUniqueQueue(SSnodeMgmt *pMgmt, STaosQall *qall, int32_t num
} }
} }
static void smProcessSharedQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { static void smProcessSharedQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SSnodeMgmt *pMgmt = pInfo->ahandle;
dTrace("msg:%p, will be processed in snode shared queue", pMsg); dTrace("msg:%p, will be processed in snode shared queue", pMsg);
sndProcessSMsg(pMgmt->pSnode, &pMsg->rpcMsg); sndProcessSMsg(pMgmt->pSnode, &pMsg->rpcMsg);
@ -40,20 +44,23 @@ static void smProcessSharedQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
} }
int32_t smStartWorker(SSnodeMgmt *pMgmt) { int32_t smStartWorker(SSnodeMgmt *pMgmt) {
pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *)); pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(SMultiWorker *));
if (pMgmt->uniqueWorkers == NULL) { if (pMgmt->uniqueWorkers == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) {
SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker)); SMultiWorker *pUniqueWorker = malloc(sizeof(SMultiWorker));
if (pUniqueWorker == NULL) { if (pUniqueWorker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
if (dndInitWorker(pMgmt, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, smProcessSharedQueue) != 0) {
dError("failed to start snode unique worker since %s", terrstr()); SMultiWorkerCfg cfg = {.maxNum = 1, .name = "snode-unique", .fp = smProcessUniqueQueue, .param = pMgmt};
if (tMultiWorkerInit(pUniqueWorker, &cfg) != 0) {
dError("failed to start snode-unique worker since %s", terrstr());
return -1; return -1;
} }
if (taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker) == NULL) { if (taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker) == NULL) {
@ -62,9 +69,14 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
} }
} }
if (dndInitWorker(pMgmt, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM, SSingleWorkerCfg cfg = {.minNum = SND_SHARED_THREAD_NUM,
SND_SHARED_THREAD_NUM, smProcessSharedQueue)) { .maxNum = SND_SHARED_THREAD_NUM,
dError("failed to start snode shared worker since %s", terrstr()); .name = "snode-shared",
.fp = (FItem)smProcessSharedQueue,
.param = pMgmt};
if (tSingleWorkerInit(&pMgmt->sharedWorker, &cfg)) {
dError("failed to start snode shared-worker since %s", terrstr());
return -1; return -1;
} }
@ -73,11 +85,11 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
void smStopWorker(SSnodeMgmt *pMgmt) { void smStopWorker(SSnodeMgmt *pMgmt) {
for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) {
SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i); SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, i);
dndCleanupWorker(worker); tMultiWorkerCleanup(pWorker);
} }
taosArrayDestroy(pMgmt->uniqueWorkers); taosArrayDestroy(pMgmt->uniqueWorkers);
dndCleanupWorker(&pMgmt->sharedWorker); tSingleWorkerCleanup(&pMgmt->sharedWorker);
} }
static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) { static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) {
@ -93,33 +105,33 @@ static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) {
} }
int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0); SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0);
if (pWorker == NULL) { if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
return -1; return -1;
} }
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
return dndWriteMsgToWorker(pWorker, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
} }
int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg); int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg);
SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index); SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index);
if (pWorker == NULL) { if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
return -1; return -1;
} }
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
return dndWriteMsgToWorker(pWorker, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
} }
int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
SDnodeWorker *pWorker = &pMgmt->sharedWorker; SSingleWorker *pWorker = &pMgmt->sharedWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
return dndWriteMsgToWorker(pWorker, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
} }
int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {

7
source/dnode/mgmt/vnode/inc/vmInt.h
View File

@ -24,21 +24,19 @@
extern "C" { extern "C" {
#endif #endif
typedef enum { VND_WRITE_QUEUE, VND_QUERY_QUEUE, VND_FETCH_QUEUE, VND_APPLY_QUEUE, VND_SYNC_QUEUE } EVndQueueType;
typedef struct SVnodesMgmt { typedef struct SVnodesMgmt {
SHashObj *hash; SHashObj *hash;
SRWLatch latch; SRWLatch latch;
SVnodesStat state; SVnodesStat state;
STfs *pTfs; STfs *pTfs;
SQWorkerPool queryPool; SQWorkerPool queryPool;
SFWorkerPool fetchPool; SQWorkerPool fetchPool;
SWWorkerPool syncPool; SWWorkerPool syncPool;
SWWorkerPool writePool; SWWorkerPool writePool;
const char *path; const char *path;
SDnode *pDnode; SDnode *pDnode;
SMgmtWrapper *pWrapper; SMgmtWrapper *pWrapper;
SDnodeWorker mgmtWorker; SSingleWorker mgmtWorker;
} SVnodesMgmt; } SVnodesMgmt;
typedef struct { typedef struct {
@ -106,6 +104,7 @@ void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype);
int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg);
int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg);

22
source/dnode/mgmt/vnode/src/vmInt.c
View File

@ -128,7 +128,16 @@ static void *vmOpenVnodeFunc(void *param) {
pMgmt->state.openVnodes, pMgmt->state.totalVnodes); pMgmt->state.openVnodes, pMgmt->state.totalVnodes);
dndReportStartup(pDnode, "open-vnodes", stepDesc); dndReportStartup(pDnode, "open-vnodes", stepDesc);
SVnodeCfg cfg = {.pWrapper = pMgmt->pWrapper, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; SMsgCb msgCb = {0};
msgCb.pWrapper = pMgmt->pWrapper;
msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
msgCb.qsizeFp = vmGetQueueSize;
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); SVnode *pImpl = vnodeOpen(pCfg->path, &cfg);
if (pImpl == NULL) { if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
@ -262,7 +271,6 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
SDnode *pDnode = pWrapper->pDnode; SDnode *pDnode = pWrapper->pDnode;
SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt)); SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt));
int32_t code = -1; int32_t code = -1;
SVnodeOpt vnodeOpt = {0};
dInfo("vnodes-mgmt start to init"); dInfo("vnodes-mgmt start to init");
if (pMgmt == NULL) goto _OVER; if (pMgmt == NULL) goto _OVER;
@ -294,13 +302,7 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
goto _OVER; goto _OVER;
} }
vnodeOpt.nthreads = tsNumOfCommitThreads; if (vnodeInit() != 0) {
vnodeOpt.putToQueryQFp = vmPutMsgToQueryQueue;
vnodeOpt.putToFetchQFp = vmPutMsgToQueryQueue;
vnodeOpt.sendReqFp = dndSendReqToDnode;
vnodeOpt.sendMnodeReqFp = dndSendReqToMnode;
vnodeOpt.sendRspFp = dndSendRsp;
if (vnodeInit(&vnodeOpt) != 0) {
dError("failed to init vnode since %s", terrstr()); dError("failed to init vnode since %s", terrstr());
goto _OVER; goto _OVER;
} }
@ -341,7 +343,7 @@ void vmGetMgmtFp(SMgmtWrapper *pWrapper) {
mgmtFp.requiredFp = vmRequire; mgmtFp.requiredFp = vmRequire;
vmInitMsgHandles(pWrapper); vmInitMsgHandles(pWrapper);
pWrapper->name = "vnodes"; pWrapper->name = "vnode";
pWrapper->fp = mgmtFp; pWrapper->fp = mgmtFp;
} }

88
source/dnode/mgmt/vnode/src/vmMsg.c
View File

@ -82,7 +82,17 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
return -1; return -1;
} }
vnodeCfg.pWrapper = pMgmt->pWrapper; SMsgCb msgCb = {0};
msgCb.pWrapper = pMgmt->pWrapper;
msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
msgCb.qsizeFp = vmGetQueueSize;
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
vnodeCfg.msgCb = msgCb;
vnodeCfg.pTfs = pMgmt->pTfs; vnodeCfg.pTfs = pMgmt->pTfs;
vnodeCfg.dbId = wrapperCfg.dbUid; vnodeCfg.dbId = wrapperCfg.dbUid;
SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg);
@ -234,42 +244,44 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
void vmInitMsgHandles(SMgmtWrapper *pWrapper) { void vmInitMsgHandles(SMgmtWrapper *pWrapper) {
// Requests handled by VNODE // Requests handled by VNODE
dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0);
dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg); dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0);
} }

134
source/dnode/mgmt/vnode/src/vmWorker.c
View File

@ -21,7 +21,9 @@ static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
dndSendRsp(pWrapper, &rsp); dndSendRsp(pWrapper, &rsp);
} }
static void vmProcessMgmtQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SVnodesMgmt *pMgmt = pInfo->ahandle;
int32_t code = -1; int32_t code = -1;
tmsg_t msgType = pMsg->rpcMsg.msgType; tmsg_t msgType = pMsg->rpcMsg.msgType;
dTrace("msg:%p, will be processed in vnode-mgmt queue", pMsg); dTrace("msg:%p, will be processed in vnode-mgmt queue", pMsg);
@ -57,31 +59,35 @@ static void vmProcessMgmtQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
static void vmProcessQueryQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { static void vmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SVnodeObj *pVnode = pInfo->ahandle;
dTrace("msg:%p, will be processed in vnode-query queue", pMsg); dTrace("msg:%p, will be processed in vnode-query queue", pMsg);
int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, &pMsg->rpcMsg); int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, &pMsg->rpcMsg);
if (code != 0) { if (code != 0) {
vmSendRsp(pVnode->pWrapper, pMsg, code); vmSendRsp(pVnode->pWrapper, pMsg, code);
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
rpcFreeCont(pMsg->rpcMsg.pCont);
taosFreeQitem(pMsg);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
rpcFreeCont(pMsg->rpcMsg.pCont);
taosFreeQitem(pMsg);
} }
static void vmProcessFetchQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SVnodeObj *pVnode = pInfo->ahandle;
dTrace("msg:%p, will be processed in vnode-fetch queue", pMsg); dTrace("msg:%p, will be processed in vnode-fetch queue", pMsg);
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg);
if (code != 0) { if (code != 0) {
vmSendRsp(pVnode->pWrapper, pMsg, code); vmSendRsp(pVnode->pWrapper, pMsg, code);
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
rpcFreeCont(pMsg->rpcMsg.pCont);
taosFreeQitem(pMsg);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
rpcFreeCont(pMsg->rpcMsg.pCont);
taosFreeQitem(pMsg);
} }
static void vmProcessWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle;
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
if (pArray == NULL) { if (pArray == NULL) {
dError("failed to process %d msgs in write-queue since %s", numOfMsgs, terrstr()); dError("failed to process %d msgs in write-queue since %s", numOfMsgs, terrstr());
@ -128,8 +134,9 @@ static void vmProcessWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numO
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
} }
static void vmProcessApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SNodeMsg *pMsg = NULL; SVnodeObj *pVnode = pInfo->ahandle;
SNodeMsg *pMsg = NULL;
for (int32_t i = 0; i < numOfMsgs; ++i) { for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pMsg); taosGetQitem(qall, (void **)&pMsg);
@ -140,8 +147,9 @@ static void vmProcessApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numO
} }
} }
static void vmProcessSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SNodeMsg *pMsg = NULL; SVnodeObj *pVnode = pInfo->ahandle;
SNodeMsg *pMsg = NULL;
for (int32_t i = 0; i < numOfMsgs; ++i) { for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pMsg); taosGetQitem(qall, (void **)&pMsg);
@ -152,7 +160,7 @@ static void vmProcessSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOf
} }
} }
static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EVndQueueType qtype) { static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) {
SRpcMsg *pRpc = &pMsg->rpcMsg; SRpcMsg *pRpc = &pMsg->rpcMsg;
int32_t code = -1; int32_t code = -1;
@ -162,25 +170,27 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EVndQueue
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) { if (pVnode == NULL) {
dError("vgId:%d, failed to write msg:%p to queue since %s", pHead->vgId, pMsg, terrstr()); dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr());
return -1; return -1;
} }
switch (qtype) { switch (qtype) {
case VND_QUERY_QUEUE: case QUERY_QUEUE:
dTrace("msg:%p, will be written into vnode-query queue", pMsg); dTrace("msg:%p, will be written into vnode-query queue", pMsg);
code = taosWriteQitem(pVnode->pQueryQ, pMsg); code = taosWriteQitem(pVnode->pQueryQ, pMsg);
break; break;
case VND_FETCH_QUEUE: case FETCH_QUEUE:
dTrace("msg:%p, will be written into vnode-fetch queue", pMsg); dTrace("msg:%p, will be written into vnode-fetch queue", pMsg);
code = taosWriteQitem(pVnode->pFetchQ, pMsg); code = taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case VND_WRITE_QUEUE: case WRITE_QUEUE:
dTrace("msg:%p, will be written into vnode-write queue", pMsg); dTrace("msg:%p, will be written into vnode-write queue", pMsg);
code = taosWriteQitem(pVnode->pWriteQ, pMsg); code = taosWriteQitem(pVnode->pWriteQ, pMsg);
case VND_SYNC_QUEUE: break;
case SYNC_QUEUE:
dTrace("msg:%p, will be written into vnode-sync queue", pMsg); dTrace("msg:%p, will be written into vnode-sync queue", pMsg);
code = taosWriteQitem(pVnode->pSyncQ, pMsg); code = taosWriteQitem(pVnode->pSyncQ, pMsg);
break;
default: default:
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
break; break;
@ -190,29 +200,21 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EVndQueue
return code; return code;
} }
int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); }
return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_SYNC_QUEUE);
}
int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, WRITE_QUEUE); }
return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_WRITE_QUEUE);
}
int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, QUERY_QUEUE); }
return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_QUERY_QUEUE);
}
int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); }
return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_FETCH_QUEUE);
}
int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
SDnodeWorker *pWorker = &pMgmt->mgmtWorker; SSingleWorker *pWorker = &pMgmt->mgmtWorker;
dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name); dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name);
return dndWriteMsgToWorker(pWorker, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
} }
static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EVndQueueType qtype) { static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt; SVnodesMgmt *pMgmt = pWrapper->pMgmt;
int32_t code = -1; int32_t code = -1;
SMsgHead *pHead = pRpc->pCont; SMsgHead *pHead = pRpc->pCont;
@ -225,20 +227,18 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EVndQue
dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
pMsg->rpcMsg = *pRpc; pMsg->rpcMsg = *pRpc;
switch (qtype) { switch (qtype) {
case VND_QUERY_QUEUE: case QUERY_QUEUE:
dTrace("msg:%p, will be put into vnode-query queue", pMsg); dTrace("msg:%p, will be put into vnode-query queue", pMsg);
code = taosWriteQitem(pVnode->pQueryQ, pMsg); code = taosWriteQitem(pVnode->pQueryQ, pMsg);
break; break;
case VND_FETCH_QUEUE: case FETCH_QUEUE:
dTrace("msg:%p, will be put into vnode-fetch queue", pMsg); dTrace("msg:%p, will be put into vnode-fetch queue", pMsg);
code = taosWriteQitem(pVnode->pFetchQ, pMsg); code = taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case VND_APPLY_QUEUE: case APPLY_QUEUE:
dTrace("msg:%p, will be put into vnode-apply queue", pMsg); dTrace("msg:%p, will be put into vnode-apply queue", pMsg);
code = taosWriteQitem(pVnode->pApplyQ, pMsg); code = taosWriteQitem(pVnode->pApplyQ, pMsg);
break; break;
case VND_WRITE_QUEUE:
case VND_SYNC_QUEUE:
default: default:
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
break; break;
@ -249,22 +249,50 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EVndQue
} }
int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
return vmPutRpcMsgToQueue(pWrapper, pRpc, VND_QUERY_QUEUE); return vmPutRpcMsgToQueue(pWrapper, pRpc, QUERY_QUEUE);
} }
int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
return vmPutRpcMsgToQueue(pWrapper, pRpc, VND_FETCH_QUEUE); return vmPutRpcMsgToQueue(pWrapper, pRpc, FETCH_QUEUE);
} }
int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
return vmPutRpcMsgToQueue(pWrapper, pRpc, VND_APPLY_QUEUE); return vmPutRpcMsgToQueue(pWrapper, pRpc, APPLY_QUEUE);
}
int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) {
int32_t size = -1;
SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId);
if (pVnode != NULL) {
switch (qtype) {
case QUERY_QUEUE:
size = taosQueueSize(pVnode->pQueryQ);
break;
case FETCH_QUEUE:
size = taosQueueSize(pVnode->pFetchQ);
break;
case WRITE_QUEUE:
size = taosQueueSize(pVnode->pWriteQ);
break;
case SYNC_QUEUE:
size = taosQueueSize(pVnode->pSyncQ);
break;
case APPLY_QUEUE:
size = taosQueueSize(pVnode->pApplyQ);
break;
default:
break;
}
}
vmReleaseVnode(pWrapper->pMgmt, pVnode);
return size;
} }
int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue);
pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue);
pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue);
pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL ||
@ -279,7 +307,7 @@ int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
@ -305,11 +333,11 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
pQPool->max = maxQueryThreads; pQPool->max = maxQueryThreads;
if (tQWorkerInit(pQPool) != 0) return -1; if (tQWorkerInit(pQPool) != 0) return -1;
SFWorkerPool *pFPool = &pMgmt->fetchPool; SQWorkerPool *pFPool = &pMgmt->fetchPool;
pFPool->name = "vnode-fetch"; pFPool->name = "vnode-fetch";
pFPool->min = minFetchThreads; pFPool->min = minFetchThreads;
pFPool->max = maxFetchThreads; pFPool->max = maxFetchThreads;
if (tFWorkerInit(pFPool) != 0) return -1; if (tQWorkerInit(pFPool) != 0) return -1;
SWWorkerPool *pWPool = &pMgmt->writePool; SWWorkerPool *pWPool = &pMgmt->writePool;
pWPool->name = "vnode-write"; pWPool->name = "vnode-write";
@ -321,7 +349,9 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
pWPool->max = maxSyncThreads; pWPool->max = maxSyncThreads;
if (tWWorkerInit(pWPool) != 0) return -1; if (tWWorkerInit(pWPool) != 0) return -1;
if (dndInitWorker(pMgmt, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "vnode-mgmt", 1, 1, vmProcessMgmtQueue) != 0) { SSingleWorkerCfg cfg = {
.minNum = 1, .maxNum = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) {
dError("failed to start vnode-mgmt worker since %s", terrstr()); dError("failed to start vnode-mgmt worker since %s", terrstr());
return -1; return -1;
} }
@ -331,8 +361,8 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
} }
void vmStopWorker(SVnodesMgmt *pMgmt) { void vmStopWorker(SVnodesMgmt *pMgmt) {
dndCleanupWorker(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->mgmtWorker);
tFWorkerCleanup(&pMgmt->fetchPool); tQWorkerCleanup(&pMgmt->fetchPool);
tQWorkerCleanup(&pMgmt->queryPool); tQWorkerCleanup(&pMgmt->queryPool);
tWWorkerCleanup(&pMgmt->writePool); tWWorkerCleanup(&pMgmt->writePool);
tWWorkerCleanup(&pMgmt->syncPool); tWWorkerCleanup(&pMgmt->syncPool);

13
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -119,19 +119,12 @@ typedef struct SMnode {
SHashObj *infosMeta; SHashObj *infosMeta;
SGrantInfo grant; SGrantInfo grant;
MndMsgFp msgFp[TDMT_MAX]; MndMsgFp msgFp[TDMT_MAX];
SendReqFp sendReqFp; SMsgCb msgCb;
SendMnodeReqFp sendMnodeReqFp;
PutToQueueFp putToWriteQFp;
PutToQueueFp putToReadQFp;
} SMnode; } SMnode;
int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *rpcMsg); void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg);
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
uint64_t mndGenerateUid(char *name, int32_t len); uint64_t mndGenerateUid(char *name, int32_t len);
void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
source/dnode/mnode/impl/inc/mndSnode.h
View File

@ -24,6 +24,7 @@ extern "C" {
int32_t mndInitSnode(SMnode *pMnode); int32_t mndInitSnode(SMnode *pMnode);
void mndCleanupSnode(SMnode *pMnode); void mndCleanupSnode(SMnode *pMnode);
SEpSet mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode);
#ifdef __cplusplus #ifdef __cplusplus
} }

264
source/dnode/mnode/impl/src/mndDb.c
View File

@ -930,12 +930,12 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
while (vindex < pDb->cfg.numOfVgroups) { while (true) {
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break; if (pIter == NULL) break;
if (pVgroup->dbUid == pDb->uid) { if (NULL == pDb || pVgroup->dbUid == pDb->uid) {
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
vgInfo.vgId = pVgroup->vgId; vgInfo.vgId = pVgroup->vgId;
vgInfo.hashBegin = pVgroup->hashBegin; vgInfo.hashBegin = pVgroup->hashBegin;
@ -960,6 +960,10 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
} }
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
if (pDb && (vindex >= pDb->cfg.numOfVgroups)) {
break;
}
} }
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
@ -981,7 +985,25 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
char *p = strchr(usedbReq.db, '.'); char *p = strchr(usedbReq.db, '.');
if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) {
memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
code = 0; static int32_t vgVersion = 1;
if (usedbReq.vgVersion < vgVersion) {
usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo));
if (usedbRsp.pVgroupInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto USE_DB_OVER;
}
mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos);
usedbRsp.vgVersion = vgVersion++;
if (taosArrayGetSize(usedbRsp.pVgroupInfos) <= 0) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
}
} else {
usedbRsp.vgVersion = usedbReq.vgVersion;
code = 0;
}
usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);
} else { } else {
pDb = mndAcquireDb(pMnode, usedbReq.db); pDb = mndAcquireDb(pMnode, usedbReq.db);
if (pDb == NULL) { if (pDb == NULL) {
@ -1341,123 +1363,149 @@ char *mnGetDbStr(char *src) {
return pos; return pos;
} }
static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { static char* getDataPosition(char* pData, SShowObj* pShow, int32_t cols, int32_t rows, int32_t capacityOfRow) {
return pData + pShow->offset[cols] * capacityOfRow + pShow->bytes[cols] * rows;
}
static void dumpDbInfoToPayload(char* data, SDbObj* pDb, SShowObj* pShow, int32_t rows, int32_t rowCapacity, int64_t numOfTables) {
int32_t cols = 0;
char* pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
char *name = mnGetDbStr(pDb->name);
if (name != NULL) {
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]);
} else {
STR_TO_VARSTR(pWrite, "NULL");
}
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int64_t *)pWrite = pDb->createdTime;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int16_t *)pWrite = pDb->cfg.numOfVgroups;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int64_t *)pWrite = numOfTables;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int16_t *)pWrite = pDb->cfg.replications;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int16_t *)pWrite = pDb->cfg.quorum;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int16_t *)pWrite = pDb->cfg.daysPerFile;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
char tmp[128] = {0};
if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) {
sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0);
} else {
sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2);
}
STR_WITH_SIZE_TO_VARSTR(pWrite, tmp, strlen(tmp));
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int32_t *)pWrite = pDb->cfg.cacheBlockSize;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int32_t *)pWrite = pDb->cfg.totalBlocks;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int32_t *)pWrite = pDb->cfg.minRows;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int32_t *)pWrite = pDb->cfg.maxRows;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int8_t *)pWrite = pDb->cfg.walLevel;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int32_t *)pWrite = pDb->cfg.fsyncPeriod;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int8_t *)pWrite = pDb->cfg.compression;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
*(int8_t *)pWrite = pDb->cfg.cacheLastRow;
cols++;
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
char *prec = NULL;
switch (pDb->cfg.precision) {
case TSDB_TIME_PRECISION_MILLI:
prec = TSDB_TIME_PRECISION_MILLI_STR;
break;
case TSDB_TIME_PRECISION_MICRO:
prec = TSDB_TIME_PRECISION_MICRO_STR;
break;
case TSDB_TIME_PRECISION_NANO:
prec = TSDB_TIME_PRECISION_NANO_STR;
break;
default:
prec = "none";
break;
}
STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
cols++;
// pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
// *(int8_t *)pWrite = pDb->cfg.update;
}
static void setInformationSchemaDbCfg(SDbObj* pDbObj) {
ASSERT(pDbObj != NULL);
strncpy(pDbObj->name, TSDB_INFORMATION_SCHEMA_DB, tListLen(pDbObj->name));
pDbObj->createdTime = 0;
pDbObj->cfg.numOfVgroups = 0;
pDbObj->cfg.quorum = 1;
pDbObj->cfg.replications = 1;
pDbObj->cfg.update = 1;
pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI;
}
static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rowsCapacity) {
SMnode *pMnode = pReq->pNode; SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0; int32_t numOfRows = 0;
SDbObj *pDb = NULL; SDbObj *pDb = NULL;
char *pWrite;
int32_t cols = 0;
while (numOfRows < rows) { while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb); pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb);
if (pShow->pIter == NULL) break; if (pShow->pIter == NULL) {
break;
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
char *name = mnGetDbStr(pDb->name);
if (name != NULL) {
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]);
} else {
STR_TO_VARSTR(pWrite, "NULL");
} }
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pDb->createdTime;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDb->cfg.numOfVgroups;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = 0; // todo
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDb->cfg.replications;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDb->cfg.quorum;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDb->cfg.daysPerFile;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
char tmp[128] = {0};
if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) {
sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0);
} else {
sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2);
}
STR_WITH_SIZE_TO_VARSTR(pWrite, tmp, strlen(tmp));
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.cacheBlockSize;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.totalBlocks;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.minRows;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.maxRows;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int8_t *)pWrite = pDb->cfg.walLevel;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.fsyncPeriod;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int8_t *)pWrite = pDb->cfg.compression;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int8_t *)pWrite = pDb->cfg.cacheLastRow;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
char *prec = NULL;
switch (pDb->cfg.precision) {
case TSDB_TIME_PRECISION_MILLI:
prec = TSDB_TIME_PRECISION_MILLI_STR;
break;
case TSDB_TIME_PRECISION_MICRO:
prec = TSDB_TIME_PRECISION_MICRO_STR;
break;
case TSDB_TIME_PRECISION_NANO:
prec = TSDB_TIME_PRECISION_NANO_STR;
break;
default:
prec = "none";
break;
}
STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
cols++;
// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
// *(int8_t *)pWrite = pDb->cfg.update;
// cols++;
dumpDbInfoToPayload(data, pDb, pShow, numOfRows, rowsCapacity, 0);
numOfRows++; numOfRows++;
sdbRelease(pSdb, pDb); sdbRelease(pSdb, pDb);
} }
mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); // Append the information_schema database into the result.
if (numOfRows < rowsCapacity) {
SDbObj dummyISDb = {0};
setInformationSchemaDbCfg(&dummyISDb);
dumpDbInfoToPayload(data, &dummyISDb, pShow, numOfRows, rowsCapacity, 14);
numOfRows += 1;
}
mndVacuumResult(data, pShow->numOfColumns, numOfRows, rowsCapacity, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;

4
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -623,7 +623,7 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq) {
.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .ahandle = pReq->rpcMsg.ahandle}; .msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .ahandle = pReq->rpcMsg.ahandle};
mInfo("dnode:%d, app:%p config:%s req send to dnode", cfgReq.dnodeId, rpcMsg.ahandle, cfgReq.config); mInfo("dnode:%d, app:%p config:%s req send to dnode", cfgReq.dnodeId, rpcMsg.ahandle, cfgReq.config);
mndSendReqToDnode(pMnode, &epSet, &rpcMsg); tmsgSendReq(&pMnode->msgCb, &epSet, &rpcMsg);
return 0; return 0;
} }
@ -752,7 +752,7 @@ static int32_t mndGetDnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p
pSchema[cols].bytes = pShow->bytes[cols]; pSchema[cols].bytes = pShow->bytes[cols];
cols++; cols++;
pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pShow->bytes[cols] = 256 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "offline_reason"); strcpy(pSchema[cols].name, "offline_reason");
pSchema[cols].bytes = pShow->bytes[cols]; pSchema[cols].bytes = pShow->bytes[cols];

109
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -16,85 +16,92 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mndInfoSchema.h" #include "mndInfoSchema.h"
static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
{.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
{.name = "vnodes", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "cores", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "offline_reason", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
}; };
static const SInfosTableSchema mnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, static const SInfosTableSchema mnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
}; };
static const SInfosTableSchema modulesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, static const SInfosTableSchema modulesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, {.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
}; };
static const SInfosTableSchema qnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, static const SInfosTableSchema qnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, {.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
}; };
static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "vgroups", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "replica", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "quorum", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "keep", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "wallevel", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "wallevel", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "comp", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "cachelast", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, {.name = "precision", .bytes = 3 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update
}; };
static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY},
{.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
}; };
static const SInfosTableSchema userIdxSchema[] = {{.name = "table_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, static const SInfosTableSchema userIdxSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, {.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, {.name = "index_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}, {.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY},
}; };
static const SInfosTableSchema userStbsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, static const SInfosTableSchema userStbsSchema[] = {{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT},
}; };
static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY},
{.name = "dest_table", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, {.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY}, {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY},
}; };
static const SInfosTableSchema userTblsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, static const SInfosTableSchema userTblsSchema[] = {
{.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "tid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
}; };
static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY}, {.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY},
{.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@ -107,13 +114,15 @@ static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name",
{.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
}; };
static const SInfosTableSchema userUsersSchema[] = {{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, static const SInfosTableSchema userUsersSchema[] = {{.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "privilege", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
}; };
static const SInfosTableSchema vgroupsSchema[] = {{.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, static const SInfosTableSchema vgroupsSchema[] = {{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
@ -122,13 +131,15 @@ static const SInfosTableSchema vgroupsSchema[] = {{.name = "vg_id", .
{.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
}; };
static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)}, static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)},
{TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)},
{TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)},
{TSDB_INS_TABLE_USER_DATABASE, userDBSchema, tListLen(userDBSchema)}, {TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)},
{TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)},
{TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)}, {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)},
{TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
@ -139,7 +150,7 @@ static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
}; };
//connection/application/
int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) { int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
SSchema *schema = calloc(colNum, sizeof(SSchema)); SSchema *schema = calloc(colNum, sizeof(SSchema));
if (NULL == schema) { if (NULL == schema) {
@ -147,7 +158,6 @@ int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, S
return -1; return -1;
} }
for (int32_t i = 0; i < colNum; ++i) { for (int32_t i = 0; i < colNum; ++i) {
strcpy(schema[i].name, pSrc[i].name); strcpy(schema[i].name, pSrc[i].name);
@ -157,7 +167,6 @@ int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, S
} }
*pDst = schema; *pDst = schema;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -165,7 +174,7 @@ int32_t mndInsInitMeta(SHashObj *hash) {
STableMetaRsp meta = {0}; STableMetaRsp meta = {0};
strcpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB); strcpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB);
meta.tableType = TSDB_NORMAL_TABLE; meta.tableType = TSDB_SYSTEM_TABLE;
meta.sversion = 1; meta.sversion = 1;
meta.tversion = 1; meta.tversion = 1;

4
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -692,11 +692,11 @@ static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, in
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pObj->createdTime; *(int64_t *)pWrite = pObj->roleTime;
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pObj->roleTime; *(int64_t *)pWrite = pObj->createdTime;
cols++; cols++;
numOfRows++; numOfRows++;

93
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -20,6 +20,7 @@
#include "mndMnode.h" #include "mndMnode.h"
#include "mndOffset.h" #include "mndOffset.h"
#include "mndShow.h" #include "mndShow.h"
#include "mndSnode.h"
#include "mndStb.h" #include "mndStb.h"
#include "mndStream.h" #include "mndStream.h"
#include "mndSubscribe.h" #include "mndSubscribe.h"
@ -31,20 +32,23 @@
#include "tname.h" #include "tname.h"
#include "tuuid.h" #include "tuuid.h"
int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet) { extern bool tsStreamSchedV;
int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) {
SCoder encoder; SCoder encoder;
tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
tEncodeSStreamTask(&encoder, pTask); tEncodeSStreamTask(&encoder, pTask);
int32_t tlen = sizeof(SMsgHead) + encoder.pos; int32_t size = encoder.pos;
int32_t tlen = sizeof(SMsgHead) + size;
tCoderClear(&encoder); tCoderClear(&encoder);
void* buf = malloc(tlen); void* buf = malloc(tlen);
if (buf == NULL) { if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
((SMsgHead*)buf)->streamTaskId = pTask->taskId; ((SMsgHead*)buf)->streamTaskId = htonl(nodeId);
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, size, TD_ENCODER);
tEncodeSStreamTask(&encoder, pTask); tEncodeSStreamTask(&encoder, pTask);
tCoderClear(&encoder); tCoderClear(&encoder);
@ -52,7 +56,7 @@ int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet
memcpy(&action.epSet, pEpSet, sizeof(SEpSet)); memcpy(&action.epSet, pEpSet, sizeof(SEpSet));
action.pCont = buf; action.pCont = buf;
action.contLen = tlen; action.contLen = tlen;
action.msgType = TDMT_SND_TASK_DEPLOY; action.msgType = type;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
rpcFreeCont(buf); rpcFreeCont(buf);
return -1; return -1;
@ -69,12 +73,27 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SS
terrno = TSDB_CODE_QRY_INVALID_INPUT; terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1; return -1;
} }
mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet); mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId);
return 0; return 0;
} }
SSnodeObj* mndSchedFetchSnode(SMnode* pMnode) {
SSnodeObj* pObj = NULL;
pObj = sdbFetch(pMnode->pSdb, SDB_SNODE, NULL, (void**)&pObj);
return pObj;
}
int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan,
const SSnodeObj* pSnode) { const SSnodeObj* pSnode) {
int32_t msgLen;
plan->execNode.nodeId = pSnode->id;
plan->execNode.epSet = mndAcquireEpFromSnode(pMnode, pSnode);
if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_SND_TASK_DEPLOY, 0);
return 0; return 0;
} }
@ -90,6 +109,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); int32_t totLevel = LIST_LENGTH(pPlan->pSubplans);
pStream->tasks = taosArrayInit(totLevel, sizeof(SArray)); pStream->tasks = taosArrayInit(totLevel, sizeof(SArray));
int32_t lastUsedVgId = 0;
for (int32_t level = 0; level < totLevel; level++) { for (int32_t level = 0; level < totLevel; level++) {
SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTask)); SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTask));
@ -97,9 +117,9 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
int32_t opNum = LIST_LENGTH(inner->pNodeList); int32_t opNum = LIST_LENGTH(inner->pNodeList);
ASSERT(opNum == 1); ASSERT(opNum == 1);
SSubplan* plan = nodesListGetNode(inner->pNodeList, level); SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
if (level == 0) { if (level == 0) {
ASSERT(plan->type == SUBPLAN_TYPE_SCAN); ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN);
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
@ -109,12 +129,15 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
continue; continue;
} }
lastUsedVgId = pVgroup->vgId;
pStream->vgNum++; pStream->vgNum++;
// send to vnode // send to vnode
SStreamTask* pTask = streamTaskNew(pStream->uid, level); SStreamTask* pTask = streamTaskNew(pStream->uid, level);
pTask->pipeSource = 1;
pTask->pipeSink = level == totLevel - 1 ? 1 : 0;
pTask->parallelizable = 1;
// TODO: set to // TODO: set to
pTask->parallel = 4;
if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) {
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
@ -122,35 +145,37 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
} }
taosArrayPush(taskOneLevel, pTask); taosArrayPush(taskOneLevel, pTask);
} }
} else if (plan->subplanType == SUBPLAN_TYPE_SCAN) {
// duplicatable
int32_t parallel = 0;
// if no snode, parallel set to fetch thread num in vnode
// if has snode, set to shared thread num in snode
parallel = SND_SHARED_THREAD_NUM;
SStreamTask* pTask = streamTaskNew(pStream->uid, level);
pTask->parallel = parallel;
// TODO:get snode id and ep
if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) {
sdbRelease(pSdb, pVgroup);
qDestroyQueryPlan(pPlan);
return -1;
}
taosArrayPush(taskOneLevel, pTask);
} else { } else {
// not duplicatable
SStreamTask* pTask = streamTaskNew(pStream->uid, level); SStreamTask* pTask = streamTaskNew(pStream->uid, level);
pTask->pipeSource = 0;
pTask->pipeSink = level == totLevel - 1 ? 1 : 0;
pTask->parallelizable = plan->subplanType == SUBPLAN_TYPE_SCAN;
pTask->nextOpDst = STREAM_NEXT_OP_DST__VND;
// TODO: get snode if (tsStreamSchedV) {
if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { ASSERT(lastUsedVgId != 0);
sdbRelease(pSdb, pVgroup); SVgObj* pVg = mndAcquireVgroup(pMnode, lastUsedVgId);
qDestroyQueryPlan(pPlan); if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVg) < 0) {
return -1; sdbRelease(pSdb, pVg);
qDestroyQueryPlan(pPlan);
return -1;
}
sdbRelease(pSdb, pVg);
} else {
SSnodeObj* pSnode = mndSchedFetchSnode(pMnode);
if (pSnode != NULL) {
if (mndAssignTaskToSnode(pMnode, pTrans, pTask, plan, pSnode) < 0) {
sdbRelease(pSdb, pSnode);
qDestroyQueryPlan(pPlan);
return -1;
}
sdbRelease(pMnode->pSdb, pSnode);
} else {
// TODO: assign to one vg
ASSERT(0);
}
} }
taosArrayPush(taskOneLevel, pTask); taosArrayPush(taskOneLevel, pTask);
} }
taosArrayPush(pStream->tasks, taskOneLevel); taosArrayPush(pStream->tasks, taskOneLevel);

18
source/dnode/mnode/impl/src/mndShow.c
View File

@ -289,6 +289,20 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
mError("failed to process show-meta req since %s", terrstr()); mError("failed to process show-meta req since %s", terrstr());
return -1; return -1;
} }
STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb));
pShow->numOfRows = 100;
int32_t offset = 0;
for(int32_t i = 0; i < meta->numOfColumns; ++i) {
pShow->numOfColumns = meta->numOfColumns;
pShow->offset[i] = offset;
int32_t bytes = meta->pSchemas[i].bytes;
pShow->rowSize += bytes;
pShow->bytes[i] = bytes;
offset += bytes;
}
} else { } else {
pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); pShow = mndAcquireShowObj(pMnode, retrieveReq.showId);
if (pShow == NULL) { if (pShow == NULL) {
@ -330,7 +344,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
size = pShow->rowSize * rowsToRead; size = pShow->rowSize * rowsToRead;
size += SHOW_STEP_SIZE; size += SHOW_STEP_SIZE;
SRetrieveTableRsp *pRsp = rpcMallocCont(size); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) { if (pRsp == NULL) {
mndReleaseShowObj((SShowObj*) pShow, false); mndReleaseShowObj((SShowObj*) pShow, false);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -338,6 +352,8 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
return -1; return -1;
} }
pRsp->handle = htobe64(pShow->id);
// if free flag is set, client wants to clean the resources // if free flag is set, client wants to clean the resources
if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
rowsRead = (*retrieveFp)(pReq, (SShowObj*) pShow, pRsp->data, rowsToRead); rowsRead = (*retrieveFp)(pReq, (SShowObj*) pShow, pRsp->data, rowsToRead);

9
source/dnode/mnode/impl/src/mndSnode.c
View File

@ -60,6 +60,15 @@ int32_t mndInitSnode(SMnode *pMnode) {
void mndCleanupSnode(SMnode *pMnode) {} void mndCleanupSnode(SMnode *pMnode) {}
SEpSet mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode) {
SEpSet epSet;
memcpy(epSet.eps->fqdn, pSnode->pDnode->fqdn, 128);
epSet.eps->port = pSnode->pDnode->port;
epSet.numOfEps = 1;
epSet.inUse = 0;
return epSet;
}
static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) { static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) {
SSnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_SNODE, &snodeId); SSnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_SNODE, &snodeId);
if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {

38
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1490,7 +1490,7 @@ static void mndExtractTableName(char *tableId, char *name) {
int32_t pos = -1; int32_t pos = -1;
int32_t num = 0; int32_t num = 0;
for (pos = 0; tableId[pos] != 0; ++pos) { for (pos = 0; tableId[pos] != 0; ++pos) {
if (tableId[pos] == '.') num++; if (tableId[pos] == TS_PATH_DELIMITER[0]) num++;
if (num == 2) break; if (num == 2) break;
} }
@ -1508,8 +1508,11 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
char *pWrite; char *pWrite;
char prefix[TSDB_DB_FNAME_LEN] = {0}; char prefix[TSDB_DB_FNAME_LEN] = {0};
SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); SDbObj* pDb = NULL;
if (pDb == NULL) return 0; if (strlen(pShow->db) > 0) {
pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return 0;
}
tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN); tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
strcat(prefix, TS_PATH_DELIMITER); strcat(prefix, TS_PATH_DELIMITER);
@ -1519,7 +1522,7 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb);
if (pShow->pIter == NULL) break; if (pShow->pIter == NULL) break;
if (pStb->dbUid != pDb->uid) { if (pDb != NULL && pStb->dbUid != pDb->uid) {
if (strncmp(pStb->db, pDb->name, prefixLen) == 0) { if (strncmp(pStb->db, pDb->name, prefixLen) == 0) {
mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid); mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid);
} }
@ -1530,12 +1533,20 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
cols = 0; cols = 0;
SName name = {0};
char stbName[TSDB_TABLE_NAME_LEN] = {0}; char stbName[TSDB_TABLE_NAME_LEN] = {0};
tstrncpy(stbName, pStb->name + prefixLen, TSDB_TABLE_NAME_LEN); mndExtractTableName(pStb->name, stbName);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, stbName); STR_TO_VARSTR(pWrite, stbName);
cols++; cols++;
char db[TSDB_DB_NAME_LEN] = {0};
tNameFromString(&name, pStb->db, T_NAME_ACCT|T_NAME_DB);
tNameGetDbName(&name, db);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, db);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pStb->createdTime; *(int64_t *)pWrite = pStb->createdTime;
cols++; cols++;
@ -1548,11 +1559,26 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
*(int32_t *)pWrite = pStb->numOfTags; *(int32_t *)pWrite = pStb->numOfTags;
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = 0; // number of tables
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pStb->updateTime; // number of tables
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, pStb->comment);
cols++;
numOfRows++; numOfRows++;
sdbRelease(pSdb, pStb); sdbRelease(pSdb, pStb);
} }
mndReleaseDb(pMnode, pDb); if (pDb != NULL) {
mndReleaseDb(pMnode, pDb);
}
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
return numOfRows; return numOfRows;

Some files were not shown because too many files have changed in this diff Show More