xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

merge 3.0

This commit is contained in:
yihaoDeng 2022-03-19 09:08:11 +08:00
parent 916dbe798b 7fc4d5d373
commit 96db2f503b
157 changed files with 8109 additions and 3767 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmake/cmake.options
View File

@ -18,6 +18,13 @@ IF(${TD_WINDOWS})
ON ON
) )
MESSAGE("build iconv Win32")
option(
BUILD_WITH_ICONV
"If build iconv on Windows"
ON
)
ENDIF () ENDIF ()
IF(${TD_LINUX} MATCHES TRUE) IF(${TD_LINUX} MATCHES TRUE)

12
cmake/iconv_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,12 @@
# iconv
ExternalProject_Add(iconv
GIT_REPOSITORY https://github.com/win-iconv/win-iconv.git
GIT_TAG v0.0.8
SOURCE_DIR "${CMAKE_CONTRIB_DIR}/iconv"
BINARY_DIR ""
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

20
contrib/CMakeLists.txt
View File

@ -83,6 +83,11 @@ if(${BUILD_WITH_NURAFT})
cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_NURAFT}) endif(${BUILD_WITH_NURAFT})
# iconv
if(${BUILD_WITH_ICONV})
cat("${CMAKE_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_ICONV})
# download dependencies # download dependencies
configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt") configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@ -208,9 +213,10 @@ endif(${BUILD_WITH_TRAFT})
# LIBUV # LIBUV
if(${BUILD_WITH_UV}) if(${BUILD_WITH_UV})
if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") if (${TD_WINDOWS})
MESSAGE("Windows need set no-sign-compare") file(READ "libuv/include/uv.h" CONTENTS)
add_compile_options(-Wno-sign-compare) string(REGEX REPLACE "/([\r]*)\nstruct uv_tcp_s {" "/\\1\ntypedef BOOL (PASCAL *LPFN_CONNECTEX) (SOCKET s, const struct sockaddr* name, int namelen, PVOID lpSendBuffer, DWORD dwSendDataLength,LPDWORD lpdwBytesSent, LPOVERLAPPED lpOverlapped);\\1\nstruct uv_tcp_s {" CONTENTS_NEW "${CONTENTS}")
file(WRITE "libuv/include/uv.h" "${CONTENTS_NEW}")
endif () endif ()
add_subdirectory(libuv) add_subdirectory(libuv)
endif(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV})
@ -243,7 +249,15 @@ if(${BUILD_WITH_SQLITE})
endif(${BUILD_WITH_SQLITE}) endif(${BUILD_WITH_SQLITE})
# pthread # pthread
if(${BUILD_PTHREAD})
add_definitions(-DPTW32_STATIC_LIB)
add_subdirectory(pthread)
endif(${BUILD_PTHREAD})
# iconv
if(${BUILD_WITH_ICONV})
add_subdirectory(iconv)
endif(${BUILD_WITH_ICONV})
# ================================================================================================ # ================================================================================================
# Build test # Build test

6
include/client/taos.h
View File

@ -257,6 +257,12 @@ DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_co
void tmqShowMsg(tmq_message_t *tmq_message); void tmqShowMsg(tmq_message_t *tmq_message);
int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message);
/* ---------------------- OTHER ---------------------------- */
typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);

72
include/common/tcommon.h
View File

@ -62,18 +62,17 @@ typedef struct SDataBlockInfo {
union {int64_t uid; int64_t blockId;}; union {int64_t uid; int64_t blockId;};
} SDataBlockInfo; } SDataBlockInfo;
typedef struct SConstantItem { //typedef struct SConstantItem {
SColumnInfo info; // SColumnInfo info;
int32_t startRow; // run-length-encoding to save the space for multiple rows // int32_t startRow; // run-length-encoding to save the space for multiple rows
int32_t endRow; // int32_t endRow;
SVariant value; // SVariant value;
} SConstantItem; //} SConstantItem;
// info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList);
typedef struct SSDataBlock { typedef struct SSDataBlock {
SColumnDataAgg *pBlockAgg; SColumnDataAgg *pBlockAgg;
SArray *pDataBlock; // SArray<SColumnInfoData> SArray *pDataBlock; // SArray<SColumnInfoData>
SArray *pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
SDataBlockInfo info; SDataBlockInfo info;
} SSDataBlock; } SSDataBlock;
@ -95,66 +94,15 @@ typedef struct SColumnInfoData {
}; };
} SColumnInfoData; } SColumnInfoData;
static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { void* blockDataDestroy(SSDataBlock* pBlock);
int64_t tbUid = pBlock->info.uid; int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
int16_t numOfCols = pBlock->info.numOfCols; void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
int16_t hasVarCol = pBlock->info.hasVarCol;
int32_t rows = pBlock->info.rows;
int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, tbUid);
tlen += taosEncodeFixedI16(buf, numOfCols);
tlen += taosEncodeFixedI16(buf, hasVarCol);
tlen += taosEncodeFixedI32(buf, rows);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
tlen += taosEncodeFixedI16(buf, pColData->info.colId);
tlen += taosEncodeFixedI16(buf, pColData->info.type);
tlen += taosEncodeFixedI32(buf, pColData->info.bytes);
int32_t colSz = rows * pColData->info.bytes;
tlen += taosEncodeBinary(buf, pColData->pData, colSz);
}
return tlen;
}
static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
int32_t sz;
buf = taosDecodeFixedI64(buf, &pBlock->info.uid);
buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols);
buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol);
buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
buf = taosDecodeFixedI32(buf, &sz);
pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData));
for (int32_t i = 0; i < sz; i++) {
SColumnInfoData data = {0};
buf = taosDecodeFixedI16(buf, &data.info.colId);
buf = taosDecodeFixedI16(buf, &data.info.type);
buf = taosDecodeFixedI32(buf, &data.info.bytes);
int32_t colSz = pBlock->info.rows * data.info.bytes;
buf = taosDecodeBinary(buf, (void**)&data.pData, colSz);
taosArrayPush(pBlock->pDataBlock, &data);
}
return (void*)buf;
}
static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) {
if (pBlock == NULL) { if (pBlock == NULL) {
return; return;
} }
blockDataDestroy(pBlock);
// int32_t numOfOutput = pBlock->info.numOfCols;
int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < sz; ++i) {
SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
tfree(pColInfoData->pData);
}
taosArrayDestroy(pBlock->pDataBlock);
tfree(pBlock->pBlockAgg);
// tfree(pBlock);
} }
static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {

17
include/common/tdatablock.h
View File

@ -52,6 +52,21 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
} while (0) } while (0)
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (!pColumnInfoData->hasNull) {
return false;
}
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return pColumnInfoData->varmeta.offset[row] == -1;
} else {
if (pColumnInfoData->nullbitmap == NULL) {
return false;
}
return colDataIsNull_f(pColumnInfoData->nullbitmap, row);
}
}
static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row,
SColumnDataAgg* pColAgg) { SColumnDataAgg* pColAgg) {
if (!pColumnInfoData->hasNull) { if (!pColumnInfoData->hasNull) {
@ -81,7 +96,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) #define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
// SColumnInfoData, rowNumber
#define colDataGetData(p1_, r_) \ #define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
: ((p1_)->pData + ((r_) * (p1_)->info.bytes))) : ((p1_)->pData + ((r_) * (p1_)->info.bytes)))

131
include/common/tmsg.h
View File

@ -24,6 +24,7 @@
#include "thash.h" #include "thash.h"
#include "tlist.h" #include "tlist.h"
#include "trow.h" #include "trow.h"
#include "tuuid.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -114,11 +115,12 @@ typedef enum _mgmt_table {
#define TSDB_ALTER_TABLE_DROP_TAG 2 #define TSDB_ALTER_TABLE_DROP_TAG 2
#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 #define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 #define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_ADD_COLUMN 5 #define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6 #define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7
#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 #define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8
#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10
#define TSDB_FILL_NONE 0 #define TSDB_FILL_NONE 0
#define TSDB_FILL_NULL 1 #define TSDB_FILL_NULL 1
@ -196,6 +198,11 @@ typedef struct {
}; };
} SMsgHead; } SMsgHead;
typedef struct {
int32_t workerType;
int32_t streamTaskId;
} SStreamExecMsgHead;
// Submit message for one table // Submit message for one table
typedef struct SSubmitBlk { typedef struct SSubmitBlk {
int64_t uid; // table unique id int64_t uid; // table unique id
@ -418,7 +425,7 @@ typedef struct {
}; };
} SColumnFilterList; } SColumnFilterList;
/* /*
* for client side struct, we only need the column id, type, bytes are not necessary * for client side struct, only column id, type, bytes are necessary
* But for data in vnode side, we need all the following information. * But for data in vnode side, we need all the following information.
*/ */
typedef struct { typedef struct {
@ -560,8 +567,6 @@ int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
void tFreeSQnodeListRsp(SQnodeListRsp* pRsp); void tFreeSQnodeListRsp(SQnodeListRsp* pRsp);
typedef struct { typedef struct {
SArray* pArray; // Array of SUseDbRsp SArray* pArray; // Array of SUseDbRsp
} SUseDbBatchRsp; } SUseDbBatchRsp;
@ -777,7 +782,6 @@ typedef struct SVgroupInfo {
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
} SVgroupInfo; } SVgroupInfo;
typedef struct { typedef struct {
int32_t numOfVgroups; int32_t numOfVgroups;
SVgroupInfo vgroups[]; SVgroupInfo vgroups[];
@ -1072,13 +1076,11 @@ typedef struct {
int8_t action; int8_t action;
} STaskAction; } STaskAction;
typedef struct SQueryNodeEpId { typedef struct SQueryNodeEpId {
int32_t nodeId; // vgId or qnodeId int32_t nodeId; // vgId or qnodeId
SEp ep; SEp ep;
} SQueryNodeEpId; } SQueryNodeEpId;
typedef struct { typedef struct {
SMsgHead header; SMsgHead header;
uint64_t sId; uint64_t sId;
@ -1090,7 +1092,6 @@ int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pR
int32_t tDeserializeSSchedulerHbReq(void* buf, int32_t bufLen, SSchedulerHbReq* pReq); int32_t tDeserializeSSchedulerHbReq(void* buf, int32_t bufLen, SSchedulerHbReq* pReq);
void tFreeSSchedulerHbReq(SSchedulerHbReq* pReq); void tFreeSSchedulerHbReq(SSchedulerHbReq* pReq);
typedef struct { typedef struct {
uint64_t seqId; uint64_t seqId;
SQueryNodeEpId epId; SQueryNodeEpId epId;
@ -1101,7 +1102,6 @@ int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pR
int32_t tDeserializeSSchedulerHbRsp(void* buf, int32_t bufLen, SSchedulerHbRsp* pRsp); int32_t tDeserializeSSchedulerHbRsp(void* buf, int32_t bufLen, SSchedulerHbRsp* pRsp);
void tFreeSSchedulerHbRsp(SSchedulerHbRsp* pRsp); void tFreeSSchedulerHbRsp(SSchedulerHbRsp* pRsp);
typedef struct { typedef struct {
SMsgHead header; SMsgHead header;
uint64_t sId; uint64_t sId;
@ -1157,8 +1157,8 @@ typedef struct {
char name[TSDB_TOPIC_FNAME_LEN]; char name[TSDB_TOPIC_FNAME_LEN];
int8_t igExists; int8_t igExists;
char* sql; char* sql;
char* physicalPlan; char* ast;
char* logicalPlan; char subscribeDbName[TSDB_DB_NAME_LEN];
} SCMCreateTopicReq; } SCMCreateTopicReq;
int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq);
@ -1896,33 +1896,18 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW)
} }
return buf; return buf;
} }
typedef enum {
TD_TIME_UNIT_UNKNOWN = -1,
TD_TIME_UNIT_YEAR = 0,
TD_TIME_UNIT_SEASON = 1,
TD_TIME_UNIT_MONTH = 2,
TD_TIME_UNIT_WEEK = 3,
TD_TIME_UNIT_DAY = 4,
TD_TIME_UNIT_HOUR = 5,
TD_TIME_UNIT_MINUTE = 6,
TD_TIME_UNIT_SEC = 7,
TD_TIME_UNIT_MILLISEC = 8,
TD_TIME_UNIT_MICROSEC = 9,
TD_TIME_UNIT_NANOSEC = 10
} ETDTimeUnit;
typedef struct { typedef struct {
int8_t version; // for compatibility(default 0) int8_t version; // for compatibility(default 0)
int8_t intervalUnit; int8_t intervalUnit; // MACRO: TIME_UNIT_XXX
int8_t slidingUnit; int8_t slidingUnit; // MACRO: TIME_UNIT_XXX
char indexName[TSDB_INDEX_NAME_LEN]; char indexName[TSDB_INDEX_NAME_LEN];
char timezone[TD_TIMEZONE_LEN]; // sma data is invalid if timezone change. char timezone[TD_TIMEZONE_LEN]; // sma data expired if timezone changes.
int32_t exprLen; int32_t exprLen;
int32_t tagsFilterLen; int32_t tagsFilterLen;
int64_t indexUid; int64_t indexUid;
tb_uid_t tableUid; // super/child/common table uid tb_uid_t tableUid; // super/child/common table uid
int64_t interval; int64_t interval;
int64_t offset; int64_t offset; // use unit by precision of DB
int64_t sliding; int64_t sliding;
char* expr; // sma expression char* expr; // sma expression
char* tagsFilter; char* tagsFilter;
@ -1967,7 +1952,7 @@ typedef struct {
typedef struct { typedef struct {
int64_t indexUid; int64_t indexUid;
TSKEY skey; // startTS of one interval/sliding TSKEY skey; // startKey of one interval/sliding window
int64_t interval; int64_t interval;
int32_t dataLen; // not including head int32_t dataLen; // not including head
int8_t intervalUnit; int8_t intervalUnit;
@ -2068,27 +2053,19 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
buf = taosDecodeFixedI64(buf, &pSma->sliding); buf = taosDecodeFixedI64(buf, &pSma->sliding);
if (pSma->exprLen > 0) { if (pSma->exprLen > 0) {
pSma->expr = (char*)calloc(pSma->exprLen, 1); if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) {
if (pSma->expr != NULL) {
buf = taosDecodeStringTo(buf, pSma->expr);
} else {
tdDestroyTSma(pSma); tdDestroyTSma(pSma);
return NULL; return NULL;
} }
} else { } else {
pSma->expr = NULL; pSma->expr = NULL;
} }
if (pSma->tagsFilterLen > 0) { if (pSma->tagsFilterLen > 0) {
pSma->tagsFilter = (char*)calloc(pSma->tagsFilterLen, 1); if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) {
if (pSma->tagsFilter != NULL) {
buf = taosDecodeStringTo(buf, pSma->tagsFilter);
} else {
tdDestroyTSma(pSma); tdDestroyTSma(pSma);
return NULL; return NULL;
} }
} else { } else {
pSma->tagsFilter = NULL; pSma->tagsFilter = NULL;
} }
@ -2179,6 +2156,26 @@ typedef struct {
SArray* topics; // SArray<SMqSubTopicEp> SArray* topics; // SArray<SMqSubTopicEp>
} SMqCMGetSubEpRsp; } SMqCMGetSubEpRsp;
typedef struct {
SMqRspHead head;
union {
SMqPollRsp consumeRsp;
SMqCMGetSubEpRsp getEpRsp;
};
void* extra;
} SMqMsgWrapper;
typedef struct {
int32_t curBlock;
int32_t curRow;
void** uData;
} SMqRowIter;
struct tmq_message_t_v1 {
SMqPollRsp rsp;
SMqRowIter iter;
};
struct tmq_message_t { struct tmq_message_t {
SMqRspHead head; SMqRspHead head;
union { union {
@ -2186,6 +2183,9 @@ struct tmq_message_t {
SMqCMGetSubEpRsp getEpRsp; SMqCMGetSubEpRsp getEpRsp;
}; };
void* extra; void* extra;
int32_t curBlock;
int32_t curRow;
void** uData;
}; };
static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
@ -2267,6 +2267,55 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p
return buf; return buf;
} }
enum {
STREAM_TASK_STATUS__RUNNING = 1,
STREAM_TASK_STATUS__STOP,
};
typedef struct {
int64_t streamId;
int32_t taskId;
int32_t level;
int8_t status;
char* qmsg;
void* executor;
// void* stateStore;
// storage handle
} SStreamTask;
static FORCE_INLINE SStreamTask* streamTaskNew(int64_t streamId, int32_t level) {
SStreamTask* pTask = (SStreamTask*)calloc(1, sizeof(SStreamTask));
if (pTask == NULL) {
return NULL;
}
pTask->taskId = tGenIdPI32();
pTask->status = STREAM_TASK_STATUS__RUNNING;
pTask->qmsg = NULL;
return pTask;
}
int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask);
void tFreeSStreamTask(SStreamTask* pTask);
typedef struct {
SMsgHead head;
SStreamTask* task;
} SStreamTaskDeployReq;
typedef struct {
int32_t reserved;
} SStreamTaskDeployRsp;
typedef struct {
SStreamExecMsgHead head;
// TODO: other info needed by task
} SStreamTaskExecReq;
typedef struct {
int32_t reserved;
} SStreamTaskExecRsp;
#pragma pack(pop) #pragma pack(pop)
#ifdef __cplusplus #ifdef __cplusplus

3
include/common/tmsgdef.h
View File

@ -189,6 +189,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp)
TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
@ -199,6 +200,8 @@ enum {
// Requests handled by SNODE // Requests handled by SNODE
TD_NEW_MSG_SEG(TDMT_SND_MSG) TD_NEW_MSG_SEG(TDMT_SND_MSG)
TD_DEF_MSG_TYPE(TDMT_SND_TASK_DEPLOY, "snode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
TD_DEF_MSG_TYPE(TDMT_SND_TASK_EXEC, "snode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
#if defined(TD_MSG_NUMBER_) #if defined(TD_MSG_NUMBER_)
TDMT_MAX TDMT_MAX

4
include/common/trow.h
View File

@ -697,9 +697,9 @@ static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, int16_t c
} }
// TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme.
if (TD_IS_TP_ROW(pRow)) { if (TD_IS_TP_ROW(pRow)) {
tdAppendColValToTpRow(pBuilder, valType, val, true, colType, colIdx, offset); tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset);
} else { } else {
tdAppendColValToKvRow(pBuilder, valType, val, true, colType, colIdx, offset, colId); tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

262
include/common/ttokendef.h
View File

@ -34,127 +34,147 @@
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_USER 19 #define TK_ACCOUNT 19
#define TK_PASS 20 #define TK_NK_ID 20
#define TK_NK_STRING 21 #define TK_PASS 21
#define TK_ALTER 22 #define TK_NK_STRING 22
#define TK_PRIVILEGE 23 #define TK_ALTER 23
#define TK_DROP 24 #define TK_PPS 24
#define TK_SHOW 25 #define TK_TSERIES 25
#define TK_USERS 26 #define TK_STORAGE 26
#define TK_DNODE 27 #define TK_STREAMS 27
#define TK_PORT 28 #define TK_QTIME 28
#define TK_NK_INTEGER 29 #define TK_DBS 29
#define TK_DNODES 30 #define TK_USERS 30
#define TK_NK_ID 31 #define TK_CONNS 31
#define TK_NK_IPTOKEN 32 #define TK_STATE 32
#define TK_QNODE 33 #define TK_USER 33
#define TK_ON 34 #define TK_PRIVILEGE 34
#define TK_QNODES 35 #define TK_DROP 35
#define TK_DATABASE 36 #define TK_SHOW 36
#define TK_DATABASES 37 #define TK_DNODE 37
#define TK_USE 38 #define TK_PORT 38
#define TK_IF 39 #define TK_NK_INTEGER 39
#define TK_NOT 40 #define TK_DNODES 40
#define TK_EXISTS 41 #define TK_NK_IPTOKEN 41
#define TK_BLOCKS 42 #define TK_LOCAL 42
#define TK_CACHE 43 #define TK_QNODE 43
#define TK_CACHELAST 44 #define TK_ON 44
#define TK_COMP 45 #define TK_QNODES 45
#define TK_DAYS 46 #define TK_DATABASE 46
#define TK_FSYNC 47 #define TK_DATABASES 47
#define TK_MAXROWS 48 #define TK_USE 48
#define TK_MINROWS 49 #define TK_IF 49
#define TK_KEEP 50 #define TK_NOT 50
#define TK_PRECISION 51 #define TK_EXISTS 51
#define TK_QUORUM 52 #define TK_BLOCKS 52
#define TK_REPLICA 53 #define TK_CACHE 53
#define TK_TTL 54 #define TK_CACHELAST 54
#define TK_WAL 55 #define TK_COMP 55
#define TK_VGROUPS 56 #define TK_DAYS 56
#define TK_SINGLE_STABLE 57 #define TK_FSYNC 57
#define TK_STREAM_MODE 58 #define TK_MAXROWS 58
#define TK_TABLE 59 #define TK_MINROWS 59
#define TK_NK_LP 60 #define TK_KEEP 60
#define TK_NK_RP 61 #define TK_PRECISION 61
#define TK_STABLE 62 #define TK_QUORUM 62
#define TK_TABLES 63 #define TK_REPLICA 63
#define TK_STABLES 64 #define TK_TTL 64
#define TK_USING 65 #define TK_WAL 65
#define TK_TAGS 66 #define TK_VGROUPS 66
#define TK_NK_DOT 67 #define TK_SINGLE_STABLE 67
#define TK_NK_COMMA 68 #define TK_STREAM_MODE 68
#define TK_COMMENT 69 #define TK_RETENTIONS 69
#define TK_BOOL 70 #define TK_FILE_FACTOR 70
#define TK_TINYINT 71 #define TK_NK_FLOAT 71
#define TK_SMALLINT 72 #define TK_TABLE 72
#define TK_INT 73 #define TK_NK_LP 73
#define TK_INTEGER 74 #define TK_NK_RP 74
#define TK_BIGINT 75 #define TK_STABLE 75
#define TK_FLOAT 76 #define TK_TABLES 76
#define TK_DOUBLE 77 #define TK_STABLES 77
#define TK_BINARY 78 #define TK_ADD 78
#define TK_TIMESTAMP 79 #define TK_COLUMN 79
#define TK_NCHAR 80 #define TK_MODIFY 80
#define TK_UNSIGNED 81 #define TK_RENAME 81
#define TK_JSON 82 #define TK_TAG 82
#define TK_VARCHAR 83 #define TK_SET 83
#define TK_MEDIUMBLOB 84 #define TK_NK_EQ 84
#define TK_BLOB 85 #define TK_USING 85
#define TK_VARBINARY 86 #define TK_TAGS 86
#define TK_DECIMAL 87 #define TK_NK_DOT 87
#define TK_SMA 88 #define TK_NK_COMMA 88
#define TK_INDEX 89 #define TK_COMMENT 89
#define TK_FULLTEXT 90 #define TK_BOOL 90
#define TK_FUNCTION 91 #define TK_TINYINT 91
#define TK_INTERVAL 92 #define TK_SMALLINT 92
#define TK_MNODES 93 #define TK_INT 93
#define TK_NK_FLOAT 94 #define TK_INTEGER 94
#define TK_NK_BOOL 95 #define TK_BIGINT 95
#define TK_NK_VARIABLE 96 #define TK_FLOAT 96
#define TK_BETWEEN 97 #define TK_DOUBLE 97
#define TK_IS 98 #define TK_BINARY 98
#define TK_NULL 99 #define TK_TIMESTAMP 99
#define TK_NK_LT 100 #define TK_NCHAR 100
#define TK_NK_GT 101 #define TK_UNSIGNED 101
#define TK_NK_LE 102 #define TK_JSON 102
#define TK_NK_GE 103 #define TK_VARCHAR 103
#define TK_NK_NE 104 #define TK_MEDIUMBLOB 104
#define TK_NK_EQ 105 #define TK_BLOB 105
#define TK_LIKE 106 #define TK_VARBINARY 106
#define TK_MATCH 107 #define TK_DECIMAL 107
#define TK_NMATCH 108 #define TK_SMA 108
#define TK_IN 109 #define TK_ROLLUP 109
#define TK_FROM 110 #define TK_INDEX 110
#define TK_AS 111 #define TK_FULLTEXT 111
#define TK_JOIN 112 #define TK_FUNCTION 112
#define TK_INNER 113 #define TK_INTERVAL 113
#define TK_SELECT 114 #define TK_TOPIC 114
#define TK_DISTINCT 115 #define TK_AS 115
#define TK_WHERE 116 #define TK_MNODES 116
#define TK_PARTITION 117 #define TK_NK_BOOL 117
#define TK_BY 118 #define TK_NK_VARIABLE 118
#define TK_SESSION 119 #define TK_BETWEEN 119
#define TK_STATE_WINDOW 120 #define TK_IS 120
#define TK_SLIDING 121 #define TK_NULL 121
#define TK_FILL 122 #define TK_NK_LT 122
#define TK_VALUE 123 #define TK_NK_GT 123
#define TK_NONE 124 #define TK_NK_LE 124
#define TK_PREV 125 #define TK_NK_GE 125
#define TK_LINEAR 126 #define TK_NK_NE 126
#define TK_NEXT 127 #define TK_LIKE 127
#define TK_GROUP 128 #define TK_MATCH 128
#define TK_HAVING 129 #define TK_NMATCH 129
#define TK_ORDER 130 #define TK_IN 130
#define TK_SLIMIT 131 #define TK_FROM 131
#define TK_SOFFSET 132 #define TK_JOIN 132
#define TK_LIMIT 133 #define TK_INNER 133
#define TK_OFFSET 134 #define TK_SELECT 134
#define TK_ASC 135 #define TK_DISTINCT 135
#define TK_DESC 136 #define TK_WHERE 136
#define TK_NULLS 137 #define TK_PARTITION 137
#define TK_FIRST 138 #define TK_BY 138
#define TK_LAST 139 #define TK_SESSION 139
#define TK_STATE_WINDOW 140
#define TK_SLIDING 141
#define TK_FILL 142
#define TK_VALUE 143
#define TK_NONE 144
#define TK_PREV 145
#define TK_LINEAR 146
#define TK_NEXT 147
#define TK_GROUP 148
#define TK_HAVING 149
#define TK_ORDER 150
#define TK_SLIMIT 151
#define TK_SOFFSET 152
#define TK_LIMIT 153
#define TK_OFFSET 154
#define TK_ASC 155
#define TK_DESC 156
#define TK_NULLS 157
#define TK_FIRST 158
#define TK_LAST 159
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301

2
include/common/ttypes.h
View File

@ -27,7 +27,7 @@ extern "C" {
typedef int32_t VarDataOffsetT; typedef int32_t VarDataOffsetT;
typedef uint32_t TDRowLenT; typedef uint32_t TDRowLenT;
typedef uint8_t TDRowValT; typedef uint8_t TDRowValT;
typedef uint16_t col_id_t; typedef int16_t col_id_t;
typedef int8_t col_type_t; typedef int8_t col_type_t;
#pragma pack(push, 1) #pragma pack(push, 1)

2
include/common/tvariant.h
View File

@ -31,7 +31,7 @@ typedef struct SVariant {
uint64_t u; uint64_t u;
double d; double d;
char *pz; char *pz;
wchar_t *wpz; TdUcs4 *ucs4;
SArray *arr; // only for 'in' query to hold value list, not value for a field SArray *arr; // only for 'in' query to hold value list, not value for a field
}; };
} SVariant; } SVariant;

3
include/dnode/snode/snode.h
View File

@ -16,6 +16,7 @@
#ifndef _TD_SNODE_H_ #ifndef _TD_SNODE_H_
#define _TD_SNODE_H_ #define _TD_SNODE_H_
#include "tcommon.h"
#include "tmsg.h" #include "tmsg.h"
#include "trpc.h" #include "trpc.h"
@ -78,7 +79,7 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad);
* @param pRsp The response message * @param pRsp The response message
* @return int32_t 0 for success, -1 for failure * @return int32_t 0 for success, -1 for failure
*/ */
int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); // int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg); int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg);

56
include/libs/nodes/cmdnodes.h
View File

@ -61,6 +61,12 @@ typedef struct SDropDatabaseStmt {
bool ignoreNotExists; bool ignoreNotExists;
} SDropDatabaseStmt; } SDropDatabaseStmt;
typedef struct SAlterDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
SDatabaseOptions* pOptions;
} SAlterDatabaseStmt;
typedef struct STableOptions { typedef struct STableOptions {
ENodeType type; ENodeType type;
int32_t keep; int32_t keep;
@ -121,6 +127,18 @@ typedef struct SDropSuperTableStmt {
bool ignoreNotExists; bool ignoreNotExists;
} SDropSuperTableStmt; } SDropSuperTableStmt;
typedef struct SAlterTableStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
int8_t alterType;
char colName[TSDB_COL_NAME_LEN];
char newColName[TSDB_COL_NAME_LEN];
STableOptions* pOptions;
SDataType dataType;
SValueNode* pVal;
} SAlterTableStmt;
typedef struct SCreateUserStmt { typedef struct SCreateUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char useName[TSDB_USER_LEN];
@ -152,6 +170,13 @@ typedef struct SDropDnodeStmt {
int32_t port; int32_t port;
} SDropDnodeStmt; } SDropDnodeStmt;
typedef struct SAlterDnodeStmt {
ENodeType type;
int32_t dnodeId;
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN];
} SAlterDnodeStmt;
typedef struct SShowStmt { typedef struct SShowStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
@ -179,11 +204,42 @@ typedef struct SCreateIndexStmt {
SIndexOptions* pOptions; SIndexOptions* pOptions;
} SCreateIndexStmt; } SCreateIndexStmt;
typedef struct SDropIndexStmt {
ENodeType type;
char indexName[TSDB_INDEX_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
} SDropIndexStmt;
typedef struct SCreateQnodeStmt { typedef struct SCreateQnodeStmt {
ENodeType type; ENodeType type;
int32_t dnodeId; int32_t dnodeId;
} SCreateQnodeStmt; } SCreateQnodeStmt;
typedef struct SDropQnodeStmt {
ENodeType type;
int32_t dnodeId;
} SDropQnodeStmt;
typedef struct SCreateTopicStmt {
ENodeType type;
char topicName[TSDB_TABLE_NAME_LEN];
char subscribeDbName[TSDB_DB_NAME_LEN];
bool ignoreExists;
SNode* pQuery;
} SCreateTopicStmt;
typedef struct SDropTopicStmt {
ENodeType type;
char topicName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists;
} SDropTopicStmt;
typedef struct SAlterLocalStmt {
ENodeType type;
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN];
} SAlterLocalStmt;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

9
include/libs/nodes/nodes.h
View File

@ -77,6 +77,7 @@ typedef enum ENodeType {
QUERY_NODE_VNODE_MODIF_STMT, QUERY_NODE_VNODE_MODIF_STMT,
QUERY_NODE_CREATE_DATABASE_STMT, QUERY_NODE_CREATE_DATABASE_STMT,
QUERY_NODE_DROP_DATABASE_STMT, QUERY_NODE_DROP_DATABASE_STMT,
QUERY_NODE_ALTER_DATABASE_STMT,
QUERY_NODE_SHOW_DATABASES_STMT, // temp QUERY_NODE_SHOW_DATABASES_STMT, // temp
QUERY_NODE_CREATE_TABLE_STMT, QUERY_NODE_CREATE_TABLE_STMT,
QUERY_NODE_CREATE_SUBTABLE_CLAUSE, QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
@ -84,6 +85,7 @@ typedef enum ENodeType {
QUERY_NODE_DROP_TABLE_CLAUSE, QUERY_NODE_DROP_TABLE_CLAUSE,
QUERY_NODE_DROP_TABLE_STMT, QUERY_NODE_DROP_TABLE_STMT,
QUERY_NODE_DROP_SUPER_TABLE_STMT, QUERY_NODE_DROP_SUPER_TABLE_STMT,
QUERY_NODE_ALTER_TABLE_STMT,
QUERY_NODE_SHOW_TABLES_STMT, // temp QUERY_NODE_SHOW_TABLES_STMT, // temp
QUERY_NODE_SHOW_STABLES_STMT, QUERY_NODE_SHOW_STABLES_STMT,
QUERY_NODE_CREATE_USER_STMT, QUERY_NODE_CREATE_USER_STMT,
@ -93,12 +95,18 @@ typedef enum ENodeType {
QUERY_NODE_USE_DATABASE_STMT, QUERY_NODE_USE_DATABASE_STMT,
QUERY_NODE_CREATE_DNODE_STMT, QUERY_NODE_CREATE_DNODE_STMT,
QUERY_NODE_DROP_DNODE_STMT, QUERY_NODE_DROP_DNODE_STMT,
QUERY_NODE_ALTER_DNODE_STMT,
QUERY_NODE_SHOW_DNODES_STMT, QUERY_NODE_SHOW_DNODES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT, QUERY_NODE_SHOW_VGROUPS_STMT,
QUERY_NODE_SHOW_MNODES_STMT, QUERY_NODE_SHOW_MNODES_STMT,
QUERY_NODE_SHOW_QNODES_STMT, QUERY_NODE_SHOW_QNODES_STMT,
QUERY_NODE_CREATE_INDEX_STMT, QUERY_NODE_CREATE_INDEX_STMT,
QUERY_NODE_DROP_INDEX_STMT,
QUERY_NODE_CREATE_QNODE_STMT, QUERY_NODE_CREATE_QNODE_STMT,
QUERY_NODE_DROP_QNODE_STMT,
QUERY_NODE_CREATE_TOPIC_STMT,
QUERY_NODE_DROP_TOPIC_STMT,
QUERY_NODE_ALTER_LOCAL_STMT,
// logic plan node // logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN, QUERY_NODE_LOGIC_PLAN_SCAN,
@ -158,6 +166,7 @@ SNodeList* nodesMakeList();
int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
void nodesDestroyList(SNodeList* pList); void nodesDestroyList(SNodeList* pList);

8
include/libs/nodes/plannodes.h
View File

@ -26,7 +26,6 @@ extern "C" {
typedef struct SLogicNode { typedef struct SLogicNode {
ENodeType type; ENodeType type;
int32_t id;
SNodeList* pTargets; // SColumnNode SNodeList* pTargets; // SColumnNode
SNode* pConditions; SNode* pConditions;
SNodeList* pChildren; SNodeList* pChildren;
@ -69,7 +68,7 @@ typedef struct SProjectLogicNode {
} SProjectLogicNode; } SProjectLogicNode;
typedef struct SVnodeModifLogicNode { typedef struct SVnodeModifLogicNode {
SLogicNode node;; SLogicNode node;
int32_t msgType; int32_t msgType;
SArray* pDataBlocks; SArray* pDataBlocks;
SVgDataBlocks* pVgDataBlocks; SVgDataBlocks* pVgDataBlocks;
@ -124,7 +123,7 @@ typedef struct SSubLogicPlan {
} SSubLogicPlan; } SSubLogicPlan;
typedef struct SQueryLogicPlan { typedef struct SQueryLogicPlan {
ENodeType type;; ENodeType type;
int32_t totalLevel; int32_t totalLevel;
SNodeList* pTopSubplans; SNodeList* pTopSubplans;
} SQueryLogicPlan; } SQueryLogicPlan;
@ -167,6 +166,7 @@ typedef struct SScanPhysiNode {
typedef SScanPhysiNode SSystemTableScanPhysiNode; typedef SScanPhysiNode SSystemTableScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode;
typedef SScanPhysiNode SStreamScanPhysiNode;
typedef struct STableScanPhysiNode { typedef struct STableScanPhysiNode {
SScanPhysiNode scan; SScanPhysiNode scan;
@ -252,7 +252,7 @@ typedef struct SSubplan {
} SSubplan; } SSubplan;
typedef struct SQueryPlan { typedef struct SQueryPlan {
ENodeType type;; ENodeType type;
uint64_t queryId; uint64_t queryId;
int32_t numOfSubplans; int32_t numOfSubplans;
SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.

3
include/libs/nodes/querynodes.h
View File

@ -23,6 +23,9 @@ extern "C" {
#include "nodes.h" #include "nodes.h"
#include "tmsg.h" #include "tmsg.h"
#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) * sizeof(SSchema)))
#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
typedef struct SRawExprNode { typedef struct SRawExprNode {
ENodeType nodeType; ENodeType nodeType;
char* p; char* p;

1
include/libs/parser/parser.h
View File

@ -26,6 +26,7 @@ typedef struct SParseContext {
uint64_t requestId; uint64_t requestId;
int32_t acctId; int32_t acctId;
const char *db; const char *db;
bool streamQuery;
void *pTransporter; void *pTransporter;
SEpSet mgmtEpSet; SEpSet mgmtEpSet;
const char *pSql; // sql string const char *pSql; // sql string

4
include/libs/qcom/query.h
View File

@ -49,6 +49,10 @@ typedef struct STableComInfo {
int32_t rowSize; // row size of the schema int32_t rowSize; // row size of the schema
} STableComInfo; } STableComInfo;
typedef struct SIndexMeta {
} SIndexMeta;
/* /*
* ASSERT(sizeof(SCTableMeta) == 24) * ASSERT(sizeof(SCTableMeta) == 24)
* ASSERT(tableType == TSDB_CHILD_TABLE) * ASSERT(tableType == TSDB_CHILD_TABLE)

10
include/libs/tfs/tfs.h
View File

@ -198,6 +198,16 @@ void tfsBasename(const STfsFile *pFile, char *dest);
*/ */
void tfsDirname(const STfsFile *pFile, char *dest); void tfsDirname(const STfsFile *pFile, char *dest);
/**
* @brief Get the absolute file name of rname.
*
* @param pTfs
* @param diskId
* @param rname relative file name
* @param aname absolute file name
*/
void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname);
/** /**
* @brief Remove file in tfs. * @brief Remove file in tfs.
* *

14
include/os/os.h
View File

@ -25,10 +25,11 @@ extern "C" {
#include <pthread.h> #include <pthread.h>
#include <semaphore.h> #include <semaphore.h>
#include <regex.h>
#if !defined(WINDOWS) #if !defined(WINDOWS)
#include <unistd.h> #include <unistd.h>
#include <dirent.h> #include <dirent.h>
#include <regex.h>
#include <sched.h> #include <sched.h>
#include <wordexp.h> #include <wordexp.h>
#include <libgen.h> #include <libgen.h>
@ -36,6 +37,12 @@ extern "C" {
#include <sys/utsname.h> #include <sys/utsname.h>
#include <sys/param.h> #include <sys/param.h>
#include <sys/mman.h> #include <sys/mman.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <termios.h>
#include <sys/statvfs.h>
#if defined(DARWIN) #if defined(DARWIN)
#else #else
@ -61,12 +68,7 @@ extern "C" {
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <wchar.h> #include <wchar.h>
#include <termios.h>
#include <wctype.h> #include <wctype.h>

1
include/os/osDef.h
View File

@ -56,6 +56,7 @@ extern "C" {
// specific // specific
typedef int (*__compar_fn_t)(const void *, const void *); typedef int (*__compar_fn_t)(const void *, const void *);
#define ssize_t int #define ssize_t int
#define _SSIZE_T_
#define bzero(ptr, size) memset((ptr), 0, (size)) #define bzero(ptr, size) memset((ptr), 0, (size))
#define strcasecmp _stricmp #define strcasecmp _stricmp
#define strncasecmp _strnicmp #define strncasecmp _strnicmp

31
include/os/osFile.h
View File

@ -22,15 +22,6 @@ extern "C" {
#include "osSocket.h" #include "osSocket.h"
#if defined(WINDOWS)
typedef int32_t FileFd;
typedef SOCKET SocketFd;
#else
typedef int32_t FileFd;
typedef int32_t SocketFd;
#endif
int64_t taosRead(FileFd fd, void *buf, int64_t count);
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID #define open OPEN_FUNC_TAOS_FORBID
@ -42,6 +33,7 @@ int64_t taosRead(FileFd fd, void *buf, int64_t count);
#define close CLOSE_FUNC_TAOS_FORBID #define close CLOSE_FUNC_TAOS_FORBID
#define fclose FCLOSE_FUNC_TAOS_FORBID #define fclose FCLOSE_FUNC_TAOS_FORBID
#define fsync FSYNC_FUNC_TAOS_FORBID #define fsync FSYNC_FUNC_TAOS_FORBID
#define getline GETLINE_FUNC_TAOS_FORBID
// #define fflush FFLUSH_FUNC_TAOS_FORBID // #define fflush FFLUSH_FUNC_TAOS_FORBID
#endif #endif
@ -49,15 +41,6 @@ int64_t taosRead(FileFd fd, void *buf, int64_t count);
#define PATH_MAX 256 #define PATH_MAX 256
#endif #endif
typedef int32_t FileFd;
typedef struct TdFile {
pthread_rwlock_t rwlock;
int refId;
FileFd fd;
FILE *fp;
} * TdFilePtr, TdFile;
typedef struct TdFile *TdFilePtr; typedef struct TdFile *TdFilePtr;
#define TD_FILE_CTEATE 0x0001 #define TD_FILE_CTEATE 0x0001
@ -95,10 +78,6 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...); void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
#if defined(WINDOWS)
#define __restrict__
#endif // WINDOWS
int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
int32_t taosEOFFile(TdFilePtr pFile); int32_t taosEOFFile(TdFilePtr pFile);
@ -111,15 +90,7 @@ int32_t taosRemoveFile(const char *path);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
#if defined(_TD_DARWIN_64)
typedef int32_t SocketFd;
int64_t taosSendFile(SocketFd fdDst, FileFd pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(FILE *pFileOut, FILE *pFileIn, int64_t *offset, int64_t size);
#else
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
#endif
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
bool taosValidFile(TdFilePtr pFile); bool taosValidFile(TdFilePtr pFile);

2
include/os/osSocket.h
View File

@ -25,6 +25,8 @@
#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
#define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
#define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
#define inet_addr INET_ADDR_FUNC_TAOS_FORBID
#define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
#endif #endif
#if defined(WINDOWS) #if defined(WINDOWS)

40
include/os/osString.h
View File

@ -20,16 +20,28 @@
extern "C" { extern "C" {
#endif #endif
typedef wchar_t TdWchar;
typedef int32_t TdUcs4;
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC
#define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID
#define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID
#define iconv ICONV_FUNC_TAOS_FORBID
#define wcwidth WCWIDTH_FUNC_TAOS_FORBID
#define wcswidth WCSWIDTH_FUNC_TAOS_FORBID
#define mbtowc MBTOWC_FUNC_TAOS_FORBID
#define mbstowcs MBSTOWCS_FUNC_TAOS_FORBID
#define wctomb WCTOMB_FUNC_TAOS_FORBID
#define wcstombs WCSTOMBS_FUNC_TAOS_FORBID
#define wcsncpy WCSNCPY_FUNC_TAOS_FORBID
#define wchar_t WCHAR_T_FUNC_TAOS_FORBID
#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#define tstrdup(str) _strdup(str) #define tstrdup(str) _strdup(str)
#define tstrndup(str, size) _strndup(str, size)
int32_t tgetline(char **lineptr, size_t *n, FILE *stream);
int32_t twcslen(const wchar_t *wcs);
#else #else
#define tstrdup(str) strdup(str) #define tstrdup(str) strdup(str)
#define tstrndup(str, size) strndup(str, size)
#define tgetline(lineptr, n, stream) getline(lineptr, n, stream)
#define twcslen wcslen
#endif #endif
#define tstrncpy(dst, src, size) \ #define tstrncpy(dst, src, size) \
@ -38,14 +50,22 @@ extern "C" {
(dst)[(size)-1] = 0; \ (dst)[(size)-1] = 0; \
} while (0) } while (0)
int32_t taosUcs4len(TdUcs4 *ucs4);
int64_t taosStr2int64(const char *str); int64_t taosStr2int64(const char *str);
// USE_LIBICONV int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs);
int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
bool taosValidateEncodec(const char *encodec); bool taosValidateEncodec(const char *encodec);
int32_t taosWcharWidth(TdWchar wchar);
int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size);
int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size);
int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size);
int32_t taosWcharToMb(char *pStr, TdWchar wchar);
int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
include/os/osSysinfo.h
View File

@ -16,7 +16,6 @@
#ifndef _TD_OS_SYSINFO_H_ #ifndef _TD_OS_SYSINFO_H_
#define _TD_OS_SYSINFO_H_ #define _TD_OS_SYSINFO_H_
#include <sys/statvfs.h>
#include "os.h" #include "os.h"
#ifdef __cplusplus #ifdef __cplusplus

2
include/util/taoserror.h
View File

@ -354,6 +354,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615)
#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616)
#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0617) #define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0617)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0618)
// query // query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)
@ -481,6 +482,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_PASSWD_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611) #define TSDB_CODE_PAR_PASSWD_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611)
#define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612) #define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612)
#define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613) #define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613)
#define TSDB_CODE_PAR_EXPRIE_STATEMENT TAOS_DEF_ERROR_CODE(0, 0x2614)
#ifdef __cplusplus #ifdef __cplusplus
} }

2
include/util/tcompare.h
View File

@ -46,7 +46,7 @@ typedef struct SPatternCompareInfo {
int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo); int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); int32_t WCSPatternMatch(const TdUcs4 *pattern, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t taosArrayCompareString(const void *a, const void *b); int32_t taosArrayCompareString(const void *a, const void *b);

7
include/util/tdef.h
View File

@ -41,7 +41,7 @@ extern const int32_t TYPE_BYTES[15];
#define DOUBLE_BYTES sizeof(double) #define DOUBLE_BYTES sizeof(double)
#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) #define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
#define TSDB_KEYSIZE sizeof(TSKEY) #define TSDB_KEYSIZE sizeof(TSKEY)
#define TSDB_NCHAR_SIZE sizeof(int32_t) #define TSDB_NCHAR_SIZE sizeof(TdUcs4)
// NULL definition // NULL definition
#define TSDB_DATA_BOOL_NULL 0x02 #define TSDB_DATA_BOOL_NULL 0x02
@ -448,6 +448,11 @@ typedef struct {
#define SND_UNIQUE_THREAD_NUM 2 #define SND_UNIQUE_THREAD_NUM 2
#define SND_SHARED_THREAD_NUM 2 #define SND_SHARED_THREAD_NUM 2
enum {
SND_WORKER_TYPE__SHARED = 1,
SND_WORKER_TYPE__UNIQUE,
};
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

25
include/util/tencode.h
View File

@ -75,7 +75,7 @@ typedef struct {
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
#define TCODER_MALLOC(SIZE, CODER) TFL_MALLOC(SIZE, &((CODER)->fl)) #define TCODER_MALLOC(PTR, TYPE, SIZE, CODER) TFL_MALLOC(PTR, TYPE, SIZE, &((CODER)->fl))
void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type);
void tCoderClear(SCoder* pCoder); void tCoderClear(SCoder* pCoder);
@ -402,6 +402,29 @@ static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val) {
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeBinaryAlloc(SCoder* pDecoder, void** val, uint64_t* len) {
if (tDecodeU64v(pDecoder, len) < 0) return -1;
if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1;
*val = malloc(*len);
if (*val == NULL) return -1;
memcpy(*val, TD_CODER_CURRENT(pDecoder), *len);
TD_CODER_MOVE_POS(pDecoder, *len);
return 0;
}
static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SCoder* pDecoder, char** val, uint64_t* len) {
if (tDecodeBinaryAlloc(pDecoder, (void**)val, len) < 0) return -1;
(*len) -= 1;
return 0;
}
static FORCE_INLINE int32_t tDecodeCStrAlloc(SCoder* pDecoder, char** val) {
uint64_t len;
return tDecodeCStrAndLenAlloc(pDecoder, val, &len);
}
static FORCE_INLINE bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); } static FORCE_INLINE bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); }
#ifdef __cplusplus #ifdef __cplusplus

10
include/util/tfreelist.h
View File

@ -29,15 +29,17 @@ struct SFreeListNode {
typedef TD_SLIST(SFreeListNode) SFreeList; typedef TD_SLIST(SFreeListNode) SFreeList;
#define TFL_MALLOC(SIZE, LIST) \ #define TFL_MALLOC(PTR, TYPE, SIZE, LIST) \
({ \ do { \
void *ptr = malloc((SIZE) + sizeof(struct SFreeListNode)); \ void *ptr = malloc((SIZE) + sizeof(struct SFreeListNode)); \
if (ptr) { \ if (ptr) { \
TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \ TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \
ptr = ((struct SFreeListNode *)ptr)->payload; \ ptr = ((struct SFreeListNode *)ptr)->payload; \
(PTR) = (TYPE)(ptr); \
}else{ \
(PTR) = NULL; \
} \ } \
ptr; \ }while(0);
})
#define tFreeListInit(pFL) TD_SLIST_INIT(pFL) #define tFreeListInit(pFL) TD_SLIST_INIT(pFL)

2
include/util/tjson.h
View File

@ -44,6 +44,7 @@ int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal);
int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal); int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal);
int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal); int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal);
int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal); int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal);
int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal);
int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal); int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal);
int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal); int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal);
int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal); int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal);
@ -60,6 +61,7 @@ int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void*
typedef int32_t (*FToObject)(const SJson* pJson, void* pObj); typedef int32_t (*FToObject)(const SJson* pJson, void* pObj);
int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj); int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj);
int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize);
int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize); int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize);
char* tjsonToString(const SJson* pJson); char* tjsonToString(const SJson* pJson);

2
include/util/types.h
View File

@ -82,7 +82,7 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 32767
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
#define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataLen(v) ((VarDataLenT *)(v))[0]
#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) #define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE)
typedef int32_t VarDataOffsetT; typedef int32_t VarDataOffsetT;

3
source/client/inc/clientInt.h
View File

@ -179,6 +179,7 @@ typedef struct SRequestObj {
uint64_t requestId; uint64_t requestId;
int32_t type; // request type int32_t type; // request type
STscObj* pTscObj; STscObj* pTscObj;
char* pDb;
char* sqlstr; // sql string char* sqlstr; // sql string
int32_t sqlLen; int32_t sqlLen;
int64_t self; int64_t self;
@ -229,7 +230,7 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t
int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery); int32_t parseSql(SRequestObj* pRequest, bool streamQuery, SQuery** pQuery);
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
// --- heartbeat // --- heartbeat

2
source/client/src/clientEnv.c
View File

@ -149,6 +149,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
return NULL; return NULL;
} }
pRequest->pDb = getDbOfConnection(pObj);
pRequest->requestId = generateRequestId(); pRequest->requestId = generateRequestId();
pRequest->metric.start = taosGetTimestampMs(); pRequest->metric.start = taosGetTimestampMs();
@ -179,6 +180,7 @@ static void doDestroyRequest(void *p) {
tfree(pRequest->msgBuf); tfree(pRequest->msgBuf);
tfree(pRequest->sqlstr); tfree(pRequest->sqlstr);
tfree(pRequest->pInfo); tfree(pRequest->pInfo);
tfree(pRequest->pDb);
doFreeReqResultInfo(&pRequest->body.resInfo); doFreeReqResultInfo(&pRequest->body.resInfo);
qDestroyQueryPlan(pRequest->body.pDag); qDestroyQueryPlan(pRequest->body.pDag);

9
source/client/src/clientImpl.c
View File

@ -137,13 +137,14 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) { int32_t parseSql(SRequestObj* pRequest, bool streamQuery, SQuery** pQuery) {
STscObj* pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
SParseContext cxt = { SParseContext cxt = {
.requestId = pRequest->requestId, .requestId = pRequest->requestId,
.acctId = pTscObj->acctId, .acctId = pTscObj->acctId,
.db = getDbOfConnection(pTscObj), .db = pRequest->pDb,
.streamQuery = streamQuery,
.pSql = pRequest->sqlstr, .pSql = pRequest->sqlstr,
.sqlLen = pRequest->sqlLen, .sqlLen = pRequest->sqlLen,
.pMsg = pRequest->msgBuf, .pMsg = pRequest->msgBuf,
@ -154,7 +155,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tfree(cxt.db);
return code; return code;
} }
@ -163,7 +163,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
} }
tfree(cxt.db);
return code; return code;
} }
@ -249,7 +248,7 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return); CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return);
if (pQuery->directRpc) { if (pQuery->directRpc) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);

62
source/client/src/tmq.c
View File

@ -484,36 +484,22 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
tscDebug("start to create topic, %s", topicName); tscDebug("start to create topic, %s", topicName);
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return);
pQueryNode->streamQuery = true;
// todo check for invalid sql statement and return with error code // todo check for invalid sql statement and return with error code
SSchema* schema = NULL; CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &pStr, NULL), _return);
int32_t numOfCols = 0;
CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, NULL), _return);
pStr = qQueryPlanToString(pRequest->body.pDag);
if (pStr == NULL) {
goto _return;
}
/*printf("%s\n", pStr);*/ /*printf("%s\n", pStr);*/
// The topic should be related to a database that the queried table is belonged to. SName name = { .acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T };
SName name = {0}; strcpy(name.dbname, pRequest->pDb);
char dbName[TSDB_DB_FNAME_LEN] = {0}; strcpy(name.tname, topicName);
// tNameGetFullDbName(&((SQueryStmtInfo*)pQueryNode)->pTableMetaInfo[0]->name, dbName);
tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB);
tNameFromString(&name, topicName, T_NAME_TABLE);
SCMCreateTopicReq req = { SCMCreateTopicReq req = {
.igExists = 1, .igExists = 1,
.physicalPlan = (char*)pStr, .ast = (char*)pStr,
.sql = (char*)sql, .sql = (char*)sql,
.logicalPlan = (char*)"no logic plan",
}; };
tNameExtractFullName(&name, req.name); tNameExtractFullName(&name, req.name);
@ -700,6 +686,10 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
} }
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
pRsp->curBlock = 0;
pRsp->curRow = 0;
// TODO: alloc mem
/*pRsp->*/
/*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
if (pRsp->consumeRsp.numOfTopics == 0) { if (pRsp->consumeRsp.numOfTopics == 0) {
/*printf("no data\n");*/ /*printf("no data\n");*/
@ -758,9 +748,9 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
goto END; goto END;
} }
// tmq's epoch is monotomically increase, // tmq's epoch is monotonically increase,
// so it's safe to discard any old epoch msg. // so it's safe to discard any old epoch msg.
// epoch will only increase when received newer epoch ep msg // Epoch will only increase when received newer epoch ep msg
SMqRspHead* head = pMsg->pData; SMqRspHead* head = pMsg->pData;
int32_t epoch = atomic_load_32(&tmq->epoch); int32_t epoch = atomic_load_32(&tmq->epoch);
if (head->epoch <= epoch) { if (head->epoch <= epoch) {
@ -1282,6 +1272,34 @@ const char* tmq_err2str(tmq_resp_err_t err) {
return "fail"; return "fail";
} }
TAOS_ROW tmq_get_row(tmq_message_t* message) {
SMqPollRsp* rsp = &message->consumeRsp;
while (1) {
if (message->curBlock < taosArrayGetSize(rsp->pBlockData)) {
SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->curBlock);
if (message->curRow < pBlock->info.rows) {
for (int i = 0; i < pBlock->info.numOfCols; i++) {
SColumnInfoData* pData = taosArrayGet(pBlock->pDataBlock, i);
if (colDataIsNull_s(pData, message->curRow))
message->uData[i] = NULL;
else {
message->uData[i] = colDataGetData(pData, message->curRow);
}
}
message->curRow++;
return message->uData;
} else {
message->curBlock++;
message->curRow = 0;
continue;
}
}
return NULL;
}
}
char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; }
#if 0 #if 0
tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) {
tmq_t* pTmq = malloc(sizeof(tmq_t)); tmq_t* pTmq = malloc(sizeof(tmq_t));

77
source/common/src/tdatablock.c
View File

@ -240,10 +240,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
} }
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) {
ASSERT(pBlock); ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock));
size_t constantCols = (pBlock->pConstantList != NULL)? taosArrayGetSize(pBlock->pConstantList):0;
ASSERT( pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock) + constantCols);
return pBlock->info.numOfCols; return pBlock->info.numOfCols;
} }
@ -1075,6 +1072,10 @@ void blockDataClearup(SSDataBlock* pDataBlock) {
} }
int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
if (0 == numOfRows) {
return TSDB_CODE_SUCCESS;
}
if (IS_VAR_DATA_TYPE(pColumn->info.type)) { if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
if (tmp == NULL) { if (tmp == NULL) {
@ -1095,7 +1096,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo
pColumn->nullbitmap = tmp; pColumn->nullbitmap = tmp;
memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows));
assert(pColumn->info.bytes);
tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes);
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -1140,7 +1141,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
taosArrayDestroy(pBlock->pDataBlock); taosArrayDestroy(pBlock->pDataBlock);
tfree(pBlock->pBlockAgg); tfree(pBlock->pBlockAgg);
tfree(pBlock); // tfree(pBlock);
return NULL; return NULL;
} }
@ -1166,3 +1167,67 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) {
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock)); return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock));
} }
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
int64_t tbUid = pBlock->info.uid;
int16_t numOfCols = pBlock->info.numOfCols;
int16_t hasVarCol = pBlock->info.hasVarCol;
int32_t rows = pBlock->info.rows;
int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, tbUid);
tlen += taosEncodeFixedI16(buf, numOfCols);
tlen += taosEncodeFixedI16(buf, hasVarCol);
tlen += taosEncodeFixedI32(buf, rows);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
tlen += taosEncodeFixedI16(buf, pColData->info.colId);
tlen += taosEncodeFixedI16(buf, pColData->info.type);
tlen += taosEncodeFixedI32(buf, pColData->info.bytes);
if (IS_VAR_DATA_TYPE(pColData->info.type)) {
tlen += taosEncodeBinary(buf, pColData->varmeta.offset, sizeof(int32_t) * rows);
} else {
tlen += taosEncodeBinary(buf, pColData->nullbitmap, BitmapLen(rows));
}
int32_t len = colDataGetLength(pColData, rows);
tlen += taosEncodeFixedI32(buf, len);
tlen += taosEncodeBinary(buf, pColData->pData, len);
}
return tlen;
}
void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
int32_t sz;
buf = taosDecodeFixedI64(buf, &pBlock->info.uid);
buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols);
buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol);
buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
buf = taosDecodeFixedI32(buf, &sz);
pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData));
for (int32_t i = 0; i < sz; i++) {
SColumnInfoData data = {0};
buf = taosDecodeFixedI16(buf, &data.info.colId);
buf = taosDecodeFixedI16(buf, &data.info.type);
buf = taosDecodeFixedI32(buf, &data.info.bytes);
if (IS_VAR_DATA_TYPE(data.info.type)) {
buf = taosDecodeBinary(buf, (void**)&data.varmeta.offset, pBlock->info.rows * sizeof(int32_t));
data.varmeta.length = pBlock->info.rows * sizeof(int32_t);
data.varmeta.allocLen = data.varmeta.length;
} else {
buf = taosDecodeBinary(buf, (void**)&data.nullbitmap, BitmapLen(pBlock->info.rows));
}
int32_t len = 0;
buf = taosDecodeFixedI32(buf, &len);
buf = taosDecodeBinary(buf, (void**)&data.pData, len);
taosArrayPush(pBlock->pDataBlock, &data);
}
return (void*)buf;
}

79
source/common/src/tmsg.c
View File

@ -1467,7 +1467,6 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) {
return 0; return 0;
} }
int32_t tSerializeSQnodeListReq(void *buf, int32_t bufLen, SQnodeListReq *pReq) { int32_t tSerializeSQnodeListReq(void *buf, int32_t bufLen, SQnodeListReq *pReq) {
SCoder encoder = {0}; SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@ -1990,11 +1989,9 @@ int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pR
int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTopicReq *pReq) { int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTopicReq *pReq) {
int32_t sqlLen = 0; int32_t sqlLen = 0;
int32_t physicalPlanLen = 0; int32_t astLen = 0;
int32_t logicalPlanLen = 0;
if (pReq->sql != NULL) sqlLen = (int32_t)strlen(pReq->sql); if (pReq->sql != NULL) sqlLen = (int32_t)strlen(pReq->sql);
if (pReq->physicalPlan != NULL) physicalPlanLen = (int32_t)strlen(pReq->physicalPlan); if (pReq->ast != NULL) astLen = (int32_t)strlen(pReq->ast);
if (pReq->logicalPlan != NULL) logicalPlanLen = (int32_t)strlen(pReq->logicalPlan);
SCoder encoder = {0}; SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@ -2003,11 +2000,9 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
if (tEncodeI32(&encoder, sqlLen) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
if (tEncodeI32(&encoder, physicalPlanLen) < 0) return -1; if (tEncodeI32(&encoder, astLen) < 0) return -1;
if (tEncodeI32(&encoder, logicalPlanLen) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
@ -2018,8 +2013,7 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo
int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicReq *pReq) { int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicReq *pReq) {
int32_t sqlLen = 0; int32_t sqlLen = 0;
int32_t physicalPlanLen = 0; int32_t astLen = 0;
int32_t logicalPlanLen = 0;
SCoder decoder = {0}; SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
@ -2028,17 +2022,20 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
if (tDecodeI32(&decoder, &physicalPlanLen) < 0) return -1; if (tDecodeI32(&decoder, &astLen) < 0) return -1;
if (tDecodeI32(&decoder, &logicalPlanLen) < 0) return -1;
if (sqlLen > 0) {
pReq->sql = calloc(1, sqlLen + 1); pReq->sql = calloc(1, sqlLen + 1);
pReq->physicalPlan = calloc(1, physicalPlanLen + 1); if (pReq->sql == NULL) return -1;
pReq->logicalPlan = calloc(1, logicalPlanLen + 1);
if (pReq->sql == NULL || pReq->physicalPlan == NULL || pReq->logicalPlan == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->physicalPlan) < 0) return -1; }
if (tDecodeCStrTo(&decoder, pReq->logicalPlan) < 0) return -1;
if (astLen > 0) {
pReq->ast = calloc(1, astLen + 1);
if (pReq->ast == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1;
}
tEndDecode(&decoder); tEndDecode(&decoder);
tCoderClear(&decoder); tCoderClear(&decoder);
@ -2047,8 +2044,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) {
tfree(pReq->sql); tfree(pReq->sql);
tfree(pReq->physicalPlan); tfree(pReq->ast);
tfree(pReq->logicalPlan);
} }
int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) { int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) {
@ -2471,7 +2467,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq
int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) {
if (tStartDecode(decoder) < 0) return -1; if (tStartDecode(decoder) < 0) return -1;
if (tDecodeI32(decoder, &pReq->num) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); TCODER_MALLOC(pReq->offsets, SMqOffset*, pReq->num * sizeof(SMqOffset), decoder);
if (pReq->offsets == NULL) return -1; if (pReq->offsets == NULL) return -1;
for (int32_t i = 0; i < pReq->num; i++) { for (int32_t i = 0; i < pReq->num; i++) {
tDecodeSMqOffset(decoder, &pReq->offsets[i]); tDecodeSMqOffset(decoder, &pReq->offsets[i]);
@ -2559,8 +2555,6 @@ int32_t tDeserializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *
void tFreeSSchedulerHbReq(SSchedulerHbReq *pReq) { taosArrayDestroy(pReq->taskAction); } void tFreeSSchedulerHbReq(SSchedulerHbReq *pReq) { taosArrayDestroy(pReq->taskAction); }
int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pRsp) { int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pRsp) {
SCoder encoder = {0}; SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@ -2680,9 +2674,9 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
if (tStartDecode(&decoder) < 0) return -1; if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
if (tDecodeCStr(&decoder, (const char **)&pReq->sql) < 0) return -1; if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1;
if (tDecodeCStr(&decoder, (const char **)&pReq->physicalPlan) < 0) return -1; if (tDecodeCStrAlloc(&decoder, &pReq->physicalPlan) < 0) return -1;
if (tDecodeCStr(&decoder, (const char **)&pReq->logicalPlan) < 0) return -1; if (tDecodeCStrAlloc(&decoder, &pReq->logicalPlan) < 0) return -1;
tEndDecode(&decoder); tEndDecode(&decoder);
tCoderClear(&decoder); tCoderClear(&decoder);
@ -2694,3 +2688,32 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
tfree(pReq->physicalPlan); tfree(pReq->physicalPlan);
tfree(pReq->logicalPlan); tfree(pReq->logicalPlan);
} }
int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) {
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->level) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1;
tEndEncode(pEncoder);
return pEncoder->pos;
}
int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) {
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1;
tEndDecode(pDecoder);
return 0;
}
void tFreeSStreamTask(SStreamTask *pTask) {
// TODO
/*free(pTask->qmsg);*/
/*free(pTask->executor);*/
/*free(pTask);*/
}

2
source/common/src/tname.c
View File

@ -217,7 +217,7 @@ int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t name
} }
int32_t tNameSetAcctId(SName* dst, int32_t acctId) { int32_t tNameSetAcctId(SName* dst, int32_t acctId) {
assert(dst != NULL && acct != NULL); assert(dst != NULL);
dst->acctId = acctId; dst->acctId = acctId;
return 0; return 0;
} }

4
source/common/src/trow.c
View File

@ -353,10 +353,10 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
for (int i = 0; i < src2->numOfCols; i++) { for (int i = 0; i < src2->numOfCols; i++) {
SCellVal sVal = {0}; SCellVal sVal = {0};
ASSERT(target->cols[i].type == src2->cols[i].type); ASSERT(target->cols[i].type == src2->cols[i].type);
if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) { if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2) < 0) {
if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) {
TASSERT(0); TASSERT(0);
} }
if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) {
tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints); tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints);
} else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) { if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) {

44
source/common/src/tvariant.c
View File

@ -199,8 +199,8 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin
case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length
size_t lenInwchar = len / TSDB_NCHAR_SIZE; size_t lenInwchar = len / TSDB_NCHAR_SIZE;
pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); pVar->ucs4 = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE);
memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); memcpy(pVar->ucs4, pz, lenInwchar * TSDB_NCHAR_SIZE);
pVar->nLen = (int32_t)len; pVar->nLen = (int32_t)len;
break; break;
@ -343,7 +343,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) {
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
dst[0] = '\''; dst[0] = '\'';
taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); taosUcs4ToMbs(pVar->ucs4, (taosUcs4len(pVar->ucs4) + 1) * TSDB_NCHAR_SIZE, dst + 1);
int32_t len = (int32_t)strlen(dst); int32_t len = (int32_t)strlen(dst);
dst[len] = '\''; dst[len] = '\'';
dst[len + 1] = 0; dst[len + 1] = 0;
@ -384,7 +384,7 @@ static FORCE_INLINE int32_t convertToBoolImpl(char *pStr, int32_t len) {
} }
} }
static FORCE_INLINE int32_t wcsconvertToBoolImpl(wchar_t *pstr, int32_t len) { static FORCE_INLINE int32_t wcsconvertToBoolImpl(TdUcs4 *pstr, int32_t len) {
if ((wcsncasecmp(pstr, L"true", len) == 0) && (len == 4)) { if ((wcsncasecmp(pstr, L"true", len) == 0) && (len == 4)) {
return TSDB_TRUE; return TSDB_TRUE;
} else if (wcsncasecmp(pstr, L"false", len) == 0 && (len == 5)) { } else if (wcsncasecmp(pstr, L"false", len) == 0 && (len == 5)) {
@ -412,11 +412,11 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
pBuf = realloc(pBuf, newSize + 1); pBuf = realloc(pBuf, newSize + 1);
} }
taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf); taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, pBuf);
free(pVariant->wpz); free(pVariant->ucs4);
pBuf[newSize] = 0; pBuf[newSize] = 0;
} else { } else {
taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest); taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, *pDest);
} }
} else { } else {
@ -460,8 +460,8 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
} }
if (*pDest == pVariant->pz) { if (*pDest == pVariant->pz) {
wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); TdUcs4 *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE);
bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); bool ret = taosMbsToUcs4(pDst, nLen, pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
if (!ret) { if (!ret) {
tfree(pWStr); tfree(pWStr);
return -1; return -1;
@ -469,21 +469,21 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
// free the binary buffer in the first place // free the binary buffer in the first place
if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
free(pVariant->wpz); free(pVariant->ucs4);
} }
pVariant->wpz = pWStr; pVariant->ucs4 = pWStr;
*pDestSize = twcslen(pVariant->wpz); *pDestSize = taosUcs4len(pVariant->ucs4);
// shrink the allocate memory, no need to check here. // shrink the allocate memory, no need to check here.
char *tmp = realloc(pVariant->wpz, (*pDestSize + 1) * TSDB_NCHAR_SIZE); char *tmp = realloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE);
assert(tmp != NULL); assert(tmp != NULL);
pVariant->wpz = (wchar_t *)tmp; pVariant->ucs4 = (TdUcs4 *)tmp;
} else { } else {
int32_t output = 0; int32_t output = 0;
bool ret = taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output); bool ret = taosMbsToUcs4(pDst, nLen, (TdUcs4*)*pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output);
if (!ret) { if (!ret) {
return -1; return -1;
} }
@ -554,7 +554,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result
*result = res; *result = res;
} else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) {
errno = 0; errno = 0;
wchar_t *endPtr = NULL; TdUcs4 *endPtr = NULL;
SToken token = {0}; SToken token = {0};
token.n = tGetToken(pVariant->pz, &token.type); token.n = tGetToken(pVariant->pz, &token.type);
@ -564,7 +564,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result
} }
if (token.type == TK_FLOAT) { if (token.type == TK_FLOAT) {
double v = wcstod(pVariant->wpz, &endPtr); double v = wcstod(pVariant->ucs4, &endPtr);
if (releaseVariantPtr) { if (releaseVariantPtr) {
free(pVariant->pz); free(pVariant->pz);
pVariant->nLen = 0; pVariant->nLen = 0;
@ -583,7 +583,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result
setNull((char *)result, type, tDataTypes[type].bytes); setNull((char *)result, type, tDataTypes[type].bytes);
return 0; return 0;
} else { } else {
int64_t val = wcstoll(pVariant->wpz, &endPtr, 10); int64_t val = wcstoll(pVariant->ucs4, &endPtr, 10);
if (releaseVariantPtr) { if (releaseVariantPtr) {
free(pVariant->pz); free(pVariant->pz);
pVariant->nLen = 0; pVariant->nLen = 0;
@ -649,7 +649,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) {
*pDest = ret; *pDest = ret;
} else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) {
int32_t ret = 0; int32_t ret = 0;
if ((ret = wcsconvertToBoolImpl(pVariant->wpz, pVariant->nLen)) < 0) { if ((ret = wcsconvertToBoolImpl(pVariant->ucs4, pVariant->nLen)) < 0) {
return ret; return ret;
} }
*pDest = ret; *pDest = ret;
@ -899,7 +899,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
return -1; return -1;
} }
} else { } else {
wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen); tasoUcs4Copy((TdUcs4*)payload, pVariant->ucs4, pVariant->nLen);
} }
} }
} else { } else {
@ -913,7 +913,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
return -1; return -1;
} }
} else { } else {
memcpy(p, pVariant->wpz, pVariant->nLen); memcpy(p, pVariant->ucs4, pVariant->nLen);
newlen = pVariant->nLen; newlen = pVariant->nLen;
} }
@ -979,7 +979,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) {
pVariant->d = v; pVariant->d = v;
} else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) {
errno = 0; errno = 0;
double v = wcstod(pVariant->wpz, NULL); double v = wcstod(pVariant->ucs4, NULL);
if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) {
free(pVariant->pz); free(pVariant->pz);
return -1; return -1;

2
source/dnode/mgmt/daemon/src/dmnMain.c
View File

@ -27,7 +27,7 @@ static struct {
} dmn = {0}; } dmn = {0};
static void dmnSigintHandle(int signum, void *info, void *ctx) { static void dmnSigintHandle(int signum, void *info, void *ctx) {
uInfo("singal:%d is received", signum); uInfo("signal:%d is received", signum);
dmn.stop = true; dmn.stop = true;
} }

7
source/dnode/mgmt/impl/inc/dndSnode.h
View File

@ -24,10 +24,15 @@ extern "C" {
int32_t dndInitSnode(SDnode *pDnode); int32_t dndInitSnode(SDnode *pDnode);
void dndCleanupSnode(SDnode *pDnode); void dndCleanupSnode(SDnode *pDnode);
void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); // void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

41
source/dnode/mgmt/impl/src/dndSnode.c
View File

@ -323,7 +323,7 @@ int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
} }
static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) {
SSnodeMgmt *pMgmt = &pDnode->smgmt; /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/
int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
SSnode *pSnode = dndAcquireSnode(pDnode); SSnode *pSnode = dndAcquireSnode(pDnode);
@ -337,19 +337,32 @@ static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
}
dndReleaseSnode(pDnode, pSnode); dndReleaseSnode(pDnode, pSnode);
} else {
for (int32_t i = 0; i < numOfMsgs; i++) {
SRpcMsg *pMsg = NULL;
taosGetQitem(qall, (void **)&pMsg);
SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rpcRsp);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
}
} }
static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) { static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) {
SSnodeMgmt *pMgmt = &pDnode->smgmt; /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/
int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
SSnode *pSnode = dndAcquireSnode(pDnode); SSnode *pSnode = dndAcquireSnode(pDnode);
if (pSnode != NULL) { if (pSnode != NULL) {
code = sndProcessSMsg(pSnode, pMsg); sndProcessSMsg(pSnode, pMsg);
}
dndReleaseSnode(pDnode, pSnode); dndReleaseSnode(pDnode, pSnode);
} else {
SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rpcRsp);
}
#if 0 #if 0
if (pMsg->msgType & 1u) { if (pMsg->msgType & 1u) {
@ -369,6 +382,12 @@ static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) {
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
static FORCE_INLINE int32_t dndGetSWTypeFromMsg(SRpcMsg *pMsg) {
SStreamExecMsgHead *pHead = pMsg->pCont;
pHead->workerType = htonl(pHead->workerType);
return pHead->workerType;
}
static FORCE_INLINE int32_t dndGetSWIdFromMsg(SRpcMsg *pMsg) { static FORCE_INLINE int32_t dndGetSWIdFromMsg(SRpcMsg *pMsg) {
SMsgHead *pHead = pMsg->pCont; SMsgHead *pHead = pMsg->pCont;
pHead->streamTaskId = htonl(pHead->streamTaskId); pHead->streamTaskId = htonl(pHead->streamTaskId);
@ -437,6 +456,18 @@ void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
dndWriteSnodeMsgToMgmtWorker(pDnode, pMsg); dndWriteSnodeMsgToMgmtWorker(pDnode, pMsg);
} }
void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SSnode *pSnode = dndAcquireSnode(pDnode);
if (pSnode != NULL) {
int32_t workerType = dndGetSWTypeFromMsg(pMsg);
if (workerType == SND_WORKER_TYPE__SHARED) {
dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg);
} else {
dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg);
}
}
}
void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg); dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg);
} }

5
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -23,6 +23,7 @@
#include "dndTransport.h" #include "dndTransport.h"
#include "dndMgmt.h" #include "dndMgmt.h"
#include "dndMnode.h" #include "dndMnode.h"
#include "dndSnode.h"
#include "dndVnodes.h" #include "dndVnodes.h"
#define INTERNAL_USER "_dnd" #define INTERNAL_USER "_dnd"
@ -153,6 +154,10 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_HEARTBEAT)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_HEARTBEAT)] = dndProcessVnodeFetchMsg;
// Requests handled by SNODE
pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_DEPLOY)] = dndProcessSnodeMgmtMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_EXEC)] = dndProcessSnodeExecMsg;
} }
static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {

10
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -85,6 +85,8 @@ typedef enum {
TRN_TYPE_REBALANCE = 1017, TRN_TYPE_REBALANCE = 1017,
TRN_TYPE_COMMIT_OFFSET = 1018, TRN_TYPE_COMMIT_OFFSET = 1018,
TRN_TYPE_CREATE_STREAM = 1019, TRN_TYPE_CREATE_STREAM = 1019,
TRN_TYPE_DROP_STREAM = 1020,
TRN_TYPE_ALTER_STREAM = 1021,
TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_BASIC_SCOPE_END,
TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_GLOBAL_SCOPE = 2000,
TRN_TYPE_CREATE_DNODE = 2001, TRN_TYPE_CREATE_DNODE = 2001,
@ -679,12 +681,6 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
return buf; return buf;
} }
typedef struct {
int32_t taskId;
int32_t level;
SSubplan* plan;
} SStreamTaskMeta;
typedef struct { typedef struct {
char name[TSDB_TOPIC_FNAME_LEN]; char name[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
@ -700,7 +696,7 @@ typedef struct {
char* sql; char* sql;
char* logicalPlan; char* logicalPlan;
char* physicalPlan; char* physicalPlan;
SArray* tasks; // SArray<SArray<SStreamTaskMeta>> SArray* tasks; // SArray<SArray<SStreamTask>>
} SStreamObj; } SStreamObj;
int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj); int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj);

2
source/dnode/mnode/impl/inc/mndScheduler.h
View File

@ -27,6 +27,8 @@ void mndCleanupScheduler(SMnode* pMnode);
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub); int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub);
int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

40
source/dnode/mnode/impl/src/mndDef.c
View File

@ -27,6 +27,22 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
// TODO encode tasks
if (pObj->tasks) {
int32_t sz = taosArrayGetSize(pObj->tasks);
tEncodeI32(pEncoder, sz);
for (int32_t i = 0; i < sz; i++) {
SArray *pArray = taosArrayGet(pObj->tasks, i);
int32_t innerSz = taosArrayGetSize(pArray);
tEncodeI32(pEncoder, innerSz);
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosArrayGet(pArray, j);
tEncodeSStreamTask(pEncoder, pTask);
}
}
} else {
tEncodeI32(pEncoder, 0);
}
return pEncoder->pos; return pEncoder->pos;
} }
@ -39,8 +55,26 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
if (tDecodeCStr(pDecoder, (const char **)&pObj->sql) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
if (tDecodeCStr(pDecoder, (const char **)&pObj->logicalPlan) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;
if (tDecodeCStr(pDecoder, (const char **)&pObj->physicalPlan) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
int32_t sz;
if (tDecodeI32(pDecoder, &sz) < 0) return -1;
if (sz != 0) {
pObj->tasks = taosArrayInit(sz, sizeof(SArray));
for (int32_t i = 0; i < sz; i++) {
int32_t innerSz;
if (tDecodeI32(pDecoder, &innerSz) < 0) return -1;
SArray *pArray = taosArrayInit(innerSz, sizeof(SStreamTask));
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask task;
if (tDecodeSStreamTask(pDecoder, &task) < 0) return -1;
taosArrayPush(pArray, &task);
}
taosArrayPush(pObj->tasks, pArray);
}
} else {
pObj->tasks = NULL;
}
return 0; return 0;
} }

153
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -31,7 +31,7 @@
#include "tname.h" #include "tname.h"
#include "tuuid.h" #include "tuuid.h"
int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
SSdb* pSdb = pMnode->pSdb; SSdb* pSdb = pMnode->pSdb;
SVgObj* pVgroup = NULL; SVgObj* pVgroup = NULL;
SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
@ -41,17 +41,18 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
} }
ASSERT(pStream->vgNum == 0); ASSERT(pStream->vgNum == 0);
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); int32_t totLevel = LIST_LENGTH(pPlan->pSubplans);
pStream->tasks = taosArrayInit(levelNum, sizeof(SArray)); pStream->tasks = taosArrayInit(totLevel, sizeof(SArray));
for (int32_t i = 0; i < levelNum; i++) { int32_t msgLen;
SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTaskMeta)); for (int32_t level = 0; level < totLevel; level++) {
SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, i); SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTask));
SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level);
int32_t opNum = LIST_LENGTH(inner->pNodeList); int32_t opNum = LIST_LENGTH(inner->pNodeList);
ASSERT(opNum == 1); ASSERT(opNum == 1);
SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); SSubplan* plan = nodesListGetNode(inner->pNodeList, level);
if (i == 0) { if (level == 0) {
ASSERT(plan->type == SUBPLAN_TYPE_SCAN); ASSERT(plan->type == SUBPLAN_TYPE_SCAN);
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
@ -63,15 +64,45 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
} }
pStream->vgNum++; pStream->vgNum++;
// send to vnode
SStreamTask* pTask = streamTaskNew(pStream->uid, level);
plan->execNode.nodeId = pVgroup->vgId; plan->execNode.nodeId = pVgroup->vgId;
plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
SStreamTaskMeta task = { if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) {
.taskId = tGenIdPI32(), sdbRelease(pSdb, pVgroup);
.level = i, qDestroyQueryPlan(pPlan);
.plan = plan, terrno = TSDB_CODE_QRY_INVALID_INPUT;
}; return -1;
// send to vnode }
taosArrayPush(taskOneLevel, &task); taosArrayPush(taskOneLevel, pTask);
SCoder encoder;
tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
tEncodeSStreamTask(&encoder, pTask);
int32_t tlen = sizeof(SMsgHead) + encoder.pos;
tCoderClear(&encoder);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
((SMsgHead*)buf)->streamTaskId = pTask->taskId;
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER);
tEncodeSStreamTask(&encoder, pTask);
tCoderClear(&encoder);
STransAction action = {0};
action.epSet = plan->execNode.epSet;
action.pCont = buf;
action.contLen = tlen;
action.msgType = TDMT_VND_TASK_DEPLOY;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
rpcFreeCont(buf);
return -1;
}
} }
} else if (plan->subplanType == SUBPLAN_TYPE_SCAN) { } else if (plan->subplanType == SUBPLAN_TYPE_SCAN) {
// duplicatable // duplicatable
@ -82,22 +113,88 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
// if has snode, set to shared thread num in snode // if has snode, set to shared thread num in snode
parallel = SND_SHARED_THREAD_NUM; parallel = SND_SHARED_THREAD_NUM;
for (int32_t j = 0; j < parallel; j++) { for (int32_t i = 0; i < parallel; i++) {
SStreamTaskMeta task = { SStreamTask* pTask = streamTaskNew(pStream->uid, level);
.taskId = tGenIdPI32(),
.level = i, // TODO:get snode id and ep
.plan = plan, plan->execNode.nodeId = pVgroup->vgId;
}; plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
taosArrayPush(taskOneLevel, &task);
if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) {
qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
taosArrayPush(taskOneLevel, pTask);
SCoder encoder;
tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
tEncodeSStreamTask(&encoder, pTask);
int32_t tlen = sizeof(SMsgHead) + encoder.pos;
tCoderClear(&encoder);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
((SMsgHead*)buf)->streamTaskId = pTask->taskId;
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER);
tEncodeSStreamTask(&encoder, pTask);
tCoderClear(&encoder);
STransAction action = {0};
action.epSet = plan->execNode.epSet;
action.pCont = buf;
action.contLen = tlen;
action.msgType = TDMT_SND_TASK_DEPLOY;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
rpcFreeCont(buf);
return -1;
}
} }
} else { } else {
// not duplicatable // not duplicatable
SStreamTaskMeta task = { SStreamTask* pTask = streamTaskNew(pStream->uid, level);
.taskId = tGenIdPI32(),
.level = i, // TODO:get snode id and ep
.plan = plan, plan->execNode.nodeId = pVgroup->vgId;
}; plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
taosArrayPush(taskOneLevel, &task);
if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) {
sdbRelease(pSdb, pVgroup);
qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
taosArrayPush(taskOneLevel, pTask);
SCoder encoder;
tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
tEncodeSStreamTask(&encoder, pTask);
int32_t tlen = sizeof(SMsgHead) + encoder.pos;
tCoderClear(&encoder);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
((SMsgHead*)buf)->streamTaskId = pTask->taskId;
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER);
tEncodeSStreamTask(&encoder, pTask);
tCoderClear(&encoder);
STransAction action = {0};
action.epSet = plan->execNode.epSet;
action.pCont = buf;
action.contLen = tlen;
action.msgType = TDMT_SND_TASK_DEPLOY;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
rpcFreeCont(buf);
return -1;
}
} }
taosArrayPush(pStream->tasks, taskOneLevel); taosArrayPush(pStream->tasks, taskOneLevel);
} }

7
source/dnode/mnode/impl/src/mndStream.c
View File

@ -18,6 +18,7 @@
#include "mndDb.h" #include "mndDb.h"
#include "mndDnode.h" #include "mndDnode.h"
#include "mndMnode.h" #include "mndMnode.h"
#include "mndScheduler.h"
#include "mndShow.h" #include "mndShow.h"
#include "mndStb.h" #include "mndStb.h"
#include "mndTrans.h" #include "mndTrans.h"
@ -229,6 +230,12 @@ static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamR
} }
mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) {
mError("stream:%ld, schedule stream since %s", streamObj.uid, terrstr());
mndTransDrop(pTrans);
return -1;
}
SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj); SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());

40
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -236,6 +236,29 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
return 0; return 0;
} }
static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) {
if (NULL == pCreate->ast) {
return TSDB_CODE_SUCCESS;
}
SNode* pAst = NULL;
int32_t code = nodesStringToNode(pCreate->ast, &pAst);
SQueryPlan* pPlan = NULL;
if (TSDB_CODE_SUCCESS == code) {
SPlanContext cxt = { .pAstRoot = pAst, .streamQuery = true };
code = qCreateQueryPlan(&cxt, &pPlan, NULL);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesNodeToString(pPlan, false, pStr, NULL);
}
nodesDestroyNode(pAst);
nodesDestroyNode(pPlan);
terrno = code;
return code;
}
static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
mDebug("topic:%s to create", pCreate->name); mDebug("topic:%s to create", pCreate->name);
SMqTopicObj topicObj = {0}; SMqTopicObj topicObj = {0};
@ -247,13 +270,23 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq
topicObj.dbUid = pDb->uid; topicObj.dbUid = pDb->uid;
topicObj.version = 1; topicObj.version = 1;
topicObj.sql = pCreate->sql; topicObj.sql = pCreate->sql;
topicObj.physicalPlan = pCreate->physicalPlan; topicObj.physicalPlan = "";
topicObj.logicalPlan = pCreate->logicalPlan; topicObj.logicalPlan = "";
topicObj.sqlLen = strlen(pCreate->sql); topicObj.sqlLen = strlen(pCreate->sql);
char* pPlanStr = NULL;
if (TSDB_CODE_SUCCESS != mndGetPlanString(pCreate, &pPlanStr)) {
mError("topic:%s, failed to get plan since %s", pCreate->name, terrstr());
return -1;
}
if (NULL != pPlanStr) {
topicObj.physicalPlan = pPlanStr;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
tfree(pPlanStr);
return -1; return -1;
} }
mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name);
@ -261,6 +294,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq
SSdbRaw *pRedoRaw = mndTopicActionEncode(&topicObj); SSdbRaw *pRedoRaw = mndTopicActionEncode(&topicObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
tfree(pPlanStr);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
@ -268,10 +302,12 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq
if (mndTransPrepare(pMnode, pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
tfree(pPlanStr);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
tfree(pPlanStr);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return 0;
} }

3
source/dnode/mnode/impl/test/topic/topic.cpp
View File

@ -65,8 +65,7 @@ void* MndTestTopic::BuildCreateTopicReq(const char* topicName, const char* sql,
strcpy(createReq.name, topicName); strcpy(createReq.name, topicName);
createReq.igExists = 0; createReq.igExists = 0;
createReq.sql = (char*)sql; createReq.sql = (char*)sql;
createReq.physicalPlan = (char*)"physicalPlan"; createReq.ast = NULL;
createReq.logicalPlan = (char*)"logicalPlan";
int32_t contLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq); int32_t contLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);

1
source/dnode/snode/CMakeLists.txt
View File

@ -7,6 +7,7 @@ target_include_directories(
) )
target_link_libraries( target_link_libraries(
snode snode
PRIVATE executor
PRIVATE transport PRIVATE transport
PRIVATE os PRIVATE os
PRIVATE common PRIVATE common

32
source/dnode/snode/inc/sndInt.h
View File

@ -38,13 +38,8 @@ enum {
STREAM_STATUS__DELETING, STREAM_STATUS__DELETING,
}; };
enum {
STREAM_TASK_STATUS__RUNNING = 1,
STREAM_TASK_STATUS__STOP,
};
typedef struct { typedef struct {
SHashObj* pHash; // taskId -> streamTask SHashObj* pHash; // taskId -> SStreamTask
} SStreamMeta; } SStreamMeta;
typedef struct SSnode { typedef struct SSnode {
@ -52,26 +47,17 @@ typedef struct SSnode {
SSnodeOpt cfg; SSnodeOpt cfg;
} SSnode; } SSnode;
typedef struct { SStreamMeta* sndMetaNew();
int64_t streamId; void sndMetaDelete(SStreamMeta* pMeta);
int32_t taskId;
int32_t IdxInLevel;
int32_t level;
} SStreamTaskInfo;
typedef struct { int32_t sndMetaDeployTask(SStreamMeta* pMeta, SStreamTask* pTask);
SStreamTaskInfo meta; SStreamTask* sndMetaGetTask(SStreamMeta* pMeta, int32_t taskId);
int8_t status; int32_t sndMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
void* executor;
void* stateStore;
// storage handle
} SStreamTask;
int32_t sndCreateTask(); int32_t sndDropTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
int32_t sndDropTaskOfStream(int64_t streamId);
int32_t sndStopTaskOfStream(int64_t streamId); int32_t sndStopTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
int32_t sndResumeTaskOfStream(int64_t streamId); int32_t sndResumeTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
#ifdef __cplusplus #ifdef __cplusplus
} }

94
source/dnode/snode/src/snode.c
View File

@ -13,44 +13,116 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "executor.h"
#include "sndInt.h" #include "sndInt.h"
#include "tuuid.h" #include "tuuid.h"
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
SSnode *pSnode = calloc(1, sizeof(SSnode)); SSnode *pSnode = calloc(1, sizeof(SSnode));
if (pSnode == NULL) {
return NULL;
}
memcpy(&pSnode->cfg, pOption, sizeof(SSnodeOpt)); memcpy(&pSnode->cfg, pOption, sizeof(SSnodeOpt));
pSnode->pMeta = sndMetaNew();
if (pSnode->pMeta == NULL) {
free(pSnode);
return NULL;
}
return pSnode; return pSnode;
} }
void sndClose(SSnode *pSnode) { free(pSnode); } void sndClose(SSnode *pSnode) {
sndMetaDelete(pSnode->pMeta);
free(pSnode);
}
int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; }
int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { /*int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {*/
*pRsp = NULL; /**pRsp = NULL;*/
return 0; /*return 0;*/
} /*}*/
void sndDestroy(const char *path) {} void sndDestroy(const char *path) {}
static int32_t sndDeployTask(SSnode *pSnode, SRpcMsg *pMsg) { SStreamMeta *sndMetaNew() {
SStreamTask *task = malloc(sizeof(SStreamTask)); SStreamMeta *pMeta = calloc(1, sizeof(SStreamMeta));
if (task == NULL) { if (pMeta == NULL) {
return NULL;
}
pMeta->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (pMeta->pHash == NULL) {
free(pMeta);
return NULL;
}
return pMeta;
}
void sndMetaDelete(SStreamMeta *pMeta) {
taosHashCleanup(pMeta->pHash);
free(pMeta);
}
int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) {
pTask->executor = qCreateStreamExecTaskInfo(pTask->qmsg, NULL);
return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *));
}
SStreamTask *sndMetaGetTask(SStreamMeta *pMeta, int32_t taskId) {
return taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
}
int32_t sndMetaRemoveTask(SStreamMeta *pMeta, int32_t taskId) {
SStreamTask *pTask = taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
if (pTask == NULL) {
return -1;
}
free(pTask->qmsg);
// TODO:free executor
free(pTask);
return taosHashRemove(pMeta->pHash, &taskId, sizeof(int32_t));
}
static int32_t sndProcessTaskExecReq(SSnode *pSnode, SRpcMsg *pMsg) {
SMsgHead *pHead = pMsg->pCont;
int32_t taskId = pHead->streamTaskId;
SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId);
if (pTask == NULL) {
return -1; return -1;
} }
task->meta.taskId = tGenIdPI32();
taosHashPut(pSnode->pMeta->pHash, &task->meta.taskId, sizeof(int32_t), &task, sizeof(void *));
return 0; return 0;
} }
int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) {
// stream deployment // stream deploy
// stream stop/resume // stream stop/resume
// operator exec // operator exec
if (pMsg->msgType == TDMT_SND_TASK_DEPLOY) {
void *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
SStreamTask *pTask = malloc(sizeof(SStreamTask));
if (pTask == NULL) {
return -1;
}
SCoder decoder;
tCoderInit(&decoder, TD_LITTLE_ENDIAN, msg, pMsg->contLen - sizeof(SMsgHead), TD_DECODER);
tDecodeSStreamTask(&decoder, pTask);
tCoderClear(&decoder);
sndMetaDeployTask(pSnode->pMeta, pTask);
} else if (pMsg->msgType == TDMT_SND_TASK_EXEC) {
sndProcessTaskExecReq(pSnode, pMsg);
} else {
ASSERT(0);
}
return 0; return 0;
} }
int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
// operator exec // operator exec
if (pMsg->msgType == TDMT_SND_TASK_EXEC) {
sndProcessTaskExecReq(pSnode, pMsg);
} else {
ASSERT(0);
}
return 0; return 0;
} }

1
source/dnode/vnode/inc/tq.h
View File

@ -55,6 +55,7 @@ int tqCommit(STQ*);
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
int32_t tqProcessRebReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg);
int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
#ifdef __cplusplus #ifdef __cplusplus
} }

6
source/dnode/vnode/inc/tsdb.h
View File

@ -107,10 +107,8 @@ int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg);
int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
// TODO: This is the basic params, and should wrap the params to a queryHandle. // TODO: This is the basic params, and should wrap the params to a queryHandle.
int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit,
int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult);
int32_t nMaxResult);
// STsdbCfg // STsdbCfg
int tsdbOptionsInit(STsdbCfg *); int tsdbOptionsInit(STsdbCfg *);

1
source/dnode/vnode/src/inc/tqInt.h
View File

@ -161,6 +161,7 @@ struct STQ {
STqMemRef tqMemRef; STqMemRef tqMemRef;
STqMetaStore* tqMeta; STqMetaStore* tqMeta;
STqPushMgr* tqPushMgr; STqPushMgr* tqPushMgr;
SHashObj* pStreamTasks;
SWal* pWal; SWal* pWal;
SMeta* pVnodeMeta; SMeta* pVnodeMeta;
}; };

1
source/dnode/vnode/src/inc/tsdbDBDef.h
View File

@ -26,6 +26,7 @@ typedef struct SDBFile SDBFile;
typedef DB_ENV* TDBEnv; typedef DB_ENV* TDBEnv;
struct SDBFile { struct SDBFile {
int32_t fid;
DB* pDB; DB* pDB;
char* path; char* path;
}; };

2
source/dnode/vnode/src/inc/tsdbDef.h
View File

@ -63,7 +63,9 @@ struct STsdb {
#define REPO_ID(r) ((r)->vgId) #define REPO_ID(r) ((r)->vgId)
#define REPO_CFG(r) (&(r)->config) #define REPO_CFG(r) (&(r)->config)
#define REPO_FS(r) (r)->fs #define REPO_FS(r) (r)->fs
#define REPO_TFS(r) (r)->pTfs
#define IS_REPO_LOCKED(r) (r)->repoLocked #define IS_REPO_LOCKED(r) (r)->repoLocked
#define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv)
int tsdbLockRepo(STsdb *pTsdb); int tsdbLockRepo(STsdb *pTsdb);
int tsdbUnlockRepo(STsdb *pTsdb); int tsdbUnlockRepo(STsdb *pTsdb);

24
source/dnode/vnode/src/inc/tsdbFS.h
View File

@ -42,23 +42,23 @@ typedef struct {
typedef struct { typedef struct {
STsdbFSMeta meta; // FS meta STsdbFSMeta meta; // FS meta
SArray * df; // data file array SArray * df; // data file array
SArray * sf; // sma data file array v2(t|r)1900.index_name_1 SArray * sf; // sma data file array v2f1900.index_name_1
} SFSStatus; } SFSStatus;
/** /**
* @brief Directory structure of .tsma data files. * @brief Directory structure of .tsma data files.
* *
* /vnode2/tsdb $ tree .sma/ * /vnode2/tsdb $ tree tsma/
* .sma/ * tsma/
* ├── v2t100.index_name_1 * ├── v2f100.index_name_1
* ├── v2t101.index_name_1 * ├── v2f101.index_name_1
* ├── v2t102.index_name_1 * ├── v2f102.index_name_1
* ├── v2t1900.index_name_3 * ├── v2f1900.index_name_3
* ├── v2t1901.index_name_3 * ├── v2f1901.index_name_3
* ├── v2t1902.index_name_3 * ├── v2f1902.index_name_3
* ├── v2t200.index_name_2 * ├── v2f200.index_name_2
* ├── v2t201.index_name_2 * ├── v2f201.index_name_2
* └── v2t202.index_name_2 * └── v2f202.index_name_2
* *
* 0 directories, 9 files * 0 directories, 9 files
*/ */

6
source/dnode/vnode/src/inc/tsdbSma.h
View File

@ -21,12 +21,14 @@ typedef struct SSmaEnv SSmaEnv;
struct SSmaEnv { struct SSmaEnv {
pthread_rwlock_t lock; pthread_rwlock_t lock;
TDBEnv dbEnv; SDiskID did;
char * path; TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level?
char * path; // relative path
SSmaStat * pStat; SSmaStat * pStat;
}; };
#define SMA_ENV_LOCK(env) ((env)->lock) #define SMA_ENV_LOCK(env) ((env)->lock)
#define SMA_ENV_DID(env) ((env)->did)
#define SMA_ENV_ENV(env) ((env)->dbEnv) #define SMA_ENV_ENV(env) ((env)->dbEnv)
#define SMA_ENV_PATH(env) ((env)->path) #define SMA_ENV_PATH(env) ((env)->path)
#define SMA_ENV_STAT(env) ((env)->pStat) #define SMA_ENV_STAT(env) ((env)->pStat)

4
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -636,7 +636,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
} }
// Decode // Decode
pCfg = (STSma *)malloc(sizeof(STSma)); pCfg = (STSma *)calloc(1, sizeof(STSma));
if (pCfg == NULL) { if (pCfg == NULL) {
return NULL; return NULL;
} }
@ -884,7 +884,7 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) {
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
STSmaWrapper *pSW = NULL; STSmaWrapper *pSW = NULL;
pSW = calloc(sizeof(*pSW), 1); pSW = calloc(1, sizeof(*pSW));
if (pSW == NULL) { if (pSW == NULL) {
return NULL; return NULL;
} }

17
source/dnode/vnode/src/tq/tq.c
View File

@ -55,6 +55,8 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, S
return NULL; return NULL;
} }
pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
return pTq; return pTq;
} }
@ -416,3 +418,18 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
return 0; return 0;
} }
int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
SStreamTask* pTask = malloc(sizeof(SStreamTask));
if (pTask == NULL) {
return -1;
}
SCoder decoder;
tCoderInit(&decoder, TD_LITTLE_ENDIAN, (uint8_t*)msg, msgLen, TD_DECODER);
tDecodeSStreamTask(&decoder, pTask);
tCoderClear(&decoder);
taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask));
return 0;
}

8
source/dnode/vnode/src/tq/tqRead.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tdatablock.h"
#include "vnode.h" #include "vnode.h"
STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
@ -128,10 +129,13 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
int j = 0; int j = 0;
for (int32_t i = 0; i < colNumNeed; i++) { for (int32_t i = 0; i < colNumNeed; i++) {
int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); int16_t colId = *(int16_t*)taosArrayGet(pHandle->pColIdList, i);
while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) {
j++; j++;
} }
if (j >= pSchemaWrapper->nCols) {
continue;
}
SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; SSchema* pColSchema = &pSchemaWrapper->pSchema[j];
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo = {0};
int sz = numOfRows * pColSchema->bytes; int sz = numOfRows * pColSchema->bytes;
@ -145,6 +149,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
return NULL; return NULL;
} }
blockDataEnsureColumnCapacity(&colInfo, numOfRows);
taosArrayPush(pArray, &colInfo); taosArrayPush(pArray, &colInfo);
} }

4
source/dnode/vnode/src/tsdb/tsdbBDBImpl.c
View File

@ -68,8 +68,8 @@ int32_t tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) {
ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_CDB | DB_INIT_MPOOL, 0); ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_CDB | DB_INIT_MPOOL, 0);
if (ret != 0) { if (ret != 0) {
// BDB_PERR("Failed to open tsdb env", ret); terrno = TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR;
tsdbWarn("Failed to open tsdb env for path %s since %d", path ? path : "NULL", ret); tsdbWarn("Failed to open tsdb env for path %s since ret %d != 0", path ? path : "NULL", ret);
return -1; return -1;
} }

4
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -23,8 +23,8 @@ static const char *TSDB_FNAME_SUFFIX[] = {
"smal", // TSDB_FILE_SMAL "smal", // TSDB_FILE_SMAL
"", // TSDB_FILE_MAX "", // TSDB_FILE_MAX
"meta", // TSDB_FILE_META "meta", // TSDB_FILE_META
"sma", // TSDB_FILE_TSMA(directory name) "tsma", // TSDB_FILE_TSMA
"sma", // TSDB_FILE_RSMA(directory name) "rsma", // TSDB_FILE_RSMA
}; };
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);

2
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -3386,7 +3386,7 @@ void filterPrepare(void* expr, void* param) {
if (size < (uint32_t)pSchema->bytes) { if (size < (uint32_t)pSchema->bytes) {
size = pSchema->bytes; size = pSchema->bytes;
} }
// to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space. // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(TdUcs4) space.
pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
tVariantDump(pCond, pInfo->q, pSchema->type, true); tVariantDump(pCond, pInfo->q, pSchema->type, true);
} }

429
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -15,9 +15,14 @@
#include "tsdbDef.h" #include "tsdbDef.h"
#undef SMA_PRINT_DEBUG_LOG static const char *TSDB_SMA_DNAME[] = {
"", // TSDB_SMA_TYPE_BLOCK
"tsma", // TSDB_SMA_TYPE_TIME_RANGE
"rsma", // TSDB_SMA_TYPE_ROLLUP
};
#undef _TEST_SMA_PRINT_DEBUG_LOG_
#define SMA_STORAGE_TSDB_DAYS 30 #define SMA_STORAGE_TSDB_DAYS 30
#define SMA_STORAGE_TSDB_TIMES 30 #define SMA_STORAGE_TSDB_TIMES 10
#define SMA_STORAGE_SPLIT_HOURS 24 #define SMA_STORAGE_SPLIT_HOURS 24
#define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8 #define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8
@ -27,8 +32,8 @@
#define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test #define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test
#define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test #define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test
typedef enum { typedef enum {
SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2t200.dat SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma
SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/rsma/sma_index_uid/v2r200.dat SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma
} ESmaStorageLevel; } ESmaStorageLevel;
typedef struct { typedef struct {
@ -41,6 +46,7 @@ typedef struct {
int32_t iter; int32_t iter;
int32_t fid; int32_t fid;
} SmaFsIter; } SmaFsIter;
typedef struct { typedef struct {
STsdb * pTsdb; STsdb * pTsdb;
SDBFile dFile; SDBFile dFile;
@ -61,40 +67,55 @@ typedef struct {
*/ */
int8_t state; // ETsdbSmaStat int8_t state; // ETsdbSmaStat
SHashObj *expiredWindows; // key: skey of time window, value: N/A SHashObj *expiredWindows; // key: skey of time window, value: N/A
STSma * pSma; STSma * pSma; // cache schema
} SSmaStatItem; } SSmaStatItem;
struct SSmaStat { struct SSmaStat {
SHashObj *smaStatItems; // key: indexName, value: SSmaStatItem SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem
T_REF_DECLARE()
}; };
// declaration of static functions // declaration of static functions
static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg);
static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg);
// TODO: This is the basic params, and should wrap the params to a queryHandle.
static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval,
int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey,
int32_t nMaxResult);
static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg);
// expired window
static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg);
static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat);
static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat);
static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path); static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did);
static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv); static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv);
static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey);
static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat);
static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat);
// read data
// TODO: This is the basic params, and should wrap the params to a queryHandle.
static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval,
int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey,
int32_t nMaxResult);
// insert data
static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData);
static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH);
static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit);
static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit);
static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData); static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData);
static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen);
static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision);
static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel);
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid); static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid);
static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey); static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey);
static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey);
static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]);
static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg);
static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg);
static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { // implementation
static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) {
snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP,
TSDB_SMA_DNAME[smaType]);
}
static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) {
SSmaEnv *pEnv = NULL; SSmaEnv *pEnv = NULL;
pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv)); pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv));
@ -117,12 +138,16 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) {
return NULL; return NULL;
} }
pEnv->did = did;
if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) { if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) {
tsdbFreeSmaEnv(pEnv); tsdbFreeSmaEnv(pEnv);
return NULL; return NULL;
} }
if (tsdbOpenBDBEnv(&pEnv->dbEnv, pEnv->path) != TSDB_CODE_SUCCESS) { char aname[TSDB_FILENAME_LEN] = {0};
tfsAbsoluteName(pTsdb->pTfs, did, path, aname);
if (tsdbOpenBDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) {
tsdbFreeSmaEnv(pEnv); tsdbFreeSmaEnv(pEnv);
return NULL; return NULL;
} }
@ -130,32 +155,18 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) {
return pEnv; return pEnv;
} }
static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) { static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv) {
if (!pEnv) { if (!pEnv) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (pEnv && *pEnv) {
return TSDB_CODE_SUCCESS;
}
if (tsdbLockRepo(pTsdb) != 0) {
return TSDB_CODE_FAILED;
}
if (*pEnv == NULL) { if (*pEnv == NULL) {
if ((*pEnv = tsdbNewSmaEnv(pTsdb, path)) == NULL) { if ((*pEnv = tsdbNewSmaEnv(pTsdb, path, did)) == NULL) {
tsdbUnlockRepo(pTsdb);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }
if (tsdbUnlockRepo(pTsdb) != 0) {
tsdbFreeSmaEnv(*pEnv);
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -181,6 +192,21 @@ void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) {
return NULL; return NULL;
} }
static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) {
if (pStat == NULL) return 0;
int ref = T_REF_INC(pStat);
tsdbDebug("vgId:%d ref sma stat %p ref %d", REPO_ID(pTsdb), pStat, ref);
return 0;
}
static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) {
if (pStat == NULL) return 0;
int ref = T_REF_DEC(pStat);
tsdbDebug("vgId:%d unref sma stat %p ref %d", REPO_ID(pTsdb), pStat, ref);
return 0;
}
static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) {
ASSERT(pSmaStat != NULL); ASSERT(pSmaStat != NULL);
@ -188,11 +214,14 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// TODO: lock. lazy mode when update expired window, or hungry mode during tsdbNew. /**
* 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tsdbNew).
* 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if
* tsdbInitSmaStat invoked in other multithread environment later.
*/
if (*pSmaStat == NULL) { if (*pSmaStat == NULL) {
*pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat)); *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat));
if (*pSmaStat == NULL) { if (*pSmaStat == NULL) {
// TODO: unlock
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -202,11 +231,9 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) {
if ((*pSmaStat)->smaStatItems == NULL) { if ((*pSmaStat)->smaStatItems == NULL) {
tfree(*pSmaStat); tfree(*pSmaStat);
// TODO: unlock
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }
// TODO: unlock
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -234,16 +261,73 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) {
int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
if (pSmaStat) { if (pSmaStat) {
// TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL); void *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
while (item != NULL) { while (item != NULL) {
tfree(item->pSma); SSmaStatItem *pItem = *(SSmaStatItem **)item;
taosHashCleanup(item->expiredWindows); if (pItem != NULL) {
tdDestroyTSma(pItem->pSma);
tfree(pItem->pSma);
taosHashCleanup(pItem->expiredWindows);
tfree(pItem);
}
item = taosHashIterate(pSmaStat->smaStatItems, item); item = taosHashIterate(pSmaStat->smaStatItems, item);
} }
taosHashCleanup(pSmaStat->smaStatItems); taosHashCleanup(pSmaStat->smaStatItems);
} }
} }
static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
SSmaEnv *pEnv = NULL;
// return if already init
switch (smaType) {
case TSDB_SMA_TYPE_TIME_RANGE:
if ((pEnv = (SSmaEnv *)atomic_load_ptr(&pTsdb->pTSmaEnv)) != NULL) {
return TSDB_CODE_SUCCESS;
}
break;
case TSDB_SMA_TYPE_ROLLUP:
if ((pEnv = (SSmaEnv *)atomic_load_ptr(&pTsdb->pRSmaEnv)) != NULL) {
return TSDB_CODE_SUCCESS;
}
break;
default:
terrno = TSDB_CODE_INVALID_PARA;
return TSDB_CODE_FAILED;
}
// init sma env
tsdbLockRepo(pTsdb);
pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&pTsdb->pTSmaEnv) : atomic_load_ptr(&pTsdb->pRSmaEnv);
if (pEnv == NULL) {
char rname[TSDB_FILENAME_LEN] = {0};
SDiskID did = {0};
tfsAllocDisk(pTsdb->pTfs, TFS_PRIMARY_LEVEL, &did);
if (did.level < 0 || did.id < 0) {
tsdbUnlockRepo(pTsdb);
return TSDB_CODE_FAILED;
}
tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname);
if (tfsMkdirRecurAt(pTsdb->pTfs, rname, did) != TSDB_CODE_SUCCESS) {
tsdbUnlockRepo(pTsdb);
return TSDB_CODE_FAILED;
}
if (tsdbInitSmaEnv(pTsdb, rname, did, &pEnv) != TSDB_CODE_SUCCESS) {
tsdbUnlockRepo(pTsdb);
return TSDB_CODE_FAILED;
}
(smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&pTsdb->pTSmaEnv, pEnv)
: atomic_store_ptr(&pTsdb->pRSmaEnv, pEnv);
}
tsdbUnlockRepo(pTsdb);
return TSDB_CODE_SUCCESS;
};
/** /**
* @brief Update expired window according to msg from stream computing module. * @brief Update expired window according to msg from stream computing module.
* *
@ -252,47 +336,41 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
* @param msg * @param msg
* @return int32_t * @return int32_t
*/ */
int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) {
STsdbCfg *pCfg = REPO_CFG(pTsdb);
SSmaEnv * pEnv = NULL;
if (!msg || !pTsdb->pMeta) { if (!msg || !pTsdb->pMeta) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
char smaPath[TSDB_FILENAME_LEN] = "/proj/.sma/"; // TODO: decode the msg from Stream Computing module => start
if (tsdbInitSmaEnv(pTsdb, smaPath, &pEnv) != TSDB_CODE_SUCCESS) { int64_t indexUid = SMA_TEST_INDEX_UID;
const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10;
TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE];
TSKEY skey1 = 1646987196 * 1e3;
for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
expiredWindows[i] = skey1 + i;
}
// TODO: decode the msg <= end
if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INIT_FAILED;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { SSmaEnv * pEnv = REPO_SMA_ENV(pTsdb, smaType);
pTsdb->pTSmaEnv = pEnv; SSmaStat *pStat = SMA_ENV_STAT(pEnv);
} else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
pTsdb->pRSmaEnv = pEnv;
} else {
ASSERT(0);
}
// TODO: decode the msg => start
int64_t indexUid = SMA_TEST_INDEX_UID;
// const char * indexName = SMA_TEST_INDEX_NAME;
const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10;
TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE];
int64_t now = taosGetTimestampMs();
for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
expiredWindows[i] = now + i;
}
// TODO: decode the msg <= end
SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv);
SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL);
tsdbRefSmaStat(pTsdb, pStat);
SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
if (pItem == NULL) { if (pItem == NULL) {
pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state
if (pItem == NULL) { if (pItem == NULL) {
// Response to stream computing: OOM // Response to stream computing: OOM
// For query, if the indexName not found, the TSDB should tell query module to query raw TS data. // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data.
tsdbUnRefSmaStat(pTsdb, pStat);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -302,29 +380,25 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) {
terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META;
taosHashCleanup(pItem->expiredWindows); taosHashCleanup(pItem->expiredWindows);
free(pItem); free(pItem);
tsdbUnRefSmaStat(pTsdb, pStat);
tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid,
tstrerror(terrno)); tstrerror(terrno));
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
pItem->pSma = pSma; pItem->pSma = pSma;
// TODO: change indexName to indexUid
if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) {
// If error occurs during put smaStatItem, free the resources of pItem // If error occurs during put smaStatItem, free the resources of pItem
taosHashCleanup(pItem->expiredWindows); taosHashCleanup(pItem->expiredWindows);
free(pItem); free(pItem);
tsdbUnRefSmaStat(pTsdb, pStat);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }
#if 0
SSmaStatItem *pItem1 = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
int size1 = taosHashGetSize(pItem1->expiredWindows);
tsdbWarn("vgId:%d smaIndex %" PRIi64 " size is %d before hashPut", REPO_ID(pTsdb), indexUid, size1);
#endif
int8_t state = TSDB_SMA_STAT_EXPIRED; int8_t state = TSDB_SMA_STAT_EXPIRED;
for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
if (taosHashPut(pItem->expiredWindows, &expiredWindows[i], sizeof(TSKEY), &state, sizeof(state)) != 0) { if (taosHashPut(pItem->expiredWindows, expiredWindows + i, sizeof(TSKEY), &state, sizeof(state)) != 0) {
// If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would
// tell query module to query raw TS data. // tell query module to query raw TS data.
// N.B. // N.B.
@ -334,37 +408,43 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) {
taosHashCleanup(pItem->expiredWindows); taosHashCleanup(pItem->expiredWindows);
tfree(pItem->pSma); tfree(pItem->pSma);
taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid));
tsdbUnRefSmaStat(pTsdb, pStat);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
tsdbDebug("vgId:%d smaIndex %" PRIi64 " tsKey %" PRIi64 " is put to hash", REPO_ID(pTsdb), indexUid,
expiredWindows[i]);
} }
#if 0 tsdbUnRefSmaStat(pTsdb, pStat);
SSmaStatItem *pItem2 = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
int size2 = taosHashGetSize(pItem1->expiredWindows);
tsdbWarn("vgId:%d smaIndex %" PRIi64 " size is %d after hashPut", REPO_ID(pTsdb), indexUid, size2);
#endif
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t tsdbResetExpiredWindow(SSmaStat *pStat, int64_t indexUid, TSKEY skey) { static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey) {
SSmaStatItem *pItem = NULL; SSmaStatItem *pItem = NULL;
// TODO: If HASH_ENTRY_LOCK used, whether rwlock needed to handle cases of removing hashNode? tsdbRefSmaStat(pTsdb, pStat);
if (pStat && pStat->smaStatItems) { if (pStat && pStat->smaStatItems) {
pItem = (SSmaStatItem *)taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid));
} }
#if 0 if ((pItem != NULL) && ((pItem = *(SSmaStatItem **)pItem) != NULL)) {
if (pItem != NULL) { // pItem resides in hash buffer all the time unless drop sma index
// TODO: reset time window for the sma data blocks // TODO: multithread protect
if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
// error handling // error handling
tsdbUnRefSmaStat(pTsdb, pStat);
tsdbWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " failed", REPO_ID(pTsdb),
skey, indexUid);
return TSDB_CODE_FAILED;
} }
} else { } else {
// error handling // error handling
tsdbUnRefSmaStat(pTsdb, pStat);
tsdbWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, REPO_ID(pTsdb), skey, indexUid);
return TSDB_CODE_FAILED;
} }
#endif
tsdbUnRefSmaStat(pTsdb, pStat);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -378,32 +458,32 @@ static int32_t tsdbResetExpiredWindow(SSmaStat *pStat, int64_t indexUid, TSKEY s
static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
// TODO: configurable for SMA_STORAGE_SPLIT_HOURS? // TODO: configurable for SMA_STORAGE_SPLIT_HOURS?
switch (intervalUnit) { switch (intervalUnit) {
case TD_TIME_UNIT_HOUR: case TIME_UNIT_HOUR:
if (interval < SMA_STORAGE_SPLIT_HOURS) { if (interval < SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET; return SMA_STORAGE_LEVEL_DFILESET;
} }
break; break;
case TD_TIME_UNIT_MINUTE: case TIME_UNIT_MINUTE:
if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) { if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET; return SMA_STORAGE_LEVEL_DFILESET;
} }
break; break;
case TD_TIME_UNIT_SEC: case TIME_UNIT_SECOND:
if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) { if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET; return SMA_STORAGE_LEVEL_DFILESET;
} }
break; break;
case TD_TIME_UNIT_MILLISEC: case TIME_UNIT_MILLISECOND:
if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) { if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET; return SMA_STORAGE_LEVEL_DFILESET;
} }
break; break;
case TD_TIME_UNIT_MICROSEC: case TIME_UNIT_MICROSECOND:
if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) { if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET; return SMA_STORAGE_LEVEL_DFILESET;
} }
break; break;
case TD_TIME_UNIT_NANOSEC: case TIME_UNIT_NANOSECOND:
if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) { if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET; return SMA_STORAGE_LEVEL_DFILESET;
} }
@ -418,7 +498,7 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
* @brief Insert TSma data blocks to DB File build by B+Tree * @brief Insert TSma data blocks to DB File build by B+Tree
* *
* @param pSmaH * @param pSmaH
* @param smaKey * @param smaKey tableUid-colId-skeyOfWindow(8-2-8)
* @param keyLen * @param keyLen
* @param pData * @param pData
* @param dataLen * @param dataLen
@ -426,89 +506,95 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
*/ */
static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) { static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) {
SDBFile *pDBFile = &pSmaH->dFile; SDBFile *pDBFile = &pSmaH->dFile;
// TODO: insert sma data blocks into B+Tree
tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d", tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d",
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
*(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
// TODO: insert sma data blocks into B+Tree(TDB)
if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) { if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
#ifdef SMA_PRINT_DEBUG_LOG #ifdef _TEST_SMA_PRINT_DEBUG_LOG_
uint32_t valueSize = 0; uint32_t valueSize = 0;
void * data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); void * data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize);
ASSERT(data != NULL); ASSERT(data != NULL);
for (uint32_t v = 0; v < valueSize; v += 8) { for (uint32_t v = 0; v < valueSize; v += 8) {
tsdbWarn("vgId:%d sma data - val[%d] is %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); tsdbWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v));
} }
#endif #endif
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
/**
* @brief Approximate value for week/month/year.
*
* @param interval
* @param intervalUnit
* @param precision
* @return int64_t
*/
static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision) { static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision) {
if (intervalUnit < TD_TIME_UNIT_MILLISEC) {
switch (intervalUnit) { switch (intervalUnit) {
case TD_TIME_UNIT_YEAR: case TIME_UNIT_YEAR: // approximate value
case TD_TIME_UNIT_SEASON: interval *= 365 * 86400 * 1e3;
case TD_TIME_UNIT_MONTH:
case TD_TIME_UNIT_WEEK:
// illegal time unit
tsdbError("invalid interval unit: %d\n", intervalUnit);
TASSERT(0);
break; break;
case TD_TIME_UNIT_DAY: // the interval for tSma calculation must <= day case TIME_UNIT_MONTH: // approximate value
interval *= 30 * 86400 * 1e3;
break;
case TIME_UNIT_WEEK: // approximate value
interval *= 7 * 86400 * 1e3;
break;
case TIME_UNIT_DAY: // the interval for tSma calculation must <= day
interval *= 86400 * 1e3; interval *= 86400 * 1e3;
break; break;
case TD_TIME_UNIT_HOUR: case TIME_UNIT_HOUR:
interval *= 3600 * 1e3; interval *= 3600 * 1e3;
break; break;
case TD_TIME_UNIT_MINUTE: case TIME_UNIT_MINUTE:
interval *= 60 * 1e3; interval *= 60 * 1e3;
break; break;
case TD_TIME_UNIT_SEC: case TIME_UNIT_SECOND:
interval *= 1e3; interval *= 1e3;
break; break;
default: default:
break; break;
} }
}
switch (precision) { switch (precision) {
case TSDB_TIME_PRECISION_MILLI: case TSDB_TIME_PRECISION_MILLI:
if (TD_TIME_UNIT_MICROSEC == intervalUnit) { // us if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval / 1e3; return interval / 1e3;
} else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
return interval / 1e6; return interval / 1e6;
} else { } else { // ms
return interval; return interval;
} }
break; break;
case TSDB_TIME_PRECISION_MICRO: case TSDB_TIME_PRECISION_MICRO:
if (TD_TIME_UNIT_MICROSEC == intervalUnit) { // us if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval; return interval;
} else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
return interval / 1e3; return interval / 1e3;
} else { } else { // ms
return interval * 1e3; return interval * 1e3;
} }
break; break;
case TSDB_TIME_PRECISION_NANO: case TSDB_TIME_PRECISION_NANO:
if (TD_TIME_UNIT_MICROSEC == intervalUnit) { if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval * 1e3; return interval * 1e3;
} else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
return interval; return interval;
} else { } else { // ms
return interval * 1e6; return interval * 1e6;
} }
break; break;
default: // ms default: // ms
if (TD_TIME_UNIT_MICROSEC == intervalUnit) { // us if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval / 1e3; return interval / 1e3;
} else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
return interval / 1e6; return interval / 1e6;
} else { } else { // ms
return interval; return interval;
} }
break; break;
@ -577,12 +663,15 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) {
} }
} }
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) { static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid) {
STsdb *pTsdb = pSmaH->pTsdb; STsdb *pTsdb = pSmaH->pTsdb;
ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL); ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL);
pSmaH->dFile.fid = fid;
char tSmaFile[TSDB_FILENAME_LEN] = {0}; char tSmaFile[TSDB_FILENAME_LEN] = {0};
snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid); snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid);
pSmaH->dFile.path = strdup(tSmaFile); pSmaH->dFile.path = strdup(tSmaFile);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -621,8 +710,9 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe
static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
STsdbCfg * pCfg = REPO_CFG(pTsdb); STsdbCfg * pCfg = REPO_CFG(pTsdb);
STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; STSmaDataWrapper *pData = (STSmaDataWrapper *)msg;
SSmaEnv * pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv);
if (!pTsdb->pTSmaEnv) { if (pEnv == NULL) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
return terrno; return terrno;
@ -640,6 +730,17 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
int64_t indexUid = SMA_TEST_INDEX_UID;
char rPath[TSDB_FILENAME_LEN] = {0};
char aPath[TSDB_FILENAME_LEN] = {0};
snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid);
tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath);
if (!taosCheckExistFile(aPath)) {
if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
}
// Step 1: Judge the storage level and days // Step 1: Judge the storage level and days
int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit);
int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel);
@ -648,7 +749,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
// Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
// - Set and open the DFile or the B+Tree file // - Set and open the DFile or the B+Tree file
// TODO: tsdbStartTSmaCommit(); // TODO: tsdbStartTSmaCommit();
tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid);
if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) {
tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb),
tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
@ -664,7 +765,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
// TODO:tsdbEndTSmaCommit(); // TODO:tsdbEndTSmaCommit();
// Step 3: reset the SSmaStat // Step 3: reset the SSmaStat
tsdbResetExpiredWindow(SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey);
tsdbDestroyTSmaWriteH(&tSmaH); tsdbDestroyTSmaWriteH(&tSmaH);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -710,7 +811,7 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) {
// TODO:tsdbEndTSmaCommit(); // TODO:tsdbEndTSmaCommit();
// reset the SSmaStat // reset the SSmaStat
tsdbResetExpiredWindow(SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey); tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -735,13 +836,16 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interv
* @brief Init of tSma FS * @brief Init of tSma FS
* *
* @param pReadH * @param pReadH
* @param indexUid
* @param skey * @param skey
* @return int32_t * @return int32_t
*/ */
static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey) { static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) {
int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pSmaH->pTsdb)->precision)); STsdb *pTsdb = pSmaH->pTsdb;
int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pTsdb)->precision));
char tSmaFile[TSDB_FILENAME_LEN] = {0}; char tSmaFile[TSDB_FILENAME_LEN] = {0};
snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pSmaH->pTsdb), fid); snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid);
pSmaH->dFile.path = strdup(tSmaFile); pSmaH->dFile.path = strdup(tSmaFile);
pSmaH->smaFsIter.iter = 0; pSmaH->smaFsIter.iter = 0;
pSmaH->smaFsIter.fid = fid; pSmaH->smaFsIter.fid = fid;
@ -798,12 +902,25 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
* @return int32_t * @return int32_t
*/ */
static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval,
int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey,
int32_t nMaxResult) { int32_t nMaxResult) {
SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid)); SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv);
if (pItem == NULL) {
// mark all window as expired and notify query module to query raw TS data. if (!pEnv) {
return TSDB_CODE_SUCCESS; terrno = TSDB_CODE_INVALID_PTR;
tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
return TSDB_CODE_FAILED;
}
tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pEnv));
SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) {
// Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if
// it's NULL.
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
terrno = TSDB_CODE_TDB_INVALID_ACTION;
tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, REPO_ID(pTsdb), indexUid);
return TSDB_CODE_FAILED;
} }
#if 0 #if 0
@ -815,16 +932,25 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_
} }
} }
#endif #endif
#if 0
if (taosHashGet(pItem->expiredWindows, &querySkey, sizeof(TSKEY)) != NULL) { #if 1
if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY)) != NULL) {
// TODO: mark this window as expired. // TODO: mark this window as expired.
tsdbDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, REPO_ID(pTsdb),
querySKey, indexUid);
} else {
tsdbDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, REPO_ID(pTsdb), querySKey,
indexUid);
} }
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
#endif #endif
STSmaReadH tReadH = {0}; STSmaReadH tReadH = {0};
tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit); tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit);
tsdbCloseDBF(&tReadH.dFile); tsdbCloseDBF(&tReadH.dFile);
tsdbInitTSmaFile(&tReadH, querySkey); tsdbInitTSmaFile(&tReadH, indexUid, querySKey);
if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) { if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) {
tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno)); tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno));
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -832,7 +958,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_
char smaKey[SMA_KEY_LEN] = {0}; char smaKey[SMA_KEY_LEN] = {0};
void *pSmaKey = &smaKey; void *pSmaKey = &smaKey;
tsdbEncodeTSmaKey(tableUid, colId, querySkey, (void **)&pSmaKey); tsdbEncodeTSmaKey(tableUid, colId, querySKey, (void **)&pSmaKey);
tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb), tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb),
tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
@ -848,12 +974,14 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_
tsdbCloseDBF(&tReadH.dFile); tsdbCloseDBF(&tReadH.dFile);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
tfree(result);
#ifdef SMA_PRINT_DEBUG_LOG #ifdef _TEST_SMA_PRINT_DEBUG_LOG_
for (uint32_t v = 0; v < valueSize; v += 8) { for (uint32_t v = 0; v < valueSize; v += 8) {
tsdbWarn("vgId:%d v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); tsdbWarn("vgId:%d get sma data v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v));
} }
#endif #endif
tfree(result); // TODO: fill the result to output
#if 0 #if 0
int32_t nResult = 0; int32_t nResult = 0;
int64_t lastKey = 0; int64_t lastKey = 0;
@ -947,7 +1075,6 @@ int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) {
* @brief Insert Time-range-wise Rollup Sma(RSma) data * @brief Insert Time-range-wise Rollup Sma(RSma) data
* *
* @param pTsdb * @param pTsdb
* @param param
* @param msg * @param msg
* @return int32_t * @return int32_t
*/ */
@ -960,9 +1087,9 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) {
} }
int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit,
tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, int32_t nMaxResult) { tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, interval, intervalUnit, tableUid, colId, querySkey, if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, interval, intervalUnit, tableUid, colId, querySKey,
nMaxResult)) < 0) { nMaxResult)) < 0) {
tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
} }

8
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -132,6 +132,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
} }
} break; } break;
case TDMT_VND_TASK_DEPLOY: {
if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
pMsg->contLen - sizeof(SMsgHead)) < 0) {
}
} break;
case TDMT_VND_CREATE_SMA: { // timeRangeSMA case TDMT_VND_CREATE_SMA: { // timeRangeSMA
SSmaCfg vCreateSmaReq = {0}; SSmaCfg vCreateSmaReq = {0};
if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) { if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) {
@ -139,6 +144,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
return -1; return -1;
} }
// record current timezone of server side
tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezone, TD_TIMEZONE_LEN);
if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) { if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) {
// TODO: handle error // TODO: handle error
tdDestroyTSma(&vCreateSmaReq.tSma); tdDestroyTSma(&vCreateSmaReq.tSma);

165
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -33,13 +33,14 @@ int main(int argc, char **argv) {
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
} }
#if 1
TEST(testCase, tSma_Meta_Encode_Decode_Test) { TEST(testCase, tSma_Meta_Encode_Decode_Test) {
// encode // encode
STSma tSma = {0}; STSma tSma = {0};
tSma.version = 0; tSma.version = 0;
tSma.intervalUnit = TD_TIME_UNIT_DAY; tSma.intervalUnit = TIME_UNIT_DAY;
tSma.interval = 1; tSma.interval = 1;
tSma.slidingUnit = TD_TIME_UNIT_HOUR; tSma.slidingUnit = TIME_UNIT_HOUR;
tSma.sliding = 0; tSma.sliding = 0;
tstrncpy(tSma.indexName, "sma_index_test", TSDB_INDEX_NAME_LEN); tstrncpy(tSma.indexName, "sma_index_test", TSDB_INDEX_NAME_LEN);
tstrncpy(tSma.timezone, "Asia/Shanghai", TD_TIMEZONE_LEN); tstrncpy(tSma.timezone, "Asia/Shanghai", TD_TIMEZONE_LEN);
@ -49,44 +50,46 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) {
STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma}; STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma};
uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper); uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper);
void *buf = calloc(bufLen, 1); void *buf = calloc(1, bufLen);
assert(buf != NULL); ASSERT_NE(buf, nullptr);
STSmaWrapper *pSW = (STSmaWrapper *)buf; STSmaWrapper *pSW = (STSmaWrapper *)buf;
uint32_t len = tEncodeTSmaWrapper(&buf, &tSmaWrapper); uint32_t len = tEncodeTSmaWrapper(&buf, &tSmaWrapper);
EXPECT_EQ(len, bufLen); ASSERT_EQ(len, bufLen);
// decode // decode
STSmaWrapper dstTSmaWrapper = {0}; STSmaWrapper dstTSmaWrapper = {0};
void * result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper); void * result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper);
assert(result != NULL); ASSERT_NE(result, nullptr);
EXPECT_EQ(tSmaWrapper.number, dstTSmaWrapper.number); ASSERT_EQ(tSmaWrapper.number, dstTSmaWrapper.number);
for (int i = 0; i < tSmaWrapper.number; ++i) { for (int i = 0; i < tSmaWrapper.number; ++i) {
STSma *pSma = tSmaWrapper.tSma + i; STSma *pSma = tSmaWrapper.tSma + i;
STSma *qSma = dstTSmaWrapper.tSma + i; STSma *qSma = dstTSmaWrapper.tSma + i;
EXPECT_EQ(pSma->version, qSma->version); ASSERT_EQ(pSma->version, qSma->version);
EXPECT_EQ(pSma->intervalUnit, qSma->intervalUnit); ASSERT_EQ(pSma->intervalUnit, qSma->intervalUnit);
EXPECT_EQ(pSma->slidingUnit, qSma->slidingUnit); ASSERT_EQ(pSma->slidingUnit, qSma->slidingUnit);
EXPECT_STRCASEEQ(pSma->indexName, qSma->indexName); ASSERT_STRCASEEQ(pSma->indexName, qSma->indexName);
EXPECT_STRCASEEQ(pSma->timezone, qSma->timezone); ASSERT_STRCASEEQ(pSma->timezone, qSma->timezone);
EXPECT_EQ(pSma->indexUid, qSma->indexUid); ASSERT_EQ(pSma->indexUid, qSma->indexUid);
EXPECT_EQ(pSma->tableUid, qSma->tableUid); ASSERT_EQ(pSma->tableUid, qSma->tableUid);
EXPECT_EQ(pSma->interval, qSma->interval); ASSERT_EQ(pSma->interval, qSma->interval);
EXPECT_EQ(pSma->sliding, qSma->sliding); ASSERT_EQ(pSma->sliding, qSma->sliding);
EXPECT_EQ(pSma->exprLen, qSma->exprLen); ASSERT_EQ(pSma->exprLen, qSma->exprLen);
EXPECT_STRCASEEQ(pSma->expr, qSma->expr); ASSERT_STRCASEEQ(pSma->expr, qSma->expr);
EXPECT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen); ASSERT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen);
EXPECT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter); ASSERT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter);
} }
// resource release // resource release
tfree(pSW);
tdDestroyTSma(&tSma); tdDestroyTSma(&tSma);
tdDestroyTSmaWrapper(&dstTSmaWrapper); tdDestroyTSmaWrapper(&dstTSmaWrapper);
} }
#endif
#if 1 #if 1
TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
@ -103,9 +106,9 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
// encode // encode
STSma tSma = {0}; STSma tSma = {0};
tSma.version = 0; tSma.version = 0;
tSma.intervalUnit = TD_TIME_UNIT_DAY; tSma.intervalUnit = TIME_UNIT_DAY;
tSma.interval = 1; tSma.interval = 1;
tSma.slidingUnit = TD_TIME_UNIT_HOUR; tSma.slidingUnit = TIME_UNIT_HOUR;
tSma.sliding = 0; tSma.sliding = 0;
tSma.indexUid = indexUid1; tSma.indexUid = indexUid1;
tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN);
@ -113,11 +116,13 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
tSma.tableUid = tbUid; tSma.tableUid = tbUid;
tSma.exprLen = strlen(expr); tSma.exprLen = strlen(expr);
tSma.expr = (char *)calloc(tSma.exprLen + 1, 1); tSma.expr = (char *)calloc(1, tSma.exprLen + 1);
ASSERT_NE(tSma.expr, nullptr);
tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
tSma.tagsFilterLen = strlen(tagsFilter); tSma.tagsFilterLen = strlen(tagsFilter);
tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1);
ASSERT_NE(tSma.tagsFilter, nullptr);
tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1);
SMeta * pMeta = NULL; SMeta * pMeta = NULL;
@ -129,18 +134,18 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
pMeta = metaOpen(smaTestDir, pMetaCfg, NULL); pMeta = metaOpen(smaTestDir, pMetaCfg, NULL);
assert(pMeta != NULL); assert(pMeta != NULL);
// save index 1 // save index 1
EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
pSmaCfg->indexUid = indexUid2; pSmaCfg->indexUid = indexUid2;
tstrncpy(pSmaCfg->indexName, smaIndexName2, TSDB_INDEX_NAME_LEN); tstrncpy(pSmaCfg->indexName, smaIndexName2, TSDB_INDEX_NAME_LEN);
pSmaCfg->version = 1; pSmaCfg->version = 1;
pSmaCfg->intervalUnit = TD_TIME_UNIT_HOUR; pSmaCfg->intervalUnit = TIME_UNIT_HOUR;
pSmaCfg->interval = 1; pSmaCfg->interval = 1;
pSmaCfg->slidingUnit = TD_TIME_UNIT_MINUTE; pSmaCfg->slidingUnit = TIME_UNIT_MINUTE;
pSmaCfg->sliding = 5; pSmaCfg->sliding = 5;
// save index 2 // save index 2
EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
// get value by indexName // get value by indexName
STSma *qSmaCfg = NULL; STSma *qSmaCfg = NULL;
@ -150,8 +155,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
printf("timezone1 = %s\n", qSmaCfg->timezone); printf("timezone1 = %s\n", qSmaCfg->timezone);
printf("expr1 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("expr1 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : "");
printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : "");
EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1);
EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid); ASSERT_EQ(qSmaCfg->tableUid, tSma.tableUid);
tdDestroyTSma(qSmaCfg); tdDestroyTSma(qSmaCfg);
tfree(qSmaCfg); tfree(qSmaCfg);
@ -161,8 +166,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
printf("timezone2 = %s\n", qSmaCfg->timezone); printf("timezone2 = %s\n", qSmaCfg->timezone);
printf("expr2 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("expr2 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : "");
printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : "");
EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2);
EXPECT_EQ(qSmaCfg->interval, tSma.interval); ASSERT_EQ(qSmaCfg->interval, tSma.interval);
tdDestroyTSma(qSmaCfg); tdDestroyTSma(qSmaCfg);
tfree(qSmaCfg); tfree(qSmaCfg);
@ -178,25 +183,25 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
printf("indexName = %s\n", indexName); printf("indexName = %s\n", indexName);
++indexCnt; ++indexCnt;
} }
EXPECT_EQ(indexCnt, nCntTSma); ASSERT_EQ(indexCnt, nCntTSma);
metaCloseSmaCurosr(pSmaCur); metaCloseSmaCurosr(pSmaCur);
// get wrapper by table uid // get wrapper by table uid
STSmaWrapper *pSW = metaGetSmaInfoByTable(pMeta, tbUid); STSmaWrapper *pSW = metaGetSmaInfoByTable(pMeta, tbUid);
assert(pSW != NULL); assert(pSW != NULL);
EXPECT_EQ(pSW->number, nCntTSma); ASSERT_EQ(pSW->number, nCntTSma);
EXPECT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1); ASSERT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1);
EXPECT_STRCASEEQ(pSW->tSma->timezone, timezone); ASSERT_STRCASEEQ(pSW->tSma->timezone, timezone);
EXPECT_STRCASEEQ(pSW->tSma->expr, expr); ASSERT_STRCASEEQ(pSW->tSma->expr, expr);
EXPECT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter); ASSERT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter);
EXPECT_EQ(pSW->tSma->indexUid, indexUid1); ASSERT_EQ(pSW->tSma->indexUid, indexUid1);
EXPECT_EQ(pSW->tSma->tableUid, tbUid); ASSERT_EQ(pSW->tSma->tableUid, tbUid);
EXPECT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2); ASSERT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2);
EXPECT_STRCASEEQ((pSW->tSma + 1)->timezone, timezone); ASSERT_STRCASEEQ((pSW->tSma + 1)->timezone, timezone);
EXPECT_STRCASEEQ((pSW->tSma + 1)->expr, expr); ASSERT_STRCASEEQ((pSW->tSma + 1)->expr, expr);
EXPECT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter); ASSERT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter);
EXPECT_EQ((pSW->tSma + 1)->indexUid, indexUid2); ASSERT_EQ((pSW->tSma + 1)->indexUid, indexUid2);
EXPECT_EQ((pSW->tSma + 1)->tableUid, tbUid); ASSERT_EQ((pSW->tSma + 1)->tableUid, tbUid);
tdDestroyTSmaWrapper(pSW); tdDestroyTSmaWrapper(pSW);
tfree(pSW); tfree(pSW);
@ -208,7 +213,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
printf("metaGetSmaTbUids: uid[%" PRIu32 "] = %" PRIi64 "\n", i, *(tb_uid_t *)taosArrayGet(pUids, i)); printf("metaGetSmaTbUids: uid[%" PRIu32 "] = %" PRIi64 "\n", i, *(tb_uid_t *)taosArrayGet(pUids, i));
// printf("metaGetSmaTbUids: index[%" PRIu32 "] = %s", i, (char *)taosArrayGet(pUids, i)); // printf("metaGetSmaTbUids: index[%" PRIu32 "] = %s", i, (char *)taosArrayGet(pUids, i));
} }
EXPECT_EQ(taosArrayGetSize(pUids), 1); ASSERT_EQ(taosArrayGetSize(pUids), 1);
taosArrayDestroy(pUids); taosArrayDestroy(pUids);
// resource release // resource release
@ -231,7 +236,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
const tb_uid_t tbUid = 1234567890; const tb_uid_t tbUid = 1234567890;
const int64_t indexUid1 = 2000000001; const int64_t indexUid1 = 2000000001;
const int64_t interval1 = 1; const int64_t interval1 = 1;
const int8_t intervalUnit1 = TD_TIME_UNIT_DAY; const int8_t intervalUnit1 = TIME_UNIT_DAY;
const uint32_t nCntTSma = 2; const uint32_t nCntTSma = 2;
TSKEY skey1 = 1646987196; TSKEY skey1 = 1646987196;
const int64_t testSmaData1 = 100; const int64_t testSmaData1 = 100;
@ -239,9 +244,9 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
// encode // encode
STSma tSma = {0}; STSma tSma = {0};
tSma.version = 0; tSma.version = 0;
tSma.intervalUnit = TD_TIME_UNIT_DAY; tSma.intervalUnit = TIME_UNIT_DAY;
tSma.interval = 1; tSma.interval = 1;
tSma.slidingUnit = TD_TIME_UNIT_HOUR; tSma.slidingUnit = TIME_UNIT_HOUR;
tSma.sliding = 0; tSma.sliding = 0;
tSma.indexUid = indexUid1; tSma.indexUid = indexUid1;
tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN);
@ -249,11 +254,13 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
tSma.tableUid = tbUid; tSma.tableUid = tbUid;
tSma.exprLen = strlen(expr); tSma.exprLen = strlen(expr);
tSma.expr = (char *)calloc(tSma.exprLen + 1, 1); tSma.expr = (char *)calloc(1, tSma.exprLen + 1);
ASSERT_NE(tSma.expr, nullptr);
tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
tSma.tagsFilterLen = strlen(tagsFilter); tSma.tagsFilterLen = strlen(tagsFilter);
tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); tSma.tagsFilter = (char *)calloc(1, tSma.tagsFilterLen + 1);
ASSERT_NE(tSma.tagsFilter, nullptr);
tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1);
SMeta * pMeta = NULL; SMeta * pMeta = NULL;
@ -265,24 +272,24 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
pMeta = metaOpen(smaTestDir, pMetaCfg, NULL); pMeta = metaOpen(smaTestDir, pMetaCfg, NULL);
assert(pMeta != NULL); assert(pMeta != NULL);
// save index 1 // save index 1
EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
// step 2: insert data // step 2: insert data
STSmaDataWrapper *pSmaData = NULL; STSmaDataWrapper *pSmaData = NULL;
STsdb tsdb = {0}; STsdb * pTsdb = (STsdb *)calloc(1, sizeof(STsdb));
STsdbCfg * pCfg = &tsdb.config; STsdbCfg * pCfg = &pTsdb->config;
tsdb.pMeta = pMeta; pTsdb->pMeta = pMeta;
tsdb.vgId = 2; pTsdb->vgId = 2;
tsdb.config.daysPerFile = 10; // default days is 10 pTsdb->config.daysPerFile = 10; // default days is 10
tsdb.config.keep1 = 30; pTsdb->config.keep1 = 30;
tsdb.config.keep2 = 90; pTsdb->config.keep2 = 90;
tsdb.config.keep = 365; pTsdb->config.keep = 365;
tsdb.config.precision = TSDB_TIME_PRECISION_MILLI; pTsdb->config.precision = TSDB_TIME_PRECISION_MILLI;
tsdb.config.update = TD_ROW_OVERWRITE_UPDATE; pTsdb->config.update = TD_ROW_OVERWRITE_UPDATE;
tsdb.config.compression = TWO_STAGE_COMP; pTsdb->config.compression = TWO_STAGE_COMP;
switch (tsdb.config.precision) { switch (pTsdb->config.precision) {
case TSDB_TIME_PRECISION_MILLI: case TSDB_TIME_PRECISION_MILLI:
skey1 *= 1e3; skey1 *= 1e3;
break; break;
@ -297,19 +304,26 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
break; break;
} }
char *msg = (char *)calloc(100, 1); SDiskCfg pDisks = {.level = 0, .primary = 1};
EXPECT_EQ(tsdbUpdateSmaWindow(&tsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0); strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN);
int32_t numOfDisks = 1;
pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks);
ASSERT_NE(pTsdb->pTfs, nullptr);
char *msg = (char *)calloc(1, 100);
ASSERT_NE(msg, nullptr);
ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0);
// init // init
int32_t allocCnt = 0; int32_t allocCnt = 0;
int32_t allocStep = 40960; int32_t allocStep = 16384;
int32_t buffer = 4096; int32_t buffer = 1024;
void * buf = NULL; void * buf = NULL;
EXPECT_EQ(tsdbMakeRoom(&buf, allocStep), 0); ASSERT_EQ(tsdbMakeRoom(&buf, allocStep), 0);
int32_t bufSize = taosTSizeof(buf); int32_t bufSize = taosTSizeof(buf);
int32_t numOfTables = 10; int32_t numOfTables = 10;
col_id_t numOfCols = 4096; col_id_t numOfCols = 4096;
EXPECT_GT(numOfCols, 0); ASSERT_GT(numOfCols, 0);
pSmaData = (STSmaDataWrapper *)buf; pSmaData = (STSmaDataWrapper *)buf;
printf(">> allocate [%d] time to %d and addr is %p\n", ++allocCnt, bufSize, pSmaData); printf(">> allocate [%d] time to %d and addr is %p\n", ++allocCnt, bufSize, pSmaData);
@ -326,7 +340,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
int32_t tableDataLen = sizeof(STSmaTbData); int32_t tableDataLen = sizeof(STSmaTbData);
for (col_id_t c = 0; c < numOfCols; ++c) { for (col_id_t c = 0; c < numOfCols; ++c) {
if (bufSize - len - tableDataLen < buffer) { if (bufSize - len - tableDataLen < buffer) {
EXPECT_EQ(tsdbMakeRoom(&buf, bufSize + allocStep), 0); ASSERT_EQ(tsdbMakeRoom(&buf, bufSize + allocStep), 0);
pSmaData = (STSmaDataWrapper *)buf; pSmaData = (STSmaDataWrapper *)buf;
pTbData = (STSmaTbData *)POINTER_SHIFT(pSmaData, len); pTbData = (STSmaTbData *)POINTER_SHIFT(pSmaData, len);
bufSize = taosTSizeof(buf); bufSize = taosTSizeof(buf);
@ -353,16 +367,16 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
} }
pSmaData->dataLen = (len - sizeof(STSmaDataWrapper)); pSmaData->dataLen = (len - sizeof(STSmaDataWrapper));
EXPECT_GE(bufSize, pSmaData->dataLen); ASSERT_GE(bufSize, pSmaData->dataLen);
// execute // execute
EXPECT_EQ(tsdbInsertTSmaData(&tsdb, (char *)pSmaData), TSDB_CODE_SUCCESS); ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
// step 3: query // step 3: query
uint32_t checkDataCnt = 0; uint32_t checkDataCnt = 0;
for (int32_t t = 0; t < numOfTables; ++t) { for (int32_t t = 0; t < numOfTables; ++t) {
for (col_id_t c = 0; c < numOfCols; ++c) { for (col_id_t c = 0; c < numOfCols; ++c) {
EXPECT_EQ(tsdbGetTSmaData(&tsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t, ASSERT_EQ(tsdbGetTSmaData(pTsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t,
c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1), c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1),
TSDB_CODE_SUCCESS); TSDB_CODE_SUCCESS);
++checkDataCnt; ++checkDataCnt;
@ -372,9 +386,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt); printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt);
// release data // release data
tfree(msg);
taosTZfree(buf); taosTZfree(buf);
// release meta // release meta
tdDestroyTSma(&tSma); tdDestroyTSma(&tSma);
tfsClose(pTsdb->pTfs);
tsdbClose(pTsdb);
metaClose(pMeta); metaClose(pMeta);
} }
#endif #endif

11
source/libs/catalog/src/catalog.c
View File

@ -2349,6 +2349,9 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet) {
}
int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) {
CTG_API_ENTER(); CTG_API_ENTER();
@ -2394,6 +2397,9 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, const char *pIndexName, SIndexMeta** pIndexMeta) {
}
int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) {
CTG_API_ENTER(); CTG_API_ENTER();
@ -2663,11 +2669,14 @@ _return:
int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) {
CTG_API_ENTER(); CTG_API_ENTER();
int32_t code = 0;
if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
//TODO CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, &pQnodeList));
_return:
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
} }

5
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -190,7 +190,6 @@ static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) {
} }
static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
COPY_SCALAR_FIELD(id);
CLONE_NODE_LIST_FIELD(pTargets); CLONE_NODE_LIST_FIELD(pTargets);
CLONE_NODE_FIELD(pConditions); CLONE_NODE_FIELD(pConditions);
CLONE_NODE_LIST_FIELD(pChildren); CLONE_NODE_LIST_FIELD(pChildren);
@ -198,7 +197,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
} }
static STableMeta* tableMetaClone(const STableMeta* pSrc) { static STableMeta* tableMetaClone(const STableMeta* pSrc) {
int32_t len = sizeof(STableMeta) + (pSrc->tableInfo.numOfTags + pSrc->tableInfo.numOfColumns) * sizeof(SSchema); int32_t len = TABLE_META_SIZE(pSrc);
STableMeta* pDst = malloc(len); STableMeta* pDst = malloc(len);
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return NULL;
@ -208,7 +207,7 @@ static STableMeta* tableMetaClone(const STableMeta* pSrc) {
} }
static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) { static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) {
int32_t len = sizeof(SVgroupsInfo) + pSrc->numOfVgroups * sizeof(SVgroupInfo); int32_t len = VGROUPS_INFO_SIZE(pSrc);
SVgroupsInfo* pDst = malloc(len); SVgroupsInfo* pDst = malloc(len);
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return NULL;

272
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -193,7 +193,17 @@ static int32_t tableMetaToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static const char* jkLogicPlanId = "Id"; static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) {
STableMeta* pNode = (STableMeta*)pObj;
int32_t code = tjsonGetUBigIntValue(pJson, jkTableMetaUid, &pNode->uid);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUBigIntValue(pJson, jkTableMetaSuid, &pNode->suid);
}
return code;
}
static const char* jkLogicPlanTargets = "Targets"; static const char* jkLogicPlanTargets = "Targets";
static const char* jkLogicPlanConditions = "Conditions"; static const char* jkLogicPlanConditions = "Conditions";
static const char* jkLogicPlanChildren = "Children"; static const char* jkLogicPlanChildren = "Children";
@ -201,10 +211,7 @@ static const char* jkLogicPlanChildren = "Children";
static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) {
const SLogicNode* pNode = (const SLogicNode*)pObj; const SLogicNode* pNode = (const SLogicNode*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkLogicPlanId, pNode->id); int32_t code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkLogicPlanConditions, nodeToJson, pNode->pConditions); code = tjsonAddObject(pJson, jkLogicPlanConditions, nodeToJson, pNode->pConditions);
} }
@ -445,6 +452,14 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) {
return physiScanNodeToJson(pObj, pJson);
}
static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) {
return jsonToPhysiScanNode(pJson, pObj);
}
static const char* jkProjectPhysiPlanProjections = "Projections"; static const char* jkProjectPhysiPlanProjections = "Projections";
static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
@ -1273,6 +1288,193 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkTableDbName = "DbName";
static const char* jkTableTableName = "tableName";
static const char* jkTableTableAlias = "tableAlias";
static int32_t tableNodeToJson(const void* pObj, SJson* pJson) {
const STableNode* pNode = (const STableNode*)pObj;
int32_t code = exprNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableTableAlias, pNode->tableAlias);
}
return code;
}
static int32_t jsonToTableNode(const SJson* pJson, void* pObj) {
STableNode* pNode = (STableNode*)pObj;
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableTableAlias, pNode->tableAlias);
}
return code;
}
static const char* jkEpSetInUse = "InUse";
static const char* jkEpSetNumOfEps = "NumOfEps";
static const char* jkEpSetEps = "Eps";
static int32_t epSetToJson(const void* pObj, SJson* pJson) {
const SEpSet* pNode = (const SEpSet*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps);
}
return code;
}
static int32_t jsonToEpSet(const SJson* pJson, void* pObj) {
SEpSet* pNode = (SEpSet*)pObj;
int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp));
}
return code;
}
static const char* jkVgroupInfoVgId = "VgId";
static const char* jkVgroupInfoHashBegin = "HashBegin";
static const char* jkVgroupInfoHashEnd = "HashEnd";
static const char* jkVgroupInfoEpSet = "EpSet";
static const char* jkVgroupInfoNumOfTable = "NumOfTable";
static int32_t vgroupInfoToJson(const void* pObj, SJson* pJson) {
const SVgroupInfo* pNode = (const SVgroupInfo*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupInfoVgId, pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashBegin, pNode->hashBegin);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashEnd, pNode->hashEnd);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkVgroupInfoEpSet, epSetToJson, &pNode->epSet);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkVgroupInfoNumOfTable, pNode->numOfTable);
}
return code;
}
static int32_t jsonToVgroupInfo(const SJson* pJson, void* pObj) {
SVgroupInfo* pNode = (SVgroupInfo*)pObj;
int32_t code = tjsonGetIntValue(pJson, jkVgroupInfoVgId, &pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUIntValue(pJson, jkVgroupInfoHashBegin, &pNode->hashBegin);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUIntValue(pJson, jkVgroupInfoHashEnd, &pNode->hashEnd);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkVgroupInfoEpSet, jsonToEpSet, &pNode->epSet);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkVgroupInfoNumOfTable, &pNode->numOfTable);
}
return code;
}
static const char* jkVgroupsInfoNum = "Num";
static const char* jkVgroupsInfoVgroups = "Vgroups";
static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) {
const SVgroupsInfo* pNode = (const SVgroupsInfo*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), pNode->numOfVgroups);
}
return code;
}
static int32_t jsonToVgroupsInfo(const SJson* pJson, void* pObj) {
SVgroupsInfo* pNode = (SVgroupsInfo*)pObj;
int32_t code = tjsonGetIntValue(pJson, jkVgroupsInfoNum, &pNode->numOfVgroups);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToArray(pJson, jkVgroupsInfoVgroups, jsonToVgroupInfo, pNode->vgroups, sizeof(SVgroupInfo));
}
return code;
}
static const char* jkRealTableMetaSize = "MetaSize";
static const char* jkRealTableMeta = "Meta";
static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize";
static const char* jkRealTableVgroupsInfo = "VgroupsInfo";
static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) {
const SRealTableNode* pNode = (const SRealTableNode*)pObj;
int32_t code = tableNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkRealTableMetaSize, TABLE_META_SIZE(pNode->pMeta));
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkRealTableMeta, tableMetaToJson, pNode->pMeta);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkRealTableVgroupsInfoSize, VGROUPS_INFO_SIZE(pNode->pVgroupList));
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkRealTableVgroupsInfo, vgroupsInfoToJson, pNode->pVgroupList);
}
return code;
}
static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) {
SRealTableNode* pNode = (SRealTableNode*)pObj;
int32_t objSize = 0;
int32_t code = jsonToTableNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkRealTableMetaSize, &objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonMakeObject(pJson, jkRealTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkRealTableVgroupsInfoSize, &objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonMakeObject(pJson, jkRealTableVgroupsInfo, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize);
}
return code;
}
static const char* jkGroupingSetType = "GroupingSetType"; static const char* jkGroupingSetType = "GroupingSetType";
static const char* jkGroupingSetParameter = "Parameters"; static const char* jkGroupingSetParameter = "Parameters";
@ -1464,7 +1666,7 @@ static const char* jkSelectStmtSlimit = "Slimit";
static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { static int32_t selectStmtTojson(const void* pObj, SJson* pJson) {
const SSelectStmt* pNode = (const SSelectStmt*)pObj; const SSelectStmt* pNode = (const SSelectStmt*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); int32_t code = tjsonAddBoolToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkSelectStmtProjections, pNode->pProjectionList); code = nodeListToJson(pJson, jkSelectStmtProjections, pNode->pProjectionList);
} }
@ -1499,6 +1701,44 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) {
SSelectStmt* pNode = (SSelectStmt*)pObj;
int32_t code = tjsonGetBoolValue(pJson, jkSelectStmtDistinct, &pNode->isDistinct);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtProjections, &pNode->pProjectionList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtFrom, &pNode->pFromTable);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtWhere, &pNode->pWhere);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtPartitionBy, &pNode->pPartitionByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtWindow, &pNode->pWindow);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtGroupBy, &pNode->pGroupByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtHaving, &pNode->pHaving);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtOrderBy, &pNode->pOrderByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtLimit, &pNode->pLimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtSlimit, &pNode->pSlimit);
}
return code;
}
static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
switch (nodeType(pObj)) { switch (nodeType(pObj)) {
case QUERY_NODE_COLUMN: case QUERY_NODE_COLUMN:
@ -1512,6 +1752,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_FUNCTION: case QUERY_NODE_FUNCTION:
return functionNodeToJson(pObj, pJson); return functionNodeToJson(pObj, pJson);
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
return realTableNodeToJson(pObj, pJson);
case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_TEMP_TABLE:
case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_JOIN_TABLE:
break; break;
@ -1565,9 +1806,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return physiTagScanNodeToJson(pObj, pJson); return physiTagScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return physiTableScanNodeToJson(pObj, pJson); return physiTableScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
break; return physiStreamScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return physiProjectNodeToJson(pObj, pJson); return physiProjectNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_JOIN: case QUERY_NODE_PHYSICAL_PLAN_JOIN:
@ -1589,6 +1829,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN: case QUERY_NODE_PHYSICAL_PLAN:
return planToJson(pObj, pJson); return planToJson(pObj, pJson);
default: default:
assert(0);
break; break;
} }
nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj))); nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj)));
@ -1607,7 +1848,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToLogicConditionNode(pJson, pObj); return jsonToLogicConditionNode(pJson, pObj);
case QUERY_NODE_FUNCTION: case QUERY_NODE_FUNCTION:
return jsonToFunctionNode(pJson, pObj); return jsonToFunctionNode(pJson, pObj);
// case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
return jsonToRealTableNode(pJson, pObj);
// case QUERY_NODE_TEMP_TABLE: // case QUERY_NODE_TEMP_TABLE:
// case QUERY_NODE_JOIN_TABLE: // case QUERY_NODE_JOIN_TABLE:
// break; // break;
@ -1633,8 +1875,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToDownstreamSourceNode(pJson, pObj); return jsonToDownstreamSourceNode(pJson, pObj);
// case QUERY_NODE_SET_OPERATOR: // case QUERY_NODE_SET_OPERATOR:
// break; // break;
// case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
// return jsonToSelectStmt(pJson, pObj); return jsonToSelectStmt(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_SCAN: // case QUERY_NODE_LOGIC_PLAN_SCAN:
// return jsonToLogicScanNode(pJson, pObj); // return jsonToLogicScanNode(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_JOIN: // case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -1647,6 +1889,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToPhysiTagScanNode(pJson, pObj); return jsonToPhysiTagScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return jsonToPhysiTableScanNode(pJson, pObj); return jsonToPhysiTableScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
return jsonToPhysiStreamScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return jsonToPhysiProjectNode(pJson, pObj); return jsonToPhysiProjectNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_JOIN: case QUERY_NODE_PHYSICAL_PLAN_JOIN:
@ -1730,7 +1974,7 @@ static int32_t jsonToNodeObject(const SJson* pJson, const char* pName, SNode** p
} }
int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen) { int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen) {
if (NULL == pNode || NULL == pStr || NULL == pLen) { if (NULL == pNode || NULL == pStr) {
terrno = TSDB_CODE_FAILED; terrno = TSDB_CODE_FAILED;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -1750,7 +1994,10 @@ int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_
*pStr = format ? tjsonToString(pJson) : tjsonToUnformattedString(pJson); *pStr = format ? tjsonToString(pJson) : tjsonToUnformattedString(pJson);
tjsonDelete(pJson); tjsonDelete(pJson);
if (NULL != pLen) {
*pLen = strlen(*pStr) + 1; *pLen = strlen(*pStr) + 1;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1765,6 +2012,7 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode) {
int32_t code = makeNodeByJson(pJson, pNode); int32_t code = makeNodeByJson(pJson, pNode);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(*pNode); nodesDestroyNode(*pNode);
*pNode = NULL;
terrno = code; terrno = code;
return code; return code;
} }

27
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -92,6 +92,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateDatabaseStmt)); return makeNode(type, sizeof(SCreateDatabaseStmt));
case QUERY_NODE_DROP_DATABASE_STMT: case QUERY_NODE_DROP_DATABASE_STMT:
return makeNode(type, sizeof(SDropDatabaseStmt)); return makeNode(type, sizeof(SDropDatabaseStmt));
case QUERY_NODE_ALTER_DATABASE_STMT:
return makeNode(type, sizeof(SAlterDatabaseStmt));
case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_DATABASES_STMT:
return makeNode(type, sizeof(SShowStmt)); return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
@ -123,6 +125,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateDnodeStmt)); return makeNode(type, sizeof(SCreateDnodeStmt));
case QUERY_NODE_DROP_DNODE_STMT: case QUERY_NODE_DROP_DNODE_STMT:
return makeNode(type, sizeof(SDropDnodeStmt)); return makeNode(type, sizeof(SDropDnodeStmt));
case QUERY_NODE_ALTER_DNODE_STMT:
return makeNode(type, sizeof(SAlterDnodeStmt));
case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_DNODES_STMT:
return makeNode(type, sizeof(SShowStmt)); return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_VGROUPS_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT:
@ -131,8 +135,16 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SShowStmt)); return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_CREATE_INDEX_STMT: case QUERY_NODE_CREATE_INDEX_STMT:
return makeNode(type, sizeof(SCreateIndexStmt)); return makeNode(type, sizeof(SCreateIndexStmt));
case QUERY_NODE_DROP_INDEX_STMT:
return makeNode(type, sizeof(SDropIndexStmt));
case QUERY_NODE_CREATE_QNODE_STMT: case QUERY_NODE_CREATE_QNODE_STMT:
return makeNode(type, sizeof(SCreateQnodeStmt)); return makeNode(type, sizeof(SCreateQnodeStmt));
case QUERY_NODE_DROP_QNODE_STMT:
return makeNode(type, sizeof(SDropQnodeStmt));
case QUERY_NODE_CREATE_TOPIC_STMT:
return makeNode(type, sizeof(SCreateTopicStmt));
case QUERY_NODE_DROP_TOPIC_STMT:
return makeNode(type, sizeof(SDropTopicStmt));
case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_LOGIC_PLAN_SCAN:
return makeNode(type, sizeof(SScanLogicNode)); return makeNode(type, sizeof(SScanLogicNode));
case QUERY_NODE_LOGIC_PLAN_JOIN: case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -158,7 +170,7 @@ SNodeptr nodesMakeNode(ENodeType type) {
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
return makeNode(type, sizeof(STableSeqScanPhysiNode)); return makeNode(type, sizeof(STableSeqScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
return makeNode(type, sizeof(SNode)); return makeNode(type, sizeof(SStreamScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectPhysiNode)); return makeNode(type, sizeof(SProjectPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_JOIN: case QUERY_NODE_PHYSICAL_PLAN_JOIN:
@ -347,6 +359,17 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc) {
if (NULL == pSrc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = nodesListAppendList(pTarget, pSrc);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pSrc);
}
return code;
}
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) {
if (NULL == pCell->pPrev) { if (NULL == pCell->pPrev) {
pList->pHead = pCell->pNext; pList->pHead = pCell->pNext;
@ -561,7 +584,7 @@ typedef struct SCollectFuncsCxt {
static EDealRes collectFuncs(SNode* pNode, void* pContext) { static EDealRes collectFuncs(SNode* pNode, void* pContext) {
SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext; SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext;
if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId)) { if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId)) {
pCxt->errCode = nodesListAppend(pCxt->pFuncs, pNode); pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode));
return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
} }
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;

26
source/libs/parser/inc/parAst.h
View File

@ -50,8 +50,10 @@ typedef enum EDatabaseOptionType {
DB_OPTION_TTL, DB_OPTION_TTL,
DB_OPTION_WAL, DB_OPTION_WAL,
DB_OPTION_VGROUPS, DB_OPTION_VGROUPS,
DB_OPTION_SINGLESTABLE, DB_OPTION_SINGLE_STABLE,
DB_OPTION_STREAMMODE, DB_OPTION_STREAM_MODE,
DB_OPTION_RETENTIONS,
DB_OPTION_FILE_FACTOR,
DB_OPTION_MAX DB_OPTION_MAX
} EDatabaseOptionType; } EDatabaseOptionType;
@ -65,6 +67,11 @@ typedef enum ETableOptionType {
TABLE_OPTION_MAX TABLE_OPTION_MAX
} ETableOptionType; } ETableOptionType;
typedef struct SAlterOption {
int32_t type;
SToken val;
} SAlterOption;
extern SToken nil_token; extern SToken nil_token;
void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt); void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt);
@ -110,12 +117,16 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt); SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt);
SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt);
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal); SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal);
SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions); SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions);
SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pDbName); SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pDbName);
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt);
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal); SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal);
SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma); SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma);
SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs);
SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment); SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment);
SDataType createDataType(uint8_t type); SDataType createDataType(uint8_t type);
SDataType createVarLenDataType(uint8_t type, const SToken* pLen); SDataType createVarLenDataType(uint8_t type, const SToken* pLen);
@ -125,6 +136,11 @@ SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables
SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables); SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables);
SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions);
SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType);
SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName);
SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName);
SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal);
SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName); SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName);
SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDbName); SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDbName);
SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, const SToken* pPassword); SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, const SToken* pPassword);
@ -132,9 +148,15 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, int
SNode* createDropUserStmt(SAstCreateContext* pCxt, const SToken* pUserName); SNode* createDropUserStmt(SAstCreateContext* pCxt, const SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode); SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode);
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions); SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions);
SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding); SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding);
SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName);
SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId);
SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId);
SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName);
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue);
#ifdef __cplusplus #ifdef __cplusplus
} }

117
source/libs/parser/inc/sql.y
View File

@ -22,6 +22,7 @@
} }
%syntax_error { %syntax_error {
if (pCxt->valid) {
if(TOKEN.z) { if(TOKEN.z) {
generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z);
} else { } else {
@ -29,6 +30,7 @@
} }
pCxt->valid = false; pCxt->valid = false;
} }
}
%left OR. %left OR.
%left AND. %left AND.
@ -41,6 +43,41 @@
%left NK_CONCAT. %left NK_CONCAT.
//%right NK_BITNOT. //%right NK_BITNOT.
/************************************************ create/alter account *****************************************/
cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
cmd ::= ALTER ACCOUNT NK_ID alter_account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
%type account_options { int32_t }
%destructor account_options { }
account_options ::= . { }
account_options ::= account_options PPS literal. { }
account_options ::= account_options TSERIES literal. { }
account_options ::= account_options STORAGE literal. { }
account_options ::= account_options STREAMS literal. { }
account_options ::= account_options QTIME literal. { }
account_options ::= account_options DBS literal. { }
account_options ::= account_options USERS literal. { }
account_options ::= account_options CONNS literal. { }
account_options ::= account_options STATE literal. { }
%type alter_account_options { int32_t }
%destructor alter_account_options { }
alter_account_options ::= alter_account_option. { }
alter_account_options ::= alter_account_options alter_account_option. { }
%type alter_account_option { int32_t }
%destructor alter_account_option { }
alter_account_option ::= PASS literal. { }
alter_account_option ::= PPS literal. { }
alter_account_option ::= TSERIES literal. { }
alter_account_option ::= STORAGE literal. { }
alter_account_option ::= STREAMS literal. { }
alter_account_option ::= QTIME literal. { }
alter_account_option ::= DBS literal. { }
alter_account_option ::= USERS literal. { }
alter_account_option ::= CONNS literal. { }
alter_account_option ::= STATE literal. { }
/************************************************ create/alter/drop/show user *****************************************/ /************************************************ create/alter/drop/show user *****************************************/
cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); } cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); }
cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); } cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); }
@ -48,12 +85,16 @@ cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B).
cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); }
cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); }
/************************************************ create/drop/show dnode **********************************************/ /************************************************ create/drop/alter/show dnode ****************************************/
cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }
cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); } cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); }
cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); }
cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); }
cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); }
cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, NULL); }
cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B) NK_STRING(C). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, &C); }
cmd ::= ALTER ALL DNODES NK_STRING(A). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, NULL); }
cmd ::= ALTER ALL DNODES NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, &B); }
%type dnode_endpoint { SToken } %type dnode_endpoint { SToken }
%destructor dnode_endpoint { } %destructor dnode_endpoint { }
@ -64,8 +105,13 @@ dnode_endpoint(A) ::= NK_STRING(B).
dnode_host_name(A) ::= NK_ID(B). { A = B; } dnode_host_name(A) ::= NK_ID(B). { A = B; }
dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; } dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; }
/************************************************ create qnode ********************************************************/ /************************************************ alter local *********************************************************/
cmd ::= ALTER LOCAL NK_STRING(A). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, NULL); }
cmd ::= ALTER LOCAL NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, &B); }
/************************************************ create/drop qnode ***************************************************/
cmd ::= CREATE QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &A); } cmd ::= CREATE QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &A); }
cmd ::= DROP QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropQnodeStmt(pCxt, &A); }
cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL); } cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL); }
/************************************************ create/drop/show/use database ***************************************/ /************************************************ create/drop/show/use database ***************************************/
@ -73,6 +119,7 @@ cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C).
cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); } cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); }
cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); }
cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); } cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); }
cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
%type not_exists_opt { bool } %type not_exists_opt { bool }
%destructor not_exists_opt { } %destructor not_exists_opt { }
@ -100,8 +147,22 @@ db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C).
db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); } db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); }
db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); }
db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); }
db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); } db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); }
db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); } db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); }
db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); }
db_options(A) ::= db_options(B) FILE_FACTOR NK_FLOAT(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FILE_FACTOR, &C); }
alter_db_options(A) ::= alter_db_option(B). { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseOption(pCxt, A, B.type, &B.val); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseOption(pCxt, B, C.type, &C.val); }
%type alter_db_option { SAlterOption }
%destructor alter_db_option { }
alter_db_option(A) ::= BLOCKS NK_INTEGER(B). { A.type = DB_OPTION_BLOCKS; A.val = B; }
alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; }
alter_db_option(A) ::= KEEP NK_INTEGER(B). { A.type = DB_OPTION_KEEP; A.val = B; }
alter_db_option(A) ::= WAL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; }
alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.val = B; }
alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; }
/************************************************ create/drop/show table/stable ***************************************/ /************************************************ create/drop/show table/stable ***************************************/
cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
@ -114,6 +175,27 @@ cmd ::= DROP STABLE exists_opt(A) full_table_name(B).
cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); }
cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); }
cmd ::= ALTER TABLE alter_table_clause(A). { pCxt->pRootNode = A; }
cmd ::= ALTER STABLE alter_table_clause(A). { pCxt->pRootNode = A; }
alter_table_clause(A) ::= full_table_name(B) alter_table_options(C). { A = createAlterTableOption(pCxt, B, C); }
alter_table_clause(A) ::=
full_table_name(B) ADD COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); }
alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); }
alter_table_clause(A) ::=
full_table_name(B) MODIFY COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &C, D); }
alter_table_clause(A) ::=
full_table_name(B) RENAME COLUMN column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &C, &D); }
alter_table_clause(A) ::=
full_table_name(B) ADD TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_TAG, &C, D); }
alter_table_clause(A) ::= full_table_name(B) DROP TAG column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_TAG, &C); }
alter_table_clause(A) ::=
full_table_name(B) MODIFY TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &C, D); }
alter_table_clause(A) ::=
full_table_name(B) RENAME TAG column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); }
alter_table_clause(A) ::=
full_table_name(B) SET TAG column_name(C) NK_EQ literal(D). { A = createAlterTableSetTag(pCxt, B, &C, D); }
%type multi_create_clause { SNodeList* } %type multi_create_clause { SNodeList* }
%destructor multi_create_clause { nodesDestroyList($$); } %destructor multi_create_clause { nodesDestroyList($$); }
multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); } multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); }
@ -186,6 +268,16 @@ table_options(A) ::= table_options(B) COMMENT NK_STRING(C).
table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); } table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); }
table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); }
table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); }
table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableRollupOption(pCxt, B, C); }
alter_table_options(A) ::= alter_table_option(B). { A = createDefaultAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); }
alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableOption(pCxt, B, C.type, &C.val); }
%type alter_table_option { SAlterOption }
%destructor alter_table_option { }
alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.val = B; }
alter_table_option(A) ::= KEEP NK_INTEGER(B). { A.type = TABLE_OPTION_KEEP; A.val = B; }
alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.val = B; }
%type col_name_list { SNodeList* } %type col_name_list { SNodeList* }
%destructor col_name_list { nodesDestroyList($$); } %destructor col_name_list { nodesDestroyList($$); }
@ -194,10 +286,18 @@ col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C).
col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); } col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); }
%type func_name_list { SNodeList* }
%destructor func_name_list { nodesDestroyList($$); }
func_name_list(A) ::= func_name(B). { A = createNodeList(pCxt, B); }
func_name_list(A) ::= func_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); }
func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); }
/************************************************ create index ********************************************************/ /************************************************ create index ********************************************************/
cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); } cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); }
cmd ::= CREATE FULLTEXT INDEX cmd ::= CREATE FULLTEXT INDEX
index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &A, &B, C, NULL); } index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &A, &B, C, NULL); }
cmd ::= DROP INDEX index_name(A) ON table_name(B). { pCxt->pRootNode = createDropIndexStmt(pCxt, &A, &B); }
index_options(A) ::= . { A = NULL; } index_options(A) ::= . { A = NULL; }
index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL
@ -212,6 +312,11 @@ func_list(A) ::= func_list(B) NK_COMMA func(C).
func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); }
/************************************************ create/drop topic ***************************************************/
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, C, NULL); }
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS db_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, &C); }
cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); }
/************************************************ show vgroups ********************************************************/ /************************************************ show vgroups ********************************************************/
cmd ::= SHOW VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, NULL); } cmd ::= SHOW VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, NULL); }
cmd ::= SHOW db_name(B) NK_DOT VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &B); } cmd ::= SHOW db_name(B) NK_DOT VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &B); }
@ -270,6 +375,10 @@ user_name(A) ::= NK_ID(B).
%destructor index_name { } %destructor index_name { }
index_name(A) ::= NK_ID(B). { A = B; } index_name(A) ::= NK_ID(B). { A = B; }
%type topic_name { SToken }
%destructor topic_name { }
topic_name(A) ::= NK_ID(B). { A = B; }
/************************************************ expression **********************************************************/ /************************************************ expression **********************************************************/
expression(A) ::= literal(B). { A = B; } expression(A) ::= literal(B). { A = B; }
//expression(A) ::= NK_QUESTION(B). { A = B; } //expression(A) ::= NK_QUESTION(B). { A = B; }

165
source/libs/parser/src/parAstCreater.c
View File

@ -246,6 +246,16 @@ static SDatabaseOptions* setDbStreamMode(SAstCreateContext* pCxt, SDatabaseOptio
return pOptions; return pOptions;
} }
static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
// todo
return pOptions;
}
static SDatabaseOptions* setDbFileFactor(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
// todo
return pOptions;
}
static void initSetDatabaseOptionFp() { static void initSetDatabaseOptionFp() {
setDbOptionFuncs[DB_OPTION_BLOCKS] = setDbBlocks; setDbOptionFuncs[DB_OPTION_BLOCKS] = setDbBlocks;
setDbOptionFuncs[DB_OPTION_CACHE] = setDbCache; setDbOptionFuncs[DB_OPTION_CACHE] = setDbCache;
@ -262,8 +272,10 @@ static void initSetDatabaseOptionFp() {
setDbOptionFuncs[DB_OPTION_TTL] = setDbTtl; setDbOptionFuncs[DB_OPTION_TTL] = setDbTtl;
setDbOptionFuncs[DB_OPTION_WAL] = setDbWal; setDbOptionFuncs[DB_OPTION_WAL] = setDbWal;
setDbOptionFuncs[DB_OPTION_VGROUPS] = setDbVgroups; setDbOptionFuncs[DB_OPTION_VGROUPS] = setDbVgroups;
setDbOptionFuncs[DB_OPTION_SINGLESTABLE] = setDbSingleStable; setDbOptionFuncs[DB_OPTION_SINGLE_STABLE] = setDbSingleStable;
setDbOptionFuncs[DB_OPTION_STREAMMODE] = setDbStreamMode; setDbOptionFuncs[DB_OPTION_STREAM_MODE] = setDbStreamMode;
setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions;
setDbOptionFuncs[DB_OPTION_FILE_FACTOR] = setDbFileFactor;
} }
static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) { static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
@ -772,6 +784,29 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
return (SNode*)pOptions; return (SNode*)pOptions;
} }
SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->numOfBlocks = -1;
pOptions->cacheBlockSize = -1;
pOptions->cachelast = -1;
pOptions->compressionLevel = -1;
pOptions->daysPerFile = -1;
pOptions->fsyncPeriod = -1;
pOptions->maxRowsPerBlock = -1;
pOptions->minRowsPerBlock = -1;
pOptions->keep = -1;
pOptions->precision = -1;
pOptions->quorum = -1;
pOptions->replica = -1;
pOptions->ttl = -1;
pOptions->walLevel = -1;
pOptions->numOfVgroups = -1;
pOptions->singleStable = -1;
pOptions->streamMode = -1;
return (SNode*)pOptions;
}
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal) { SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal) {
return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal); return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal);
} }
@ -799,6 +834,17 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, con
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions) {
if (!checkDbName(pCxt, pDbName)) {
return NULL;
}
SAlterDatabaseStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
strncpy(pStmt->dbName, pDbName->z, pDbName->n);
pStmt->pOptions = (SDatabaseOptions*)pOptions;
return (SNode*)pStmt;
}
SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS); STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
@ -807,6 +853,14 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
return (SNode*)pOptions; return (SNode*)pOptions;
} }
SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt) {
STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->keep = -1;
pOptions->ttl = -1;
return (SNode*)pOptions;
}
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal) { SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal) {
return (SNode*)setTableOptionFuncs[type](pCxt, (STableOptions*)pOptions, pVal); return (SNode*)setTableOptionFuncs[type](pCxt, (STableOptions*)pOptions, pVal);
} }
@ -816,6 +870,11 @@ SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pS
return pOptions; return pOptions;
} }
SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs) {
// todo
return pOptions;
}
SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) { SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) {
SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF);
CHECK_OUT_OF_MEM(pCol); CHECK_OUT_OF_MEM(pCol);
@ -901,6 +960,49 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_OPTIONS;
pStmt->pOptions = (STableOptions*)pOptions;
return (SNode*)pStmt;
}
SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
strncpy(pStmt->colName, pColName->z, pColName->n);
pStmt->dataType = dataType;
return (SNode*)pStmt;
}
SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
strncpy(pStmt->colName, pColName->z, pColName->n);
return (SNode*)pStmt;
}
SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
strncpy(pStmt->colName, pOldColName->z, pOldColName->n);
strncpy(pStmt->newColName, pNewColName->z, pNewColName->n);
return (SNode*)pStmt;
}
SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_TAG_VAL;
strncpy(pStmt->colName, pTagName->z, pTagName->n);
pStmt->pVal = (SValueNode*)pVal;
return (SNode*)pStmt;
}
SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) { SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
SUseDatabaseStmt* pStmt = (SUseDatabaseStmt*)nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT); SUseDatabaseStmt* pStmt = (SUseDatabaseStmt*)nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
@ -998,6 +1100,17 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode) {
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue) {
SAlterDnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DNODE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->dnodeId = strtol(pDnode->z, NULL, 10);
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
}
return (SNode*)pStmt;
}
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions) { SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions) {
if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) {
return NULL; return NULL;
@ -1022,9 +1135,57 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt
return (SNode*)pOptions; return (SNode*)pOptions;
} }
SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName) {
if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) {
return NULL;
}
SDropIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT);
CHECK_OUT_OF_MEM(pStmt);
strncpy(pStmt->indexName, pIndexName->z, pIndexName->n);
strncpy(pStmt->tableName, pTableName->z, pTableName->n);
return (SNode*)pStmt;
}
SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId) { SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId) {
SCreateQnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_QNODE_STMT); SCreateQnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_QNODE_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10);; pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10);;
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId) {
SDropQnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_DROP_QNODE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10);;
return (SNode*)pStmt;
}
SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName) {
SCreateTopicStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
strncpy(pStmt->topicName, pTopicName->z, pTopicName->n);
pStmt->ignoreExists = ignoreExists;
pStmt->pQuery = pQuery;
if (NULL != pSubscribeDbName) {
strncpy(pStmt->subscribeDbName, pSubscribeDbName->z, pSubscribeDbName->n);
}
return (SNode*)pStmt;
}
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName) {
SDropTopicStmt* pStmt = nodesMakeNode(QUERY_NODE_DROP_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
strncpy(pStmt->topicName, pTopicName->z, pTopicName->n);
pStmt->ignoreNotExists = ignoreNotExists;
return (SNode*)pStmt;
}
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue) {
SAlterLocalStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_LOCAL_STMT);
CHECK_OUT_OF_MEM(pStmt);
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
}
return (SNode*)pStmt;
}

12
source/libs/parser/src/parInsert.c
View File

@ -391,7 +391,7 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha
} }
// Remove quotation marks // Remove quotation marks
if (TSDB_DATA_TYPE_BINARY == type) { if (TK_NK_STRING == pToken->type) {
if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
} }
@ -617,18 +617,18 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p
if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { if (TSDB_DATA_TYPE_BINARY == pa->schema->type) {
const char* rowEnd = tdRowEnd(rb->pBuf); const char* rowEnd = tdRowEnd(rb->pBuf);
STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len); STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len);
tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, true, pa->toffset, pa->colIdx);
} else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0; int32_t output = 0;
const char* rowEnd = tdRowEnd(rb->pBuf); const char* rowEnd = tdRowEnd(rb->pBuf);
if (!taosMbsToUcs4(value, len, (char*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
} }
varDataSetLen(rowEnd, output); varDataSetLen(rowEnd, output);
tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, true, pa->toffset, pa->colIdx);
} else { } else {
tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, true, pa->toffset, pa->colIdx); tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -725,7 +725,7 @@ static int32_t KvRowAppend(const void *value, int32_t len, void *param) {
} else if (TSDB_DATA_TYPE_NCHAR == type) { } else if (TSDB_DATA_TYPE_NCHAR == type) {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0; int32_t output = 0;
if (!taosMbsToUcs4(value, len, varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
} }

4
source/libs/parser/src/parTokenizer.c
View File

@ -28,6 +28,7 @@ typedef struct SKeyword {
// keywords in sql string // keywords in sql string
static SKeyword keywordTable[] = { static SKeyword keywordTable[] = {
{"ACCOUNT", TK_ACCOUNT},
{"ALL", TK_ALL}, {"ALL", TK_ALL},
{"ALTER", TK_ALTER}, {"ALTER", TK_ALTER},
{"AND", TK_AND}, {"AND", TK_AND},
@ -121,6 +122,7 @@ static SKeyword keywordTable[] = {
{"TAGS", TK_TAGS}, {"TAGS", TK_TAGS},
{"TIMESTAMP", TK_TIMESTAMP}, {"TIMESTAMP", TK_TIMESTAMP},
{"TINYINT", TK_TINYINT}, {"TINYINT", TK_TINYINT},
{"TOPIC", TK_TOPIC},
{"TTL", TK_TTL}, {"TTL", TK_TTL},
{"UNION", TK_UNION}, {"UNION", TK_UNION},
{"UNSIGNED", TK_UNSIGNED}, {"UNSIGNED", TK_UNSIGNED},
@ -167,7 +169,6 @@ static SKeyword keywordTable[] = {
// {"SCORES", TK_SCORES}, // {"SCORES", TK_SCORES},
// {"GRANTS", TK_GRANTS}, // {"GRANTS", TK_GRANTS},
// {"DOT", TK_DOT}, // {"DOT", TK_DOT},
// {"ACCOUNT", TK_ACCOUNT},
// {"DESCRIBE", TK_DESCRIBE}, // {"DESCRIBE", TK_DESCRIBE},
// {"SYNCDB", TK_SYNCDB}, // {"SYNCDB", TK_SYNCDB},
// {"LOCAL", TK_LOCAL}, // {"LOCAL", TK_LOCAL},
@ -230,7 +231,6 @@ static SKeyword keywordTable[] = {
// {"TBNAME", TK_TBNAME}, // {"TBNAME", TK_TBNAME},
// {"VNODES", TK_VNODES}, // {"VNODES", TK_VNODES},
// {"PARTITIONS", TK_PARTITIONS}, // {"PARTITIONS", TK_PARTITIONS},
// {"TOPIC", TK_TOPIC},
// {"TOPICS", TK_TOPICS}, // {"TOPICS", TK_TOPICS},
// {"COMPACT", TK_COMPACT}, // {"COMPACT", TK_COMPACT},
// {"MODIFY", TK_MODIFY}, // {"MODIFY", TK_MODIFY},

293
source/libs/parser/src/parTranslater.c
View File

@ -21,14 +21,6 @@
#include "parUtil.h" #include "parUtil.h"
#include "ttime.h" #include "ttime.h"
static bool afterGroupBy(ESqlClause clause) {
return clause > SQL_CLAUSE_GROUP_BY;
}
static bool beforeHaving(ESqlClause clause) {
return clause < SQL_CLAUSE_HAVING;
}
typedef struct STranslateContext { typedef struct STranslateContext {
SParseContext* pParseCxt; SParseContext* pParseCxt;
int32_t errCode; int32_t errCode;
@ -41,6 +33,15 @@ typedef struct STranslateContext {
} STranslateContext; } STranslateContext;
static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode);
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode);
static bool afterGroupBy(ESqlClause clause) {
return clause > SQL_CLAUSE_GROUP_BY;
}
static bool beforeHaving(ESqlClause clause) {
return clause < SQL_CLAUSE_HAVING;
}
static EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t errCode, ...) { static EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t errCode, ...) {
va_list vArgList; va_list vArgList;
@ -498,6 +499,10 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect)
} }
static int32_t setTableVgroupList(SParseContext* pCxt, SName* name, SRealTableNode* pRealTable) { static int32_t setTableVgroupList(SParseContext* pCxt, SName* name, SRealTableNode* pRealTable) {
if (pCxt->streamQuery) {
return TSDB_CODE_SUCCESS;
}
if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) {
SArray* vgroupList = NULL; SArray* vgroupList = NULL;
int32_t code = catalogGetTableDistVgInfo(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, name, &vgroupList); int32_t code = catalogGetTableDistVgInfo(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, name, &vgroupList);
@ -833,6 +838,41 @@ static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, SAlterDbReq* pReq) {
SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, pReq->db);
pReq->totalBlocks = pStmt->pOptions->numOfBlocks;
pReq->daysToKeep0 = pStmt->pOptions->keep;
pReq->daysToKeep1 = -1;
pReq->daysToKeep2 = -1;
pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
pReq->walLevel = pStmt->pOptions->walLevel;
pReq->quorum = pStmt->pOptions->quorum;
pReq->cacheLastRow = pStmt->pOptions->cachelast;
return;
}
static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) {
SAlterDbReq alterReq = {0};
buildAlterDbReq(pCxt, pStmt, &alterReq);
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_DB;
pCxt->pCmdMsg->msgLen = tSerializeSAlterDbReq(NULL, 0, &alterReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSAlterDbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &alterReq);
return TSDB_CODE_SUCCESS;
}
static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) { static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) {
*pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
SNode* pNode; SNode* pNode;
@ -926,6 +966,73 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS
return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists); return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists);
} }
static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) {
pAlterReq->pFields = taosArrayInit(2, sizeof(TAOS_FIELD));
if (NULL == pAlterReq->pFields) {
return TSDB_CODE_OUT_OF_MEMORY;
}
switch (pStmt->alterType) {
case TSDB_ALTER_TABLE_ADD_TAG:
case TSDB_ALTER_TABLE_DROP_TAG:
case TSDB_ALTER_TABLE_ADD_COLUMN:
case TSDB_ALTER_TABLE_DROP_COLUMN:
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: {
TAOS_FIELD field = { .type = pStmt->dataType.type, .bytes = pStmt->dataType.bytes };
strcpy(field.name, pStmt->colName);
taosArrayPush(pAlterReq->pFields, &field);
break;
}
case TSDB_ALTER_TABLE_UPDATE_TAG_NAME:
case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: {
TAOS_FIELD oldField = {0};
strcpy(oldField.name, pStmt->colName);
taosArrayPush(pAlterReq->pFields, &oldField);
TAOS_FIELD newField = {0};
strcpy(oldField.name, pStmt->newColName);
taosArrayPush(pAlterReq->pFields, &newField);
break;
}
default:
break;
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) {
SMAltertbReq alterReq = {0};
SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
strcpy(tableName.dbname, pStmt->dbName);
strcpy(tableName.tname, pStmt->tableName);
tNameExtractFullName(&tableName, alterReq.name);
alterReq.alterType = pStmt->alterType;
alterReq.numOfFields = 1;
if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) {
// todo
} else {
if (TSDB_CODE_SUCCESS != setAlterTableField(pStmt, &alterReq)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_STB;
pCxt->pCmdMsg->msgLen = tSerializeSMAlterStbReq(NULL, 0, &alterReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSMAlterStbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &alterReq);
return TSDB_CODE_SUCCESS;
}
static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) { static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) {
SName name = {0}; SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
@ -1063,6 +1170,28 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pStmt) {
SMCfgDnodeReq cfgReq = {0};
cfgReq.dnodeId = pStmt->dnodeId;
strcpy(cfgReq.config, pStmt->config);
strcpy(cfgReq.value, pStmt->value);
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_CONFIG_DNODE;
pCxt->pCmdMsg->msgLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSMCfgDnodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &cfgReq);
return TSDB_CODE_SUCCESS;
}
static int32_t nodeTypeToShowType(ENodeType nt) { static int32_t nodeTypeToShowType(ENodeType nt) {
switch (nt) { switch (nt) {
case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_DATABASES_STMT:
@ -1201,6 +1330,27 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p
} }
} }
static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) {
SVDropTSmaReq dropSmaReq = {0};
strcpy(dropSmaReq.indexName, pStmt->indexName);
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA;
pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
void* pBuf = pCxt->pCmdMsg->pMsg;
tSerializeSVDropTSmaReq(&pBuf, &dropSmaReq);
return TSDB_CODE_SUCCESS;
}
static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* pStmt) { static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* pStmt) {
SMCreateQnodeReq createReq = { .dnodeId = pStmt->dnodeId }; SMCreateQnodeReq createReq = { .dnodeId = pStmt->dnodeId };
@ -1220,6 +1370,99 @@ static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* p
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateDropQnode(STranslateContext* pCxt, SDropQnodeStmt* pStmt) {
SDDropQnodeReq dropReq = { .dnodeId = pStmt->dnodeId };
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_DND_DROP_QNODE;
pCxt->pCmdMsg->msgLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSMCreateDropQSBNodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &dropReq);
return TSDB_CODE_SUCCESS;
}
static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) {
SCMCreateTopicReq createReq = {0};
if (NULL != pStmt->pQuery) {
pCxt->pParseCxt->streamQuery = true;
int32_t code = translateQuery(pCxt, pStmt->pQuery);
if (TSDB_CODE_SUCCESS == code) {
code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL);
}
if (TSDB_CODE_SUCCESS != code ) {
return code;
}
} else {
strcpy(createReq.subscribeDbName, pStmt->subscribeDbName);
}
createReq.sql = strdup(pCxt->pParseCxt->pSql);
if (NULL == createReq.sql) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
strcpy(name.dbname, pCxt->pParseCxt->db);
strcpy(name.tname, pStmt->topicName);
tNameExtractFullName(&name, createReq.name);
createReq.igExists = pStmt->ignoreExists;
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_TOPIC;
pCxt->pCmdMsg->msgLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSCMCreateTopicReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createReq);
tFreeSCMCreateTopicReq(&createReq);
return TSDB_CODE_SUCCESS;
}
static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt) {
SMDropTopicReq dropReq = {0};
SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
strcpy(name.dbname, pCxt->pParseCxt->db);
strcpy(name.tname, pStmt->topicName);
tNameExtractFullName(&name, dropReq.name);
dropReq.igNotExists = pStmt->ignoreNotExists;
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_DROP_TOPIC;
pCxt->pCmdMsg->msgLen = tSerializeSMDropTopicReq(NULL, 0, &dropReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSMDropTopicReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &dropReq);
return TSDB_CODE_SUCCESS;
}
static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) {
// todo
return TSDB_CODE_SUCCESS;
}
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pNode)) { switch (nodeType(pNode)) {
@ -1232,6 +1475,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DROP_DATABASE_STMT: case QUERY_NODE_DROP_DATABASE_STMT:
code = translateDropDatabase(pCxt, (SDropDatabaseStmt*)pNode); code = translateDropDatabase(pCxt, (SDropDatabaseStmt*)pNode);
break; break;
case QUERY_NODE_ALTER_DATABASE_STMT:
code = translateAlterDatabase(pCxt, (SAlterDatabaseStmt*)pNode);
break;
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
code = translateCreateSuperTable(pCxt, (SCreateTableStmt*)pNode); code = translateCreateSuperTable(pCxt, (SCreateTableStmt*)pNode);
break; break;
@ -1241,6 +1487,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DROP_SUPER_TABLE_STMT: case QUERY_NODE_DROP_SUPER_TABLE_STMT:
code = translateDropSuperTable(pCxt, (SDropSuperTableStmt*)pNode); code = translateDropSuperTable(pCxt, (SDropSuperTableStmt*)pNode);
break; break;
case QUERY_NODE_ALTER_TABLE_STMT:
code = translateAlterTable(pCxt, (SAlterTableStmt*)pNode);
break;
case QUERY_NODE_CREATE_USER_STMT: case QUERY_NODE_CREATE_USER_STMT:
code = translateCreateUser(pCxt, (SCreateUserStmt*)pNode); code = translateCreateUser(pCxt, (SCreateUserStmt*)pNode);
break; break;
@ -1259,6 +1508,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DROP_DNODE_STMT: case QUERY_NODE_DROP_DNODE_STMT:
code = translateDropDnode(pCxt, (SDropDnodeStmt*)pNode); code = translateDropDnode(pCxt, (SDropDnodeStmt*)pNode);
break; break;
case QUERY_NODE_ALTER_DNODE_STMT:
code = translateAlterDnode(pCxt, (SAlterDnodeStmt*)pNode);
break;
case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_DATABASES_STMT:
case QUERY_NODE_SHOW_STABLES_STMT: case QUERY_NODE_SHOW_STABLES_STMT:
case QUERY_NODE_SHOW_USERS_STMT: case QUERY_NODE_SHOW_USERS_STMT:
@ -1274,9 +1526,24 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_CREATE_INDEX_STMT: case QUERY_NODE_CREATE_INDEX_STMT:
code = translateCreateIndex(pCxt, (SCreateIndexStmt*)pNode); code = translateCreateIndex(pCxt, (SCreateIndexStmt*)pNode);
break; break;
case QUERY_NODE_DROP_INDEX_STMT:
code = translateDropIndex(pCxt, (SDropIndexStmt*)pNode);
break;
case QUERY_NODE_CREATE_QNODE_STMT: case QUERY_NODE_CREATE_QNODE_STMT:
code = translateCreateQnode(pCxt, (SCreateQnodeStmt*)pNode); code = translateCreateQnode(pCxt, (SCreateQnodeStmt*)pNode);
break; break;
case QUERY_NODE_DROP_QNODE_STMT:
code = translateDropQnode(pCxt, (SDropQnodeStmt*)pNode);
break;
case QUERY_NODE_CREATE_TOPIC_STMT:
code = translateCreateTopic(pCxt, (SCreateTopicStmt*)pNode);
break;
case QUERY_NODE_DROP_TOPIC_STMT:
code = translateDropTopic(pCxt, (SDropTopicStmt*)pNode);
break;
case QUERY_NODE_ALTER_LOCAL_STMT:
code = translateAlterLocal(pCxt, (SAlterLocalStmt*)pNode);
break;
default: default:
break; break;
} }
@ -1696,6 +1963,11 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
return rewriteToVnodeModifOpStmt(pQuery, pBufArray); return rewriteToVnodeModifOpStmt(pQuery, pBufArray);
} }
static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) {
// todo
return TSDB_CODE_SUCCESS;
}
static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pQuery->pRoot)) { switch (nodeType(pQuery->pRoot)) {
@ -1707,6 +1979,11 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_CREATE_MULTI_TABLE_STMT: case QUERY_NODE_CREATE_MULTI_TABLE_STMT:
code = rewriteCreateMultiTable(pCxt, pQuery); code = rewriteCreateMultiTable(pCxt, pQuery);
break; break;
case QUERY_NODE_ALTER_TABLE_STMT:
if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == ((SAlterTableStmt*)pQuery->pRoot)->alterType) {
code = rewriteAlterTable(pCxt, pQuery);
}
break;
default: default:
break; break;
} }

14
source/libs/parser/src/parUtil.c
View File

@ -46,17 +46,19 @@ static char* getSyntaxErrFormat(int32_t errCode) {
case TSDB_CODE_PAR_NOT_SINGLE_GROUP: case TSDB_CODE_PAR_NOT_SINGLE_GROUP:
return "Not a single-group group function"; return "Not a single-group group function";
case TSDB_CODE_PAR_TAGS_NOT_MATCHED: case TSDB_CODE_PAR_TAGS_NOT_MATCHED:
return "tags number not matched"; return "Tags number not matched";
case TSDB_CODE_PAR_INVALID_TAG_NAME: case TSDB_CODE_PAR_INVALID_TAG_NAME:
return "invalid tag name : %s"; return "Invalid tag name : %s";
case TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG: case TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG:
return "name or password too long"; return "Name or password too long";
case TSDB_CODE_PAR_PASSWD_EMPTY: case TSDB_CODE_PAR_PASSWD_EMPTY:
return "password can not be empty"; return "Password can not be empty";
case TSDB_CODE_PAR_INVALID_PORT: case TSDB_CODE_PAR_INVALID_PORT:
return "port should be an integer that is less than 65535 and greater than 0"; return "Port should be an integer that is less than 65535 and greater than 0";
case TSDB_CODE_PAR_INVALID_ENDPOINT: case TSDB_CODE_PAR_INVALID_ENDPOINT:
return "endpoint should be in the format of 'fqdn:port'"; return "Endpoint should be in the format of 'fqdn:port'";
case TSDB_CODE_PAR_EXPRIE_STATEMENT:
return "This statement is no longer supported";
case TSDB_CODE_OUT_OF_MEMORY: case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory"; return "Out of memory";
default: default:

3711
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

3
source/libs/parser/test/mockCatalogService.cpp
View File

@ -13,12 +13,11 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "mockCatalogService.h"
#include <iomanip> #include <iomanip>
#include <iostream> #include <iostream>
#include <map> #include <map>
#include "tdatablock.h" #include "tdatablock.h"
#include "mockCatalogService.h"
#include "tname.h" #include "tname.h"
#include "ttypes.h" #include "ttypes.h"

74
source/libs/parser/test/parserAstTest.cpp
View File

@ -302,6 +302,13 @@ TEST_F(ParserTest, createUser) {
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(ParserTest, alterAccount) {
setDatabase("root", "test");
bind("alter account ac_wxy pass '123456'");
ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT));
}
TEST_F(ParserTest, createDnode) { TEST_F(ParserTest, createDnode) {
setDatabase("root", "test"); setDatabase("root", "test");
@ -312,6 +319,16 @@ TEST_F(ParserTest, createDnode) {
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(ParserTest, alterDnode) {
setDatabase("root", "test");
bind("alter dnode 1 'resetLog'");
ASSERT_TRUE(run());
bind("alter dnode 1 'debugFlag' '134'");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, createDatabase) { TEST_F(ParserTest, createDatabase) {
setDatabase("root", "test"); setDatabase("root", "test");
@ -340,6 +357,23 @@ TEST_F(ParserTest, createDatabase) {
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(ParserTest, alterDatabase) {
setDatabase("root", "test");
bind("alter database wxy_db BLOCKS 200");
ASSERT_TRUE(run());
bind("alter database wxy_db "
"BLOCKS 200 "
"CACHELAST 1 "
"FSYNC 200 "
"KEEP 200 "
"QUORUM 2 "
"WAL 1 "
);
ASSERT_TRUE(run());
}
TEST_F(ParserTest, showDatabase) { TEST_F(ParserTest, showDatabase) {
setDatabase("root", "test"); setDatabase("root", "test");
@ -406,9 +440,49 @@ TEST_F(ParserTest, createSmaIndex) {
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(ParserTest, dropIndex) {
setDatabase("root", "test");
bind("drop index index1 on t1");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, createQnode) { TEST_F(ParserTest, createQnode) {
setDatabase("root", "test"); setDatabase("root", "test");
bind("create qnode on dnode 1"); bind("create qnode on dnode 1");
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(ParserTest, dropQnode) {
setDatabase("root", "test");
bind("drop qnode on dnode 1");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, createTopic) {
setDatabase("root", "test");
bind("create topic tp1 as select * from t1");
ASSERT_TRUE(run());
bind("create topic if not exists tp1 as select * from t1");
ASSERT_TRUE(run());
bind("create topic tp1 as test");
ASSERT_TRUE(run());
bind("create topic if not exists tp1 as test");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, dropTopic) {
setDatabase("root", "test");
bind("drop topic tp1");
ASSERT_TRUE(run());
bind("drop topic if exists tp1");
ASSERT_TRUE(run());
}

467
source/libs/planner/src/planLogicCreater.c
View File

@ -18,13 +18,13 @@
#include "functionMgt.h" #include "functionMgt.h"
typedef struct SLogicPlanContext { typedef struct SLogicPlanContext {
int32_t errCode; SPlanContext* pPlanCxt;
int32_t planNodeId;
int32_t acctId;
} SLogicPlanContext; } SLogicPlanContext;
static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt); typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**);
static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable);
static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode);
static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode);
typedef struct SRewriteExprCxt { typedef struct SRewriteExprCxt {
int32_t errCode; int32_t errCode;
@ -66,7 +66,6 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
} }
typedef struct SNameExprCxt { typedef struct SNameExprCxt {
int32_t planNodeId;
int32_t rewriteId; int32_t rewriteId;
} SNameExprCxt; } SNameExprCxt;
@ -76,7 +75,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_LOGIC_CONDITION:
case QUERY_NODE_FUNCTION: { case QUERY_NODE_FUNCTION: {
SNameExprCxt* pCxt = (SNameExprCxt*)pContext; SNameExprCxt* pCxt = (SNameExprCxt*)pContext;
sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId++); sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d", pCxt->rewriteId++);
return DEAL_RES_IGNORE_CHILD; return DEAL_RES_IGNORE_CHILD;
} }
default: default:
@ -86,132 +85,195 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
SNameExprCxt nameCxt = { .planNodeId = planNodeId, .rewriteId = rewriteId }; static int32_t rewriteId = 1;
SNameExprCxt nameCxt = { .rewriteId = rewriteId };
nodesWalkList(pExprs, doNameExpr, &nameCxt); nodesWalkList(pExprs, doNameExpr, &nameCxt);
SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
return cxt.errCode; return cxt.errCode;
} }
static SLogicNode* pushLogicNode(SLogicPlanContext* pCxt, SLogicNode* pRoot, SLogicNode* pNode) { static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLogicNode* pNewRoot) {
if (TSDB_CODE_SUCCESS != pCxt->errCode) { if (NULL == pNewRoot->pChildren) {
goto error; pNewRoot->pChildren = nodesMakeList();
if (NULL == pNewRoot->pChildren) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS != nodesListAppend(pNewRoot->pChildren, (SNode*)*pOldRoot)) {
return TSDB_CODE_OUT_OF_MEMORY;
} }
if (NULL == pRoot) { (*pOldRoot)->pParent = pNewRoot;
return pNode; *pOldRoot = pNewRoot;
return TSDB_CODE_SUCCESS;
} }
if (NULL == pNode) { static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, FCreateLogicNode func, SLogicNode** pRoot) {
return pRoot; SLogicNode* pNode = NULL;
int32_t code = func(pCxt, pSelect, &pNode);
if (TSDB_CODE_SUCCESS == code && NULL != pNode) {
code = pushLogicNode(pCxt, pRoot, pNode);
}
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(pNode);
}
return code;
} }
if (NULL == pNode->pChildren) { static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) {
pNode->pChildren = nodesMakeList();
if (NULL == pNode->pChildren) {
goto error;
}
}
if (TSDB_CODE_SUCCESS != nodesListAppend(pNode->pChildren, (SNode*)pRoot)) {
goto error;
}
pRoot->pParent = pNode;
return pNode;
error:
nodesDestroyNode((SNode*)pNode);
return pRoot;
}
static SLogicNode* createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable) {
SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
CHECK_ALLOC(pScan, NULL); if (NULL == pScan) {
pScan->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*); TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*);
TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*);
pScan->scanType = pCxt->pPlanCxt->streamQuery ? SCAN_TYPE_STREAM : SCAN_TYPE_TABLE;
// set columns to scan
SNodeList* pCols = NULL;
CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan);
if (NULL != pCols) {
pScan->pScanCols = nodesCloneList(pCols);
CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan);
}
// set output
if (NULL != pCols) {
pScan->node.pTargets = nodesCloneList(pCols);
CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan);
}
pScan->scanType = SCAN_TYPE_TABLE;
pScan->scanFlag = MAIN_SCAN; pScan->scanFlag = MAIN_SCAN;
pScan->scanRange = TSWINDOW_INITIALIZER; pScan->scanRange = TSWINDOW_INITIALIZER;
pScan->tableName.type = TSDB_TABLE_NAME_T; pScan->tableName.type = TSDB_TABLE_NAME_T;
pScan->tableName.acctId = pCxt->acctId; pScan->tableName.acctId = pCxt->pPlanCxt->acctId;
strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.dbname, pRealTable->table.dbName);
strcpy(pScan->tableName.tname, pRealTable->table.tableName); strcpy(pScan->tableName.tname, pRealTable->table.tableName);
return (SLogicNode*)pScan; // set columns to scan
SNodeList* pCols = NULL;
int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols);
if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
pScan->pScanCols = nodesCloneList(pCols);
if (NULL == pScan) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
static SLogicNode* createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable) { // set output
SLogicNode* pRoot = createQueryLogicNode(pCxt, pTable->pSubquery); if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
CHECK_ALLOC(pRoot, NULL); pScan->node.pTargets = nodesCloneList(pCols);
if (NULL == pScan) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pScan;
} else {
nodesDestroyNode(pScan);
}
return code;
}
static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable, SLogicNode** pLogicNode) {
int32_t code = createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode);
if (TSDB_CODE_SUCCESS == code) {
SNode* pNode; SNode* pNode;
FOREACH(pNode, pRoot->pTargets) { FOREACH(pNode, (*pLogicNode)->pTargets) {
strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias); strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias);
} }
return pRoot; }
return code;
} }
static SLogicNode* createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable) { static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable, SLogicNode** pLogicNode) {
SJoinLogicNode* pJoin = (SJoinLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_JOIN); SJoinLogicNode* pJoin = (SJoinLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_JOIN);
CHECK_ALLOC(pJoin, NULL); if (NULL == pJoin) {
pJoin->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
pJoin->joinType = pJoinTable->joinType; pJoin->joinType = pJoinTable->joinType;
int32_t code = TSDB_CODE_SUCCESS;
// set left and right node // set left and right node
pJoin->node.pChildren = nodesMakeList(); pJoin->node.pChildren = nodesMakeList();
CHECK_ALLOC(pJoin->node.pChildren, (SLogicNode*)pJoin); if (NULL == pJoin->node.pChildren) {
SLogicNode* pLeft = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft); code = TSDB_CODE_OUT_OF_MEMORY;
CHECK_ALLOC(pLeft, (SLogicNode*)pJoin); }
CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pLeft), (SLogicNode*)pJoin);
SLogicNode* pRight = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight); SLogicNode* pLeft = NULL;
CHECK_ALLOC(pRight, (SLogicNode*)pJoin); if (TSDB_CODE_SUCCESS == code) {
CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pRight), (SLogicNode*)pJoin); code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft, &pLeft);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pLeft);
}
}
SLogicNode* pRight = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight, &pRight);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pRight);
}
}
// set on conditions // set on conditions
if (NULL != pJoinTable->pOnCond) { if (TSDB_CODE_SUCCESS == code && NULL != pJoinTable->pOnCond) {
pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond); pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond);
CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin); if (NULL == pJoin->pOnConditions) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
// set the output // set the output
if (TSDB_CODE_SUCCESS == code) {
pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
CHECK_ALLOC(pJoin->node.pTargets, (SLogicNode*)pJoin); if (NULL == pJoin->pOnConditions) {
SNodeList* pTargets = nodesCloneList(pRight->pTargets); code = TSDB_CODE_OUT_OF_MEMORY;
CHECK_ALLOC(pTargets, (SLogicNode*)pJoin); }
nodesListAppendList(pJoin->node.pTargets, pTargets); if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppendList(pJoin->node.pTargets, nodesCloneList(pRight->pTargets));
return (SLogicNode*)pJoin; }
} }
static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable) { if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pJoin;
} else {
nodesDestroyNode(pJoin);
}
return code;
}
static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) {
switch (nodeType(pTable)) { switch (nodeType(pTable)) {
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable); return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable, pLogicNode);
case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_TEMP_TABLE:
return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable); return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable, pLogicNode);
case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_JOIN_TABLE:
return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable); return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable, pLogicNode);
default: default:
break; break;
} }
return TSDB_CODE_FAILED;
}
static int32_t createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) {
SLogicNode* pNode = NULL;
int32_t code = doCreateLogicNodeByTable(pCxt, pSelect, pTable, &pNode);
if (TSDB_CODE_SUCCESS == code) {
pNode->pConditions = nodesCloneNode(pSelect->pWhere);
if (NULL != pSelect->pWhere && NULL == pNode->pConditions) {
nodesDestroyNode(pNode);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pLogicNode = pNode;
}
return code;
}
static SColumnNode* createColumnByExpr(SExprNode* pExpr) {
SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
return NULL; return NULL;
} }
pCol->node.resType = pExpr->resType;
strcpy(pCol->colName, pExpr->aliasName);
return pCol;
}
typedef struct SCreateColumnCxt { typedef struct SCreateColumnCxt {
int32_t errCode; int32_t errCode;
@ -245,196 +307,241 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static SNodeList* createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs) { static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) {
SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() }; SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList) };
CHECK_ALLOC(cxt.pList, NULL); if (NULL == cxt.pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
nodesWalkList(pExprs, doCreateColumn, &cxt); nodesWalkList(pExprs, doCreateColumn, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) { if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(cxt.pList); nodesDestroyList(cxt.pList);
return NULL; return cxt.errCode;
} }
return cxt.pList; if (NULL == *pList) {
*pList = cxt.pList;
}
return cxt.errCode;
} }
static SLogicNode* createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SNodeList* pAggFuncs = NULL; SNodeList* pAggFuncs = NULL;
CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs), NULL); int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (NULL == pAggFuncs && NULL == pSelect->pGroupByList) { if (NULL == pAggFuncs && NULL == pSelect->pGroupByList) {
return NULL; return TSDB_CODE_SUCCESS;
} }
SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG);
CHECK_ALLOC(pAgg, NULL); if (NULL == pAgg) {
pAgg->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
// set grouyp keys, agg funcs and having conditions // set grouyp keys, agg funcs and having conditions
if (NULL != pSelect->pGroupByList) { if (NULL != pSelect->pGroupByList) {
pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList);
CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg); if (NULL == pAgg->pGroupKeys) {
code = TSDB_CODE_OUT_OF_MEMORY;
} }
if (NULL != pAggFuncs) { }
if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncs) {
pAgg->pAggFuncs = nodesCloneList(pAggFuncs); pAgg->pAggFuncs = nodesCloneList(pAggFuncs);
CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg); if (NULL == pAgg->pAggFuncs) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
// rewrite the expression in subsequent clauses // rewrite the expression in subsequent clauses
CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); if (TSDB_CODE_SUCCESS == code) {
CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (NULL != pSelect->pHaving) { if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) {
pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving);
CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg); if (NULL == pAgg->node.pConditions) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
// set the output // set the output
pAgg->node.pTargets = nodesMakeList(); if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pGroupKeys) {
CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets);
if (NULL != pAgg->pGroupKeys) {
SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys);
CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg);
nodesListAppendList(pAgg->node.pTargets, pTargets);
} }
if (NULL != pAgg->pAggFuncs) { if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) {
SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs); code = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs, &pAgg->node.pTargets);
CHECK_ALLOC(pTargets, (SLogicNode*)pAgg);
nodesListAppendList(pAgg->node.pTargets, pTargets);
} }
return (SLogicNode*)pAgg; if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pAgg;
} else {
nodesDestroyNode(pAgg);
} }
static SLogicNode* createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect) { return code;
}
static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW);
CHECK_ALLOC(pWindow, NULL); if (NULL == pWindow) {
pWindow->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
pWindow->winType = WINDOW_TYPE_INTERVAL; pWindow->winType = WINDOW_TYPE_INTERVAL;
SValueNode* pIntervalNode = (SValueNode*)((SRawExprNode*)(pInterval->pInterval))->pNode; pWindow->interval = ((SValueNode*)pInterval->pInterval)->datum.i;
pWindow->intervalUnit = ((SValueNode*)pInterval->pInterval)->unit;
pWindow->interval = pIntervalNode->datum.i;
pWindow->offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0); pWindow->offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0);
pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval);
pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit);
int32_t code = TSDB_CODE_SUCCESS;
if (NULL != pInterval->pFill) { if (NULL != pInterval->pFill) {
pWindow->pFill = nodesCloneNode(pInterval->pFill); pWindow->pFill = nodesCloneNode(pInterval->pFill);
CHECK_ALLOC(pWindow->pFill, (SLogicNode*)pWindow); if (NULL == pWindow->pFill) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
SNodeList* pFuncs = NULL; if (TSDB_CODE_SUCCESS == code) {
CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pFuncs), NULL); code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs);
if (NULL != pFuncs) {
pWindow->pFuncs = nodesCloneList(pFuncs);
CHECK_ALLOC(pWindow->pFuncs, (SLogicNode*)pWindow);
} }
CHECK_CODE(rewriteExpr(pWindow->node.id, 1, pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW), (SLogicNode*)pWindow); if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
pWindow->node.pTargets = createColumnByRewriteExps(pCxt, pWindow->pFuncs);
CHECK_ALLOC(pWindow->node.pTargets, (SLogicNode*)pWindow);
return (SLogicNode*)pWindow;
} }
static SLogicNode* createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pWindow;
} else {
nodesDestroyNode(pWindow);
}
return code;
}
static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (NULL == pSelect->pWindow) { if (NULL == pSelect->pWindow) {
return NULL; return TSDB_CODE_SUCCESS;
} }
switch (nodeType(pSelect->pWindow)) { switch (nodeType(pSelect->pWindow)) {
case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_INTERVAL_WINDOW:
return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect); return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode);
default: default:
break; break;
} }
return NULL; return TSDB_CODE_FAILED;
} }
static SNodeList* createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs) { static int32_t createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pCols) {
SNodeList* pList = nodesMakeList(); SNodeList* pList = nodesMakeList();
CHECK_ALLOC(pList, NULL); if (NULL == pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pNode; SNode* pNode;
FOREACH(pNode, pExprs) { FOREACH(pNode, pExprs) {
SExprNode* pExpr = (SExprNode*)pNode; if (TSDB_CODE_SUCCESS != nodesListAppend(pList, createColumnByExpr((SExprNode*)pNode))) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
goto error;
}
pCol->node.resType = pExpr->resType;
strcpy(pCol->colName, pExpr->aliasName);
if (TSDB_CODE_SUCCESS != nodesListAppend(pList, (SNode*)pCol)) {
goto error;
}
}
return pList;
error:
nodesDestroyList(pList); nodesDestroyList(pList);
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
}
} }
static SLogicNode* createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { *pCols = pList;
return TSDB_CODE_SUCCESS;
}
static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT); SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT);
CHECK_ALLOC(pProject, NULL); if (NULL == pProject) {
pProject->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = TSDB_CODE_SUCCESS;
pProject->pProjections = nodesCloneList(pSelect->pProjectionList); pProject->pProjections = nodesCloneList(pSelect->pProjectionList);
if (NULL == pProject->pProjections) {
pProject->node.pTargets = createColumnByProjections(pCxt,pSelect->pProjectionList); code = TSDB_CODE_OUT_OF_MEMORY;
CHECK_ALLOC(pProject->node.pTargets, (SLogicNode*)pProject);
return (SLogicNode*)pProject;
} }
static SLogicNode* createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) {
SLogicNode* pRoot = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable); code = createColumnByProjections(pCxt,pSelect->pProjectionList, &pProject->node.pTargets);
if (TSDB_CODE_SUCCESS == pCxt->errCode && NULL != pSelect->pWhere) {
pRoot->pConditions = nodesCloneNode(pSelect->pWhere);
CHECK_ALLOC(pRoot->pConditions, pRoot);
} }
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pRoot = pushLogicNode(pCxt, pRoot, createWindowLogicNode(pCxt, pSelect)); if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pProject;
} else {
nodesDestroyNode(pProject);
} }
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pRoot = pushLogicNode(pCxt, pRoot, createAggLogicNode(pCxt, pSelect)); return code;
} }
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pRoot = pushLogicNode(pCxt, pRoot, createProjectLogicNode(pCxt, pSelect)); static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SLogicNode* pRoot = NULL;
int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot);
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot);
} }
return pRoot; if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createProjectLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = pRoot;
} else {
nodesDestroyNode(pRoot);
}
return code;
} }
static int32_t getMsgType(ENodeType sqlType) { static int32_t getMsgType(ENodeType sqlType) {
return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) ? TDMT_VND_CREATE_TABLE : TDMT_VND_SUBMIT; return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) ? TDMT_VND_CREATE_TABLE : TDMT_VND_SUBMIT;
} }
static SLogicNode* createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) {
SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF); SVnodeModifLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF);
CHECK_ALLOC(pModif, NULL); if (NULL == pModif) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pModif->pDataBlocks = pStmt->pDataBlocks; pModif->pDataBlocks = pStmt->pDataBlocks;
pModif->msgType = getMsgType(pStmt->sqlNodeType); pModif->msgType = getMsgType(pStmt->sqlNodeType);
return (SLogicNode*)pModif; *pLogicNode = (SLogicNode*)pModif;
return TSDB_CODE_SUCCESS;
} }
static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt) { static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode) {
switch (nodeType(pStmt)) { switch (nodeType(pStmt)) {
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt); return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode);
case QUERY_NODE_VNODE_MODIF_STMT: case QUERY_NODE_VNODE_MODIF_STMT:
return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt); return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode);
default: default:
break; break;
} }
return NULL; // to avoid compiler error return TSDB_CODE_FAILED;
} }
int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) { int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) {
SLogicPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 1, .acctId = pCxt->acctId }; SLogicPlanContext cxt = { .pPlanCxt = pCxt };
SLogicNode* pRoot = createQueryLogicNode(&cxt, pCxt->pAstRoot); int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, pLogicNode);
if (TSDB_CODE_SUCCESS != cxt.errCode) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pRoot); return code;
return cxt.errCode;
} }
*pLogicNode = pRoot;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

10
source/libs/planner/src/planPhysiCreater.c
View File

@ -259,15 +259,21 @@ static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* p
return (SPhysiNode*)pTableScan; return (SPhysiNode*)pTableScan;
} }
static SPhysiNode* createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) {
SStreamScanPhysiNode* pTableScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
CHECK_ALLOC(pTableScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan);
return (SPhysiNode*)pTableScan;
}
static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) {
switch (pScanLogicNode->scanType) { switch (pScanLogicNode->scanType) {
case SCAN_TYPE_TAG: case SCAN_TYPE_TAG:
return createTagScanPhysiNode(pCxt, pScanLogicNode); return createTagScanPhysiNode(pCxt, pScanLogicNode);
case SCAN_TYPE_TABLE: case SCAN_TYPE_TABLE:
return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode); return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode);
case SCAN_TYPE_STABLE:
case SCAN_TYPE_STREAM: case SCAN_TYPE_STREAM:
break; return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode);
default: default:
break; break;
} }

2
source/libs/planner/src/planSpliter.c
View File

@ -89,7 +89,7 @@ static SSubLogicPlan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode*
pSubplan->id.groupId = pCxt->groupId; pSubplan->id.groupId = pCxt->groupId;
pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan); pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan);
TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo); TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo*);
SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS); SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS);
return pSubplan; return pSubplan;
} }

8
source/libs/scalar/src/filter.c
View File

@ -1813,7 +1813,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
if(type == TSDB_DATA_TYPE_NCHAR && if(type == TSDB_DATA_TYPE_NCHAR &&
(unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)){ (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)){
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
if (len < 0){ if (len < 0){
qError("filterInitValFieldData taosUcs4ToMbs error 1"); qError("filterInitValFieldData taosUcs4ToMbs error 1");
return TSDB_CODE_QRY_APP_ERROR; return TSDB_CODE_QRY_APP_ERROR;
@ -2992,7 +2992,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){ if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){
char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData));
if (len < 0){ if (len < 0){
qError("castConvert1 taosUcs4ToMbs error"); qError("castConvert1 taosUcs4ToMbs error");
}else{ }else{
@ -3052,7 +3052,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
} else { } else {
if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){ if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){
char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData));
if (len < 0){ if (len < 0){
qError("castConvert1 taosUcs4ToMbs error"); qError("castConvert1 taosUcs4ToMbs error");
}else{ }else{
@ -3433,7 +3433,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
varDataCopy(dst, src); varDataCopy(dst, src);
continue; continue;
} }
bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4*)varDataVal(dst), bufSize, &len);
if(!ret) { if(!ret) {
qError("filterConverNcharColumns taosMbsToUcs4 error"); qError("filterConverNcharColumns taosMbsToUcs4 error");
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;

2
source/libs/scalar/src/sclvector.c
View File

@ -316,7 +316,7 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t
tmp = realloc(tmp, bufSize); tmp = realloc(tmp, bufSize);
} }
int len = taosUcs4ToMbs(varDataVal(pIn->data), varDataLen(pIn->data), tmp); int len = taosUcs4ToMbs((TdUcs4*)varDataVal(pIn->data), varDataLen(pIn->data), tmp);
if (len < 0){ if (len < 0){
sclError("castConvert taosUcs4ToMbs error 1"); sclError("castConvert taosUcs4ToMbs error 1");
tfree(tmp); tfree(tmp);

60
source/libs/sync/inc/syncCommit.h Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_COMMIT_H
#define _TD_LIBS_SYNC_COMMIT_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "syncInt.h"
#include "taosdef.h"
// \* Leader i advances its commitIndex.
// \* This is done as a separate step from handling AppendEntries responses,
// \* in part to minimize atomic regions, and in part so that leaders of
// \* single-server clusters are able to mark entries committed.
// AdvanceCommitIndex(i) ==
// /\ state[i] = Leader
// /\ LET \* The set of servers that agree up through index.
// Agree(index) == {i} \cup {k \in Server :
// matchIndex[i][k] >= index}
// \* The maximum indexes for which a quorum agrees
// agreeIndexes == {index \in 1..Len(log[i]) :
// Agree(index) \in Quorum}
// \* New value for commitIndex'[i]
// newCommitIndex ==
// IF /\ agreeIndexes /= {}
// /\ log[i][Max(agreeIndexes)].term = currentTerm[i]
// THEN
// Max(agreeIndexes)
// ELSE
// commitIndex[i]
// IN commitIndex' = [commitIndex EXCEPT ![i] = newCommitIndex]
// /\ UNCHANGED <<messages, serverVars, candidateVars, leaderVars, log>>
//
void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode);
bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index);
bool syncAgree(SSyncNode* pSyncNode, SyncIndex index);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_COMMIT_H*/

6
source/libs/sync/inc/syncEnv.h
View File

@ -31,10 +31,10 @@ extern "C" {
#define TIMER_MAX_MS 0x7FFFFFFF #define TIMER_MAX_MS 0x7FFFFFFF
#define ENV_TICK_TIMER_MS 1000 #define ENV_TICK_TIMER_MS 1000
#define PING_TIMER_MS 1000 #define PING_TIMER_MS 1000
#define ELECT_TIMER_MS_MIN 150 #define ELECT_TIMER_MS_MIN 1500
#define ELECT_TIMER_MS_MAX 300 #define ELECT_TIMER_MS_MAX 3000
#define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN)
#define HEARTBEAT_TIMER_MS 30 #define HEARTBEAT_TIMER_MS 300
#define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0})

1
source/libs/sync/inc/syncInt.h
View File

@ -236,7 +236,6 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode);
// raft vote -------------- // raft vote --------------
void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId);
void syncNodeVoteForSelf(SSyncNode* pSyncNode); void syncNodeVoteForSelf(SSyncNode* pSyncNode);
void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode);
// for debug -------------- // for debug --------------
void syncNodePrint(SSyncNode* pObj); void syncNodePrint(SSyncNode* pObj);

6
source/libs/sync/inc/syncMessage.h
View File

@ -163,15 +163,15 @@ typedef struct SyncClientRequest {
} SyncClientRequest; } SyncClientRequest;
SyncClientRequest* syncClientRequestBuild(uint32_t dataLen); SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak); SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak); // step 1
void syncClientRequestDestroy(SyncClientRequest* pMsg); void syncClientRequestDestroy(SyncClientRequest* pMsg);
void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen); void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg); void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len); char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len); SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2
void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3
cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg);
char* syncClientRequest2Str(const SyncClientRequest* pMsg); char* syncClientRequest2Str(const SyncClientRequest* pMsg);

40
source/libs/sync/inc/syncOnMessage.h
View File

@ -25,6 +25,46 @@ extern "C" {
#include <stdlib.h> #include <stdlib.h>
#include "taosdef.h" #include "taosdef.h"
// TLA+ Spec
// Receive(m) ==
// LET i == m.mdest
// j == m.msource
// IN \* Any RPC with a newer term causes the recipient to advance
// \* its term first. Responses with stale terms are ignored.
// \/ UpdateTerm(i, j, m)
// \/ /\ m.mtype = RequestVoteRequest
// /\ HandleRequestVoteRequest(i, j, m)
// \/ /\ m.mtype = RequestVoteResponse
// /\ \/ DropStaleResponse(i, j, m)
// \/ HandleRequestVoteResponse(i, j, m)
// \/ /\ m.mtype = AppendEntriesRequest
// /\ HandleAppendEntriesRequest(i, j, m)
// \/ /\ m.mtype = AppendEntriesResponse
// /\ \/ DropStaleResponse(i, j, m)
// \/ HandleAppendEntriesResponse(i, j, m)
// DuplicateMessage(m) ==
// /\ Send(m)
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
// DropMessage(m) ==
// /\ Discard(m)
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
// Next == /\ \/ \E i \in Server : Restart(i)
// \/ \E i \in Server : Timeout(i)
// \/ \E i,j \in Server : RequestVote(i, j)
// \/ \E i \in Server : BecomeLeader(i)
// \/ \E i \in Server, v \in Value : ClientRequest(i, v)
// \/ \E i \in Server : AdvanceCommitIndex(i)
// \/ \E i,j \in Server : AppendEntries(i, j)
// \/ \E m \in DOMAIN messages : Receive(m)
// \/ \E m \in DOMAIN messages : DuplicateMessage(m)
// \/ \E m \in DOMAIN messages : DropMessage(m)
// \* History variable that tracks every log ever:
// /\ allLogs' = allLogs \cup {log[i] : i \in Server}
//
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

7
source/libs/sync/inc/syncRaftEntry.h
View File

@ -40,12 +40,13 @@ typedef struct SSyncRaftEntry {
} SSyncRaftEntry; } SSyncRaftEntry;
SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); SSyncRaftEntry* syncEntryBuild(uint32_t dataLen);
SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4
void syncEntryDestory(SSyncRaftEntry* pEntry); void syncEntryDestory(SSyncRaftEntry* pEntry);
char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5
SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6
cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry);
char* syncEntry2Str(const SSyncRaftEntry* pEntry); char* syncEntry2Str(const SSyncRaftEntry* pEntry);
void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7
// for debug ---------------------- // for debug ----------------------
void syncEntryPrint(const SSyncRaftEntry* pObj); void syncEntryPrint(const SSyncRaftEntry* pObj);

3
source/libs/sync/inc/syncRaftStore.h
View File

@ -48,6 +48,9 @@ void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId);
void raftStoreClearVote(SRaftStore *pRaftStore); void raftStoreClearVote(SRaftStore *pRaftStore);
void raftStoreNextTerm(SRaftStore *pRaftStore); void raftStoreNextTerm(SRaftStore *pRaftStore);
void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term);
int32_t raftStoreFromJson(SRaftStore *pRaftStore, cJSON *pJson);
cJSON * raftStore2Json(SRaftStore *pRaftStore);
char * raftStore2Str(SRaftStore *pRaftStore);
// for debug ------------------- // for debug -------------------
void raftStorePrint(SRaftStore *pObj); void raftStorePrint(SRaftStore *pObj);

165
source/libs/sync/src/syncAppendEntries.c
View File

@ -87,13 +87,17 @@
// //
int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
int32_t ret = 0; int32_t ret = 0;
syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesCb== term:%lu", ths->pRaftStore->currentTerm);
syncAppendEntriesLog2(logBuf, pMsg);
if (pMsg->term > ths->pRaftStore->currentTerm) { if (pMsg->term > ths->pRaftStore->currentTerm) {
syncNodeUpdateTerm(ths, pMsg->term); syncNodeUpdateTerm(ths, pMsg->term);
} }
assert(pMsg->term <= ths->pRaftStore->currentTerm); assert(pMsg->term <= ths->pRaftStore->currentTerm);
// reset elect timer
if (pMsg->term == ths->pRaftStore->currentTerm) { if (pMsg->term == ths->pRaftStore->currentTerm) {
ths->leaderCache = pMsg->srcId; ths->leaderCache = pMsg->srcId;
syncNodeResetElectTimer(ths); syncNodeResetElectTimer(ths);
@ -101,8 +105,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
assert(pMsg->dataLen >= 0); assert(pMsg->dataLen >= 0);
SyncTerm localPreLogTerm = 0; SyncTerm localPreLogTerm = 0;
if (pMsg->prevLogTerm >= SYNC_INDEX_BEGIN && pMsg->prevLogTerm <= ths->pLogStore->getLastIndex(ths->pLogStore)) { if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) {
SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm); SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogIndex);
assert(pEntry != NULL); assert(pEntry != NULL);
localPreLogTerm = pEntry->term; localPreLogTerm = pEntry->term;
syncEntryDestory(pEntry); syncEntryDestory(pEntry);
@ -111,9 +115,9 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
bool logOK = bool logOK =
(pMsg->prevLogIndex == SYNC_INDEX_INVALID) || (pMsg->prevLogIndex == SYNC_INDEX_INVALID) ||
((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) &&
(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogIndex == localPreLogTerm)); (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogTerm == localPreLogTerm));
// reject // reject request
if ((pMsg->term < ths->pRaftStore->currentTerm) || if ((pMsg->term < ths->pRaftStore->currentTerm) ||
((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) {
SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
@ -134,36 +138,118 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
// return to follower state // return to follower state
if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) {
syncNodeBecomeFollower(ths); syncNodeBecomeFollower(ths);
// ret or reply?
return ret;
} }
// accept request // accept request
if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) {
bool matchSuccess = false; bool preMatch = false;
if (pMsg->prevLogIndex == SYNC_INDEX_INVALID && if (pMsg->prevLogIndex == SYNC_INDEX_INVALID &&
ths->pLogStore->getLastIndex(ths->pLogStore) == SYNC_INDEX_INVALID) { ths->pLogStore->getLastIndex(ths->pLogStore) == SYNC_INDEX_INVALID) {
matchSuccess = true; preMatch = true;
} }
if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) {
SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm); SSyncRaftEntry* pPreEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogIndex);
assert(pEntry != NULL); assert(pPreEntry != NULL);
if (pMsg->prevLogTerm == pEntry->term) { if (pMsg->prevLogTerm == pPreEntry->term) {
matchSuccess = true; preMatch = true;
} }
syncEntryDestory(pEntry); syncEntryDestory(pPreEntry);
} }
if (matchSuccess) { if (preMatch) {
// delete conflict entries // must has preIndex in local log
if (ths->pLogStore->getLastIndex(ths->pLogStore) > pMsg->prevLogIndex) { assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore));
SyncIndex fromIndex = pMsg->prevLogIndex + 1;
ths->pLogStore->truncate(ths->pLogStore, fromIndex); bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore);
bool hasAppendEntries = pMsg->dataLen > 0;
if (hasExtraEntries && hasAppendEntries) {
// conflict
bool conflict = false;
SyncIndex extraIndex = pMsg->prevLogIndex + 1;
SSyncRaftEntry* pExtraEntry = logStoreGetEntry(ths->pLogStore, extraIndex);
assert(pExtraEntry != NULL);
SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen);
assert(pAppendEntry != NULL);
assert(extraIndex == pAppendEntry->index);
if (pExtraEntry->term == pAppendEntry->term) {
conflict = true;
} }
// append one entry if (conflict) {
if (pMsg->dataLen > 0) { // roll back
SSyncRaftEntry* pEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore);
ths->pLogStore->appendEntry(ths->pLogStore, pEntry); SyncIndex delEnd = extraIndex;
syncEntryDestory(pEntry);
// notice! reverse roll back!
for (SyncIndex index = delEnd; index >= delBegin; --index) {
if (ths->pFsm->FpRollBackCb != NULL) {
SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index);
assert(pRollBackEntry != NULL);
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0);
rpcFreeCont(rpcMsg.pCont);
syncEntryDestory(pRollBackEntry);
}
}
// delete confict entries
ths->pLogStore->truncate(ths->pLogStore, extraIndex);
// append new entries
ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry);
// pre commit
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
if (ths->pFsm != NULL) {
if (ths->pFsm->FpPreCommitCb != NULL) {
ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0);
}
}
rpcFreeCont(rpcMsg.pCont);
}
// free memory
syncEntryDestory(pExtraEntry);
syncEntryDestory(pAppendEntry);
} else if (hasExtraEntries && !hasAppendEntries) {
// do nothing
} else if (!hasExtraEntries && hasAppendEntries) {
SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen);
assert(pAppendEntry != NULL);
// append new entries
ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry);
// pre commit
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
if (ths->pFsm != NULL) {
if (ths->pFsm->FpPreCommitCb != NULL) {
ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0);
}
}
rpcFreeCont(rpcMsg.pCont);
// free memory
syncEntryDestory(pAppendEntry);
} else if (!hasExtraEntries && !hasAppendEntries) {
// do nothing
} else {
assert(0);
} }
SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
@ -171,13 +257,19 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
pReply->destId = pMsg->srcId; pReply->destId = pMsg->srcId;
pReply->term = ths->pRaftStore->currentTerm; pReply->term = ths->pRaftStore->currentTerm;
pReply->success = true; pReply->success = true;
if (hasAppendEntries) {
pReply->matchIndex = pMsg->prevLogIndex + 1; pReply->matchIndex = pMsg->prevLogIndex + 1;
} else {
pReply->matchIndex = pMsg->prevLogIndex;
}
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
syncAppendEntriesReplyDestroy(pReply); syncAppendEntriesReplyDestroy(pReply);
} else { } else {
SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
pReply->srcId = ths->myRaftId; pReply->srcId = ths->myRaftId;
@ -192,11 +284,38 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
syncAppendEntriesReplyDestroy(pReply); syncAppendEntriesReplyDestroy(pReply);
} }
// maybe update commit index from leader
if (pMsg->commitIndex > ths->commitIndex) { if (pMsg->commitIndex > ths->commitIndex) {
// has commit entry in local
if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) {
// commit SyncIndex beginIndex = ths->commitIndex + 1;
SyncIndex endIndex = pMsg->commitIndex;
// update commit index
ths->commitIndex = pMsg->commitIndex; ths->commitIndex = pMsg->commitIndex;
// call back Wal
ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex);
// execute fsm
if (ths->pFsm != NULL) {
for (SyncIndex i = beginIndex; i <= endIndex; ++i) {
if (i != SYNC_INDEX_INVALID) {
SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, i);
assert(pEntry != NULL);
SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pEntry, &rpcMsg);
if (ths->pFsm->FpCommitCb != NULL) {
ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0);
}
rpcFreeCont(rpcMsg.pCont);
syncEntryDestory(pEntry);
}
}
}
} }
} }
} }

17
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -14,6 +14,7 @@
*/ */
#include "syncAppendEntriesReply.h" #include "syncAppendEntriesReply.h"
#include "syncCommit.h"
#include "syncIndexMgr.h" #include "syncIndexMgr.h"
#include "syncInt.h" #include "syncInt.h"
#include "syncRaftLog.h" #include "syncRaftLog.h"
@ -36,10 +37,14 @@
// //
int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {
int32_t ret = 0; int32_t ret = 0;
syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplyCb== term:%lu", ths->pRaftStore->currentTerm);
syncAppendEntriesReplyLog2(logBuf, pMsg);
if (pMsg->term < ths->pRaftStore->currentTerm) { if (pMsg->term < ths->pRaftStore->currentTerm) {
sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, ths->pRaftStore->currentTerm); sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term,
ths->pRaftStore->currentTerm);
return ret; return ret;
} }
@ -51,17 +56,19 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p
assert(pMsg->term == ths->pRaftStore->currentTerm); assert(pMsg->term == ths->pRaftStore->currentTerm);
if (pMsg->success) { if (pMsg->success) {
// nextIndex = reply.matchIndex + 1 // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1]
syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1);
// matchIndex = reply.matchIndex // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex]
syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex);
// maybe commit // maybe commit
syncNodeMaybeAdvanceCommitIndex(ths); syncMaybeAdvanceCommitIndex(ths);
} else { } else {
SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId));
// notice! int64, uint64
if (nextIndex > SYNC_INDEX_BEGIN) { if (nextIndex > SYNC_INDEX_BEGIN) {
--nextIndex; --nextIndex;
} else { } else {

Some files were not shown because too many files have changed in this diff Show More