diff --git a/.gitignore b/.gitignore index 3044348648..fd693adc3b 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ taosdalipu/ Target/ *.failed *.sql +sim sim/ psim/ pysim/ @@ -107,3 +108,4 @@ TAGS contrib/* !contrib/CMakeLists.txt !contrib/test +sql \ No newline at end of file diff --git a/example/src/tmq.c b/example/src/tmq.c index 21f60ada5d..5b8f66f666 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -18,7 +18,6 @@ #include #include #include "taos.h" -#include "osSleep.h" static int running = 1; static void msg_process(TAOS_RES* msg) { diff --git a/include/client/taos.h b/include/client/taos.h index 6ff55aef19..0f1633ed6f 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -93,14 +93,14 @@ typedef struct taosField { typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); -typedef struct TAOS_BIND_v2 { - int buffer_type; - void *buffer; - int32_t buffer_length; - int32_t *length; - char *is_null; - int num; -} TAOS_BIND_v2; +typedef struct TAOS_MULTI_BIND { + int buffer_type; + void *buffer; + uintptr_t buffer_length; + int32_t *length; + char *is_null; + int num; +} TAOS_MULTI_BIND; typedef enum { SET_CONF_RET_SUCC = 0, @@ -129,35 +129,35 @@ DLL_EXPORT void taos_close(TAOS *taos); const char *taos_data_type(int type); -DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); -DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags); -DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); +DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags); +DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); -DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx); -DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); +DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); +DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); +DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); -DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result -DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT void taos_free_result(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 7c308f9354..fa428a994a 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -66,13 +66,13 @@ typedef struct SDataBlockInfo { int32_t rows; int32_t rowSize; union { - int64_t uid; // from which table of uid, comes from this data block + int64_t uid; // from which table of uid, comes from this data block int64_t blockId; }; - uint64_t groupId; // no need to serialize - int16_t numOfCols; - int16_t hasVarCol; - int16_t capacity; + uint64_t groupId; // no need to serialize + int16_t numOfCols; + int16_t hasVarCol; + int16_t capacity; } SDataBlockInfo; typedef struct SSDataBlock { @@ -99,6 +99,15 @@ typedef struct SColumnInfoData { }; } SColumnInfoData; +typedef struct SQueryTableDataCond { + STimeWindow twindow; + int32_t order; // desc|asc order to iterate the data block + int32_t numOfCols; + SColumnInfo *colList; + bool loadExternalRows; // load external rows or not + int32_t type; // data block load type: +} SQueryTableDataCond; + void* blockDataDestroy(SSDataBlock* pBlock); int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); @@ -122,59 +131,6 @@ static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); } -static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { - int32_t tlen = 0; - int32_t sz = 0; - // tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); - tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); - tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); - tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics); - if (pRsp->numOfTopics == 0) return tlen; - tlen += taosEncodeSSchemaWrapper(buf, pRsp->schema); - if (pRsp->pBlockData) { - sz = taosArrayGetSize(pRsp->pBlockData); - } - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); - tlen += tEncodeDataBlock(buf, pBlock); - } - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { - int32_t sz; - // buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); - buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); - buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); - buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics); - if (pRsp->numOfTopics == 0) return buf; - pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - if (pRsp->schema == NULL) return NULL; - buf = taosDecodeSSchemaWrapper(buf, pRsp->schema); - buf = taosDecodeFixedI32(buf, &sz); - pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock block = {0}; - tDecodeDataBlock(buf, &block); - taosArrayPush(pRsp->pBlockData, &block); - } - return buf; -} - -static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { - if (pRsp->schema) { - if (pRsp->schema->nCols) { - taosMemoryFreeClear(pRsp->schema->pSchema); - } - taosMemoryFree(pRsp->schema); - } - taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner); - pRsp->pBlockData = NULL; -} - //====================================================================================================================== // the following structure shared by parser and executor typedef struct SColumn { @@ -195,22 +151,22 @@ typedef struct SColumn { } SColumn; typedef struct STableBlockDistInfo { - uint16_t rowSize; - uint16_t numOfFiles; - uint32_t numOfTables; - uint64_t totalSize; - uint64_t totalRows; - int32_t maxRows; - int32_t minRows; - int32_t firstSeekTimeUs; - uint32_t numOfRowsInMemTable; - uint32_t numOfSmallBlocks; - SArray *dataBlockInfos; + uint16_t rowSize; + uint16_t numOfFiles; + uint32_t numOfTables; + uint64_t totalSize; + uint64_t totalRows; + int32_t maxRows; + int32_t minRows; + int32_t firstSeekTimeUs; + uint32_t numOfRowsInMemTable; + uint32_t numOfSmallBlocks; + SArray* dataBlockInfos; } STableBlockDistInfo; enum { FUNC_PARAM_TYPE_VALUE = 0x1, - FUNC_PARAM_TYPE_COLUMN= 0x2, + FUNC_PARAM_TYPE_COLUMN = 0x2, }; typedef struct SFunctParam { @@ -229,7 +185,6 @@ typedef struct SResSchame { char name[TSDB_COL_NAME_LEN]; } SResSchema; -// TODO move away to executor.h typedef struct SExprBasicInfo { SResSchema resSchema; int16_t numOfParams; // argument value of each function @@ -241,7 +196,7 @@ typedef struct SExprInfo { struct tExprNode* pExpr; } SExprInfo; -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 26cbd35e90..7e543fb770 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -177,6 +177,7 @@ typedef struct SField { char name[TSDB_COL_NAME_LEN]; uint8_t type; int32_t bytes; + int8_t flags; } SField; typedef struct SRetention { @@ -286,6 +287,12 @@ typedef struct SSchema { #define SSCHMEA_BYTES(s) ((s)->bytes) #define SSCHMEA_NAME(s) ((s)->name) +typedef struct { + int32_t nCols; + int32_t sver; + SSchema* pSchema; +} SSchemaWrapper; + STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols); typedef struct { @@ -296,13 +303,11 @@ typedef struct { int32_t ttl; int32_t numOfColumns; int32_t numOfTags; - int32_t numOfSmas; int32_t commentLen; int32_t ast1Len; int32_t ast2Len; SArray* pColumns; // array of SField SArray* pTags; // array of SField - SArray* pSmas; // array of SField char* comment; char* pAst1; char* pAst2; @@ -1507,67 +1512,89 @@ typedef struct { } SDDropTopicReq; typedef struct { - float xFilesFactor; - int32_t delay; - int32_t qmsg1Len; - int32_t qmsg2Len; - char* qmsg1; // pAst1:qmsg1:SRetention1 => trigger aggr task1 - char* qmsg2; // pAst2:qmsg2:SRetention2 => trigger aggr task2 + float xFilesFactor; + int32_t delay; + int32_t qmsg1Len; + int32_t qmsg2Len; + char* qmsg1; // pAst1:qmsg1:SRetention1 => trigger aggr task1 + char* qmsg2; // pAst2:qmsg2:SRetention2 => trigger aggr task2 } SRSmaParam; +int tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam); +int tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam); + +typedef struct SVCreateStbReq { + const char* name; + tb_uid_t suid; + int8_t rollup; + SSchemaWrapper schema; + SSchemaWrapper schemaTag; + SRSmaParam pRSmaParam; +} SVCreateStbReq; + +int tEncodeSVCreateStbReq(SCoder* pCoder, const SVCreateStbReq* pReq); +int tDecodeSVCreateStbReq(SCoder* pCoder, SVCreateStbReq* pReq); + +typedef struct SVDropStbReq { + // data +} SVDropStbReq; + +typedef struct SVCreateStbRsp { + int code; +} SVCreateStbRsp; + typedef struct SVCreateTbReq { - char* name; - uint32_t ttl; - uint32_t keep; - union { - uint8_t info; - struct { - uint8_t rollup : 1; // 1 means rollup sma - uint8_t type : 7; - }; - }; + tb_uid_t uid; + int64_t ctime; + const char* name; + int32_t ttl; + int8_t type; union { struct { tb_uid_t suid; - col_id_t nCols; - col_id_t nBSmaCols; - SSchema* pSchema; - col_id_t nTagCols; - SSchema* pTagSchema; - SRSmaParam* pRSmaParam; - } stbCfg; + const void* pTag; + } ctb; struct { - tb_uid_t suid; - SKVRow pTag; - } ctbCfg; - struct { - col_id_t nCols; - col_id_t nBSmaCols; - SSchema* pSchema; - SRSmaParam* pRSmaParam; - } ntbCfg; + SSchemaWrapper schema; + } ntb; }; -} SVCreateTbReq, SVUpdateTbReq; +} SVCreateTbReq; + +int tEncodeSVCreateTbReq(SCoder* pCoder, const SVCreateTbReq* pReq); +int tDecodeSVCreateTbReq(SCoder* pCoder, SVCreateTbReq* pReq); + +typedef struct { + int32_t nReqs; + union { + SVCreateTbReq* pReqs; + SArray* pArray; + }; +} SVCreateTbBatchReq; + +int tEncodeSVCreateTbBatchReq(SCoder* pCoder, const SVCreateTbBatchReq* pReq); +int tDecodeSVCreateTbBatchReq(SCoder* pCoder, SVCreateTbBatchReq* pReq); typedef struct { int32_t code; } SVCreateTbRsp, SVUpdateTbRsp; +int tEncodeSVCreateTbRsp(SCoder* pCoder, const SVCreateTbRsp* pRsp); +int tDecodeSVCreateTbRsp(SCoder* pCoder, SVCreateTbRsp* pRsp); + int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); typedef struct { - int64_t ver; // use a general definition - SArray* pArray; -} SVCreateTbBatchReq; - -int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); -void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); - -typedef struct { - SArray* rspList; // SArray + int32_t nRsps; + union { + SVCreateTbRsp* pRsps; + SArray* pArray; + }; } SVCreateTbBatchRsp; +int tEncodeSVCreateTbBatchRsp(SCoder* pCoder, const SVCreateTbBatchRsp* pRsp); +int tDecodeSVCreateTbBatchRsp(SCoder* pCoder, SVCreateTbBatchRsp* pRsp); + int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); @@ -1622,103 +1649,11 @@ typedef struct { char data[]; } SVShowTablesFetchRsp; -typedef struct SMqCMGetSubEpReq { +typedef struct { int64_t consumerId; int32_t epoch; char cgroup[TSDB_CGROUP_LEN]; -} SMqCMGetSubEpReq; - -static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pMsg->contLen); - tlen += taosEncodeFixedI32(buf, pMsg->vgId); - return tlen; -} - -typedef struct SMqHbRsp { - int8_t status; // idle or not - int8_t vnodeChanged; - int8_t epChanged; // should use new epset - int8_t reserved; - SEpSet epSet; -} SMqHbRsp; - -static FORCE_INLINE int32_t taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI8(buf, pRsp->status); - tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged); - tlen += taosEncodeFixedI8(buf, pRsp->epChanged); - tlen += taosEncodeSEpSet(buf, &pRsp->epSet); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) { - buf = taosDecodeFixedI8(buf, &pRsp->status); - buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged); - buf = taosDecodeFixedI8(buf, &pRsp->epChanged); - buf = taosDecodeSEpSet(buf, &pRsp->epSet); - return buf; -} - -typedef struct SMqHbOneTopicBatchRsp { - char topicName[TSDB_TOPIC_FNAME_LEN]; - SArray* rsps; // SArray -} SMqHbOneTopicBatchRsp; - -static FORCE_INLINE int32_t taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pBatchRsp->topicName); - int32_t sz = taosArrayGetSize(pBatchRsp->rsps); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqHbRsp* pRsp = (SMqHbRsp*)taosArrayGet(pBatchRsp->rsps, i); - tlen += taosEncodeSMqHbRsp(buf, pRsp); - } - return tlen; -} - -static FORCE_INLINE void* taosDecodeSMqHbOneTopicBatchRsp(void* buf, SMqHbOneTopicBatchRsp* pBatchRsp) { - int32_t sz; - buf = taosDecodeStringTo(buf, pBatchRsp->topicName); - buf = taosDecodeFixedI32(buf, &sz); - pBatchRsp->rsps = taosArrayInit(sz, sizeof(SMqHbRsp)); - for (int32_t i = 0; i < sz; i++) { - SMqHbRsp rsp; - buf = taosDecodeSMqHbRsp(buf, &rsp); - buf = taosArrayPush(pBatchRsp->rsps, &rsp); - } - return buf; -} - -typedef struct SMqHbBatchRsp { - int64_t consumerId; - SArray* batchRsps; // SArray -} SMqHbBatchRsp; - -static FORCE_INLINE int32_t taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId); - int32_t sz; - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*)taosArrayGet(pBatchRsp->batchRsps, i); - tlen += taosEncodeSMqHbOneTopicBatchRsp(buf, pRsp); - } - return tlen; -} - -static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBatchRsp) { - buf = taosDecodeFixedI64(buf, &pBatchRsp->consumerId); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pBatchRsp->batchRsps = taosArrayInit(sz, sizeof(SMqHbOneTopicBatchRsp)); - for (int32_t i = 0; i < sz; i++) { - SMqHbOneTopicBatchRsp rsp; - buf = taosDecodeSMqHbOneTopicBatchRsp(buf, &rsp); - buf = taosArrayPush(pBatchRsp->batchRsps, &rsp); - } - return buf; -} +} SMqAskEpReq; typedef struct { int32_t key; @@ -2087,15 +2022,11 @@ int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset); int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq); int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq); -typedef struct { - uint32_t nCols; - SSchema* pSchema; -} SSchemaWrapper; - static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) { SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); if (pSW == NULL) return pSW; pSW->nCols = pSchemaWrapper->nCols; + pSW->sver = pSchemaWrapper->sver; pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { taosMemoryFree(pSW); @@ -2132,8 +2063,8 @@ static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) { static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) { if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1; - if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1; - if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1; + if (tEncodeI32v(pEncoder, pSchema->bytes) < 0) return -1; + if (tEncodeI16v(pEncoder, pSchema->colId) < 0) return -1; if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; return 0; } @@ -2141,15 +2072,16 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) { if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1; - if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1; - if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSchema->bytes) < 0) return -1; + if (tDecodeI16v(pDecoder, &pSchema->colId) < 0) return -1; if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; return 0; } static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { int32_t tlen = 0; - tlen += taosEncodeFixedU32(buf, pSW->nCols); + tlen += taosEncodeVariantI32(buf, pSW->nCols); + tlen += taosEncodeVariantI32(buf, pSW->sver); for (int32_t i = 0; i < pSW->nCols; i++) { tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]); } @@ -2157,7 +2089,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr } static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { - buf = taosDecodeFixedU32(buf, &pSW->nCols); + buf = taosDecodeVariantI32(buf, &pSW->nCols); + buf = taosDecodeVariantI32(buf, &pSW->sver); pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { return NULL; @@ -2170,23 +2103,25 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp } static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) { - if (tEncodeU32(pEncoder, pSW->nCols) < 0) return -1; + if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1; + if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1; for (int32_t i = 0; i < pSW->nCols; i++) { if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; } - return pEncoder->pos; + + return 0; } static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapper* pSW) { - if (tDecodeU32(pDecoder, &pSW->nCols) < 0) return -1; - void* ptr = taosMemoryRealloc(pSW->pSchema, pSW->nCols * sizeof(SSchema)); - if (ptr == NULL) { - return -1; - } - pSW->pSchema = (SSchema*)ptr; + if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; + + pSW->pSchema = (SSchema*)TCODER_MALLOC(pDecoder, sizeof(SSchema) * pSW->nCols); + if (pSW->pSchema == NULL) return -1; for (int32_t i = 0; i < pSW->nCols; i++) { if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; } + return 0; } @@ -2442,22 +2377,6 @@ typedef struct { int64_t consumerId; } SMqRspHead; -#if 0 -typedef struct { - SMsgHead head; - - int64_t consumerId; - int64_t blockingTime; - int32_t epoch; - int8_t withSchema; - char cgroup[TSDB_CGROUP_LEN]; - - int64_t currentOffset; - uint64_t reqId; - char topic[TSDB_TOPIC_FNAME_LEN]; -} SMqPollReq; -#endif - typedef struct { SMsgHead head; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; @@ -2481,18 +2400,6 @@ typedef struct { SSchemaWrapper schema; } SMqSubTopicEp; -typedef struct { - SMqRspHead head; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - // TODO: replace with topic name - int32_t numOfTopics; - // TODO: remove from msg - SSchemaWrapper* schema; - SArray* pBlockData; // SArray -} SMqPollRsp; - typedef struct { SMqRspHead head; int64_t reqOffset; @@ -2616,7 +2523,7 @@ typedef struct { SMqRspHead head; char cgroup[TSDB_CGROUP_LEN]; SArray* topics; // SArray -} SMqCMGetSubEpRsp; +} SMqAskEpRsp; static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { // taosMemoryFree(pSubTopicEp->schema.pSchema); @@ -2638,10 +2545,6 @@ static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { return buf; } -static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) { - taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); -} - static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { int32_t tlen = 0; tlen += taosEncodeString(buf, pTopicEp->topic); @@ -2674,7 +2577,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE return buf; } -static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { +static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pRsp) { int32_t tlen = 0; // tlen += taosEncodeString(buf, pRsp->cgroup); int32_t sz = taosArrayGetSize(pRsp->topics); @@ -2686,7 +2589,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu return tlen; } -static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { +static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) { // buf = taosDecodeStringTo(buf, pRsp->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); @@ -2702,6 +2605,10 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p return buf; } +static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { + taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); +} + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 97ee66a2da..1976db3a12 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -145,7 +145,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-mq-ask-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) diff --git a/include/common/trow.h b/include/common/trow.h index bc5413f9ce..8732497dbb 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -23,7 +23,6 @@ #include "tbuffer.h" #include "tdataformat.h" #include "tdef.h" -#include "tschema.h" #include "ttypes.h" #include "tutil.h" @@ -58,12 +57,12 @@ extern "C" { #define TD_ROWS_ALL_NORM 0x00U #define TD_ROWS_NULL_NORM 0x01U -#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM +#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v)) -#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) -#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) +#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) +#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) -#define KvConvertRatio (0.9f) +#define KvConvertRatio (0.9f) #define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio)) #ifdef TD_SUPPORT_BITMAP @@ -341,7 +340,8 @@ static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, return false; } -static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { +static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, + int8_t bitmapMode) { switch (bitmapMode) { case 0: tdGetBitmapValTypeII(pBitmap, colIdx, pValType); diff --git a/include/common/tschema.h b/include/common/tschema.h deleted file mode 100644 index 7a270fc9d9..0000000000 --- a/include/common/tschema.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_COMMON_SCHEMA_H_ -#define _TD_COMMON_SCHEMA_H_ - -#include "os.h" -#include "tarray.h" -#include "ttypes.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#if 0 -typedef struct STColumn { - /// column name - char *cname; - union { - /// for encode purpose - uint64_t info; - struct { - uint64_t sma : 1; - /// column data type - uint64_t type : 7; - /// column id - uint64_t cid : 16; - /// max bytes of the column - uint64_t bytes : 32; - /// reserved - uint64_t reserve : 8; - }; - }; - /// comment about the column - char *comment; -} STColumn; - -typedef struct STSchema { - /// schema version - uint16_t sver; - /// number of columns - uint16_t ncols; - /// sma attributes - struct { - bool sma; - SArray *smaArray; - }; - /// column info - STColumn cols[]; -} STSchema; - -typedef struct { - uint64_t size; - STSchema *pSchema; -} STShemaBuilder; - -#define tSchemaBuilderInit(target, capacity) \ - { .size = (capacity), .pSchema = (target) } -void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver); -void tSchemaBuilderSetSMA(bool sma, SArray *smaArray); -int32_t tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment); - -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_COMMON_SCHEMA_H_*/ \ No newline at end of file diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index d7d53ad1d0..d534b0405f 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -221,7 +221,7 @@ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw); * @param pRaw The raw data. * @return int32_t 0 for success, -1 for failure. */ -int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw); +int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw); /** * @brief Acquire a row from sdb diff --git a/include/libs/function/function.h b/include/libs/function/function.h index ef32533e5f..ec0d644aba 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -202,7 +202,7 @@ typedef struct SqlFunctionCtx { SPoint1 end; SFuncExecFuncs fpSet; SScalarFuncExecFuncs sfp; - SExprInfo *pExpr; + struct SExprInfo *pExpr; struct SDiskbasedBuf *pBuf; struct SSDataBlock *pSrcBlock; int32_t curBufPage; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 56e25d49c1..126a2ccf99 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include "querynodes.h" #include "function.h" +#include "querynodes.h" typedef enum EFunctionType { // aggregate function @@ -123,10 +123,10 @@ struct SCatalog; typedef struct SFmGetFuncInfoParam { struct SCatalog* pCtg; - void *pRpc; - const SEpSet* pMgmtEps; - char* pErrBuf; - int32_t errBufLen; + void* pRpc; + const SEpSet* pMgmtEps; + char* pErrBuf; + int32_t errBufLen; } SFmGetFuncInfoParam; int32_t fmFuncMgtInit(); @@ -143,6 +143,7 @@ bool fmIsDatetimeFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); bool fmIsPseudoColumnFunc(int32_t funcId); +bool fmIsScanPseudoColumnFunc(int32_t funcId); bool fmIsWindowPseudoColumnFunc(int32_t funcId); bool fmIsWindowClauseFunc(int32_t funcId); bool fmIsSpecialDataRequiredFunc(int32_t funcId); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 829770ed1b..d4197cd399 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -20,50 +20,46 @@ extern "C" { #endif -#include "querynodes.h" #include "query.h" +#include "querynodes.h" #include "tname.h" typedef struct SLogicNode { - ENodeType type; - SNodeList* pTargets; // SColumnNode - SNode* pConditions; - SNodeList* pChildren; + ENodeType type; + SNodeList* pTargets; // SColumnNode + SNode* pConditions; + SNodeList* pChildren; struct SLogicNode* pParent; - int32_t optimizedFlag; + int32_t optimizedFlag; } SLogicNode; -typedef enum EScanType { - SCAN_TYPE_TAG, - SCAN_TYPE_TABLE, - SCAN_TYPE_SYSTEM_TABLE, - SCAN_TYPE_STREAM -} EScanType; +typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType; typedef struct SScanLogicNode { - SLogicNode node; - SNodeList* pScanCols; + SLogicNode node; + SNodeList* pScanCols; + SNodeList* pScanPseudoCols; struct STableMeta* pMeta; - SVgroupsInfo* pVgroupList; - EScanType scanType; - uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count - STimeWindow scanRange; - SName tableName; - bool showRewrite; - double ratio; - SNodeList* pDynamicScanFuncs; - int32_t dataRequired; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; + SVgroupsInfo* pVgroupList; + EScanType scanType; + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count + STimeWindow scanRange; + SName tableName; + bool showRewrite; + double ratio; + SNodeList* pDynamicScanFuncs; + int32_t dataRequired; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; } SScanLogicNode; typedef struct SJoinLogicNode { SLogicNode node; - EJoinType joinType; - SNode* pOnConditions; + EJoinType joinType; + SNode* pOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -75,47 +71,43 @@ typedef struct SAggLogicNode { typedef struct SProjectLogicNode { SLogicNode node; SNodeList* pProjections; - char stmtName[TSDB_TABLE_NAME_LEN]; - int64_t limit; - int64_t offset; - int64_t slimit; - int64_t soffset; + char stmtName[TSDB_TABLE_NAME_LEN]; + int64_t limit; + int64_t offset; + int64_t slimit; + int64_t soffset; } SProjectLogicNode; typedef struct SVnodeModifLogicNode { - SLogicNode node; - int32_t msgType; - SArray* pDataBlocks; + SLogicNode node; + int32_t msgType; + SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; } SVnodeModifLogicNode; typedef struct SExchangeLogicNode { SLogicNode node; - int32_t srcGroupId; - uint8_t precision; + int32_t srcGroupId; + uint8_t precision; } SExchangeLogicNode; -typedef enum EWindowType { - WINDOW_TYPE_INTERVAL = 1, - WINDOW_TYPE_SESSION, - WINDOW_TYPE_STATE -} EWindowType; +typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; typedef struct SWindowLogicNode { - SLogicNode node; + SLogicNode node; EWindowType winType; - SNodeList* pFuncs; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; - SFillNode* pFill; - int64_t sessionGap; - SNode* pTspk; - SNode* pStateExpr; - int8_t triggerType; - int64_t watermark; + SNodeList* pFuncs; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; + SFillNode* pFill; + int64_t sessionGap; + SNode* pTspk; + SNode* pStateExpr; + int8_t triggerType; + int64_t watermark; } SWindowLogicNode; typedef struct SSortLogicNode { @@ -137,59 +129,60 @@ typedef enum ESubplanType { typedef struct SSubplanId { uint64_t queryId; - int32_t groupId; - int32_t subplanId; + int32_t groupId; + int32_t subplanId; } SSubplanId; typedef struct SLogicSubplan { - ENodeType type; - SSubplanId id; - SNodeList* pChildren; - SNodeList* pParents; - SLogicNode* pNode; - ESubplanType subplanType; + ENodeType type; + SSubplanId id; + SNodeList* pChildren; + SNodeList* pParents; + SLogicNode* pNode; + ESubplanType subplanType; SVgroupsInfo* pVgroupList; - int32_t level; - int32_t splitFlag; + int32_t level; + int32_t splitFlag; } SLogicSubplan; typedef struct SQueryLogicPlan { - ENodeType type; + ENodeType type; SNodeList* pTopSubplans; } SQueryLogicPlan; typedef struct SSlotDescNode { ENodeType type; - int16_t slotId; + int16_t slotId; SDataType dataType; - bool reserve; - bool output; - bool tag; + bool reserve; + bool output; + bool tag; } SSlotDescNode; typedef struct SDataBlockDescNode { - ENodeType type; - int16_t dataBlockId; + ENodeType type; + int16_t dataBlockId; SNodeList* pSlots; - int32_t totalRowSize; - int32_t outputRowSize; - uint8_t precision; + int32_t totalRowSize; + int32_t outputRowSize; + uint8_t precision; } SDataBlockDescNode; typedef struct SPhysiNode { - ENodeType type; + ENodeType type; SDataBlockDescNode* pOutputDataBlockDesc; - SNode* pConditions; - SNodeList* pChildren; - struct SPhysiNode* pParent; + SNode* pConditions; + SNodeList* pChildren; + struct SPhysiNode* pParent; } SPhysiNode; typedef struct SScanPhysiNode { SPhysiNode node; SNodeList* pScanCols; - uint64_t uid; // unique id of the table - int8_t tableType; - SName tableName; + SNodeList* pScanPseudoCols; + uint64_t uid; // unique id of the table + int8_t tableType; + SName tableName; } SScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode; @@ -197,23 +190,23 @@ typedef SScanPhysiNode SStreamScanPhysiNode; typedef struct SSystemTableScanPhysiNode { SScanPhysiNode scan; - SEpSet mgmtEpSet; - bool showRewrite; - int32_t accountId; + SEpSet mgmtEpSet; + bool showRewrite; + int32_t accountId; } SSystemTableScanPhysiNode; typedef struct STableScanPhysiNode { SScanPhysiNode scan; - uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count - STimeWindow scanRange; - double ratio; - int32_t dataRequired; - SNodeList* pDynamicScanFuncs; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count + STimeWindow scanRange; + double ratio; + int32_t dataRequired; + SNodeList* pDynamicScanFuncs; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; @@ -221,89 +214,89 @@ typedef STableScanPhysiNode STableSeqScanPhysiNode; typedef struct SProjectPhysiNode { SPhysiNode node; SNodeList* pProjections; - int64_t limit; - int64_t offset; - int64_t slimit; - int64_t soffset; + int64_t limit; + int64_t offset; + int64_t slimit; + int64_t soffset; } SProjectPhysiNode; typedef struct SJoinPhysiNode { SPhysiNode node; - EJoinType joinType; - SNode* pOnConditions; // in or out tuple ? + EJoinType joinType; + SNode* pOnConditions; // in or out tuple ? SNodeList* pTargets; } SJoinPhysiNode; typedef struct SAggPhysiNode { SPhysiNode node; - SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function + SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function SNodeList* pGroupKeys; SNodeList* pAggFuncs; } SAggPhysiNode; typedef struct SDownstreamSourceNode { - ENodeType type; + ENodeType type; SQueryNodeAddr addr; - uint64_t taskId; - uint64_t schedId; + uint64_t taskId; + uint64_t schedId; } SDownstreamSourceNode; typedef struct SExchangePhysiNode { SPhysiNode node; - int32_t srcGroupId; // group id of datasource suplans + int32_t srcGroupId; // group id of datasource suplans SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; typedef struct SWinodwPhysiNode { SPhysiNode node; - SNodeList* pExprs; // these are expression list of parameter expression of function + SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; - SNode* pTspk; // timestamp primary key - int8_t triggerType; - int64_t watermark; + SNode* pTspk; // timestamp primary key + int8_t triggerType; + int64_t watermark; } SWinodwPhysiNode; typedef struct SIntervalPhysiNode { SWinodwPhysiNode window; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; - SFillNode* pFill; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; + SFillNode* pFill; } SIntervalPhysiNode; typedef struct SMultiTableIntervalPhysiNode { SIntervalPhysiNode interval; - SNodeList* pPartitionKeys; + SNodeList* pPartitionKeys; } SMultiTableIntervalPhysiNode; typedef struct SSessionWinodwPhysiNode { SWinodwPhysiNode window; - int64_t gap; + int64_t gap; } SSessionWinodwPhysiNode; typedef struct SStateWinodwPhysiNode { SWinodwPhysiNode window; - SNode* pStateKey; + SNode* pStateKey; } SStateWinodwPhysiNode; typedef struct SSortPhysiNode { SPhysiNode node; - SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function - SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode + SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function + SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode SNodeList* pTargets; } SSortPhysiNode; typedef struct SPartitionPhysiNode { SPhysiNode node; - SNodeList* pExprs; // these are expression list of partition_by_clause + SNodeList* pExprs; // these are expression list of partition_by_clause SNodeList* pPartitionKeys; SNodeList* pTargets; } SPartitionPhysiNode; typedef struct SDataSinkNode { - ENodeType type; + ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; } SDataSinkNode; @@ -313,45 +306,41 @@ typedef struct SDataDispatcherNode { typedef struct SDataInserterNode { SDataSinkNode sink; - int32_t numOfTables; - uint32_t size; - char *pData; + int32_t numOfTables; + uint32_t size; + char* pData; } SDataInserterNode; typedef struct SSubplan { - ENodeType type; - SSubplanId id; // unique id of the subplan - ESubplanType subplanType; - int32_t msgType; // message type for subplan, used to denote the send message type to vnode. - int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. - char dbFName[TSDB_DB_FNAME_LEN]; - SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node - SQueryNodeStat execNodeStat; // only for scan subplan - SNodeList* pChildren; // the datasource subplan,from which to fetch the result - SNodeList* pParents; // the data destination subplan, get data from current subplan - SPhysiNode* pNode; // physical plan of current subplan - SDataSinkNode* pDataSink; // data of the subplan flow into the datasink + ENodeType type; + SSubplanId id; // unique id of the subplan + ESubplanType subplanType; + int32_t msgType; // message type for subplan, used to denote the send message type to vnode. + int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. + char dbFName[TSDB_DB_FNAME_LEN]; + SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node + SQueryNodeStat execNodeStat; // only for scan subplan + SNodeList* pChildren; // the datasource subplan,from which to fetch the result + SNodeList* pParents; // the data destination subplan, get data from current subplan + SPhysiNode* pNode; // physical plan of current subplan + SDataSinkNode* pDataSink; // data of the subplan flow into the datasink } SSubplan; -typedef enum EExplainMode { - EXPLAIN_MODE_DISABLE = 1, - EXPLAIN_MODE_STATIC, - EXPLAIN_MODE_ANALYZE -} EExplainMode; +typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode; typedef struct SExplainInfo { EExplainMode mode; - bool verbose; - double ratio; + bool verbose; + double ratio; } SExplainInfo; typedef struct SQueryPlan { - ENodeType type; - uint64_t queryId; - int32_t numOfSubplans; - SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. + ENodeType type; + uint64_t queryId; + int32_t numOfSubplans; + SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SExplainInfo explainInfo; - SNodeList* pPlaceholderValues; + SNodeList* pPlaceholderValues; } SQueryPlan; void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 1605fe8ac8..238c4c8538 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -293,7 +293,10 @@ typedef struct SExplainStmt { void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext); -int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols); +typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType; + +int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, + SNodeList** pCols); typedef bool (*FFuncClassifier)(int32_t funcId); int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 58482735ba..875cdc2755 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -88,11 +88,11 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); void qDestroyStmtDataBlock(void* pBlock); -int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); -int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); +int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); +int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 78d0911502..b78cbb639c 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -50,7 +50,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo // @pSource one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); -int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams); +int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx); // Convert to subplan to string for the scheduler to send to the executor int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 0be27e6d3a..ffe435a8b2 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -158,6 +158,8 @@ typedef enum { int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); +bool syncEnvIsStart(); + extern int32_t sDebugFlag; //----------------------------------------- diff --git a/include/util/taoserror.h b/include/util/taoserror.h index f0c3cc4b14..2c249a2d8d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -139,16 +139,9 @@ int32_t* taosGetErrno(); // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) #define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0301) -#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0302) -#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0303) -#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0304) -#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0305) -#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0306) -#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0307) -#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0308) -#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0309) -#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x030A) -#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x030B) +#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302) +#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) +#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0304) // mnode-show #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310) diff --git a/include/util/tdef.h b/include/util/tdef.h index aa0d0bd8ff..7a1fe5dd7b 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -125,12 +125,12 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_INS_TABLE_QUERIES "queries" #define TSDB_INS_TABLE_VNODES "vnodes" -#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" -#define TSDB_PERFS_TABLE_CONNECTIONS "connections" -#define TSDB_PERFS_TABLE_QUERIES "queries" -#define TSDB_PERFS_TABLE_TOPICS "topics" -#define TSDB_PERFS_TABLE_CONSUMERS "consumers" -#define TSDB_PERFS_TABLE_SUBSCRIBES "subscribes" +#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" +#define TSDB_PERFS_TABLE_CONNECTIONS "connections" +#define TSDB_PERFS_TABLE_QUERIES "queries" +#define TSDB_PERFS_TABLE_TOPICS "topics" +#define TSDB_PERFS_TABLE_CONSUMERS "consumers" +#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions" #define TSDB_INDEX_TYPE_SMA "SMA" #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT" @@ -195,6 +195,8 @@ typedef enum EOperatorType { OP_TYPE_JSON_CONTAINS } EOperatorType; +#define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR + typedef enum ELogicConditionType { LOGIC_COND_TYPE_AND = 1, LOGIC_COND_TYPE_OR, @@ -270,7 +272,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 -#define TSDB_MAX_JSON_TAG_LEN 16384 +#define TSDB_MAX_JSON_TAG_LEN 16384 #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 @@ -284,8 +286,9 @@ typedef enum ELogicConditionType { #define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 #define TSDB_SHOW_SQL_LEN 512 -#define TSDB_SHOW_SUBQUERY_LEN 1000 #define TSDB_SLOW_QUERY_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_SHOW_LIST_LEN 1000 #define TSDB_TRANS_STAGE_LEN 12 #define TSDB_TRANS_TYPE_LEN 16 @@ -372,7 +375,7 @@ typedef enum ELogicConditionType { #define TSDB_DB_STREAM_MODE_OFF 0 #define TSDB_DB_STREAM_MODE_ON 1 #define TSDB_DEFAULT_DB_STREAM_MODE 0 -#define TSDB_DB_SINGLE_STABLE_ON 0 +#define TSDB_DB_SINGLE_STABLE_ON 0 #define TSDB_DB_SINGLE_STABLE_OFF 1 #define TSDB_DEFAULT_DB_SINGLE_STABLE 0 diff --git a/include/util/tencode.h b/include/util/tencode.h index 11357ba533..0236fe58da 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -17,7 +17,8 @@ #define _TD_UTIL_ENCODE_H_ #include "tcoding.h" -#include "tfreelist.h" +#include "tlist.h" +// #include "tfreelist.h" #ifdef __cplusplus extern "C" { @@ -62,10 +63,14 @@ struct SCoderNode { CODER_NODE_FIELDS }; +typedef struct SCoderMem { + struct SCoderMem* next; +} SCoderMem; + typedef struct { td_coder_t type; td_endian_t endian; - SFreeList fl; + SCoderMem* mList; CODER_NODE_FIELDS TD_SLIST(SCoderNode) stack; } SCoder; @@ -74,7 +79,31 @@ typedef struct { #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -#define TCODER_MALLOC(PTR, TYPE, SIZE, CODER) TFL_MALLOC(PTR, TYPE, SIZE, &((CODER)->fl)) +#define TCODER_MALLOC(PCODER, SIZE) \ + ({ \ + void* ptr = NULL; \ + SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \ + if (pMem) { \ + pMem->next = (PCODER)->mList; \ + (PCODER)->mList = pMem; \ + ptr = (void*)&pMem[1]; \ + } \ + ptr; \ + }) + +#define tEncodeSize(E, S, SIZE) \ + ({ \ + SCoder coder = {0}; \ + int ret = 0; \ + tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \ + if ((E)(&coder, S) == 0) { \ + SIZE = coder.pos; \ + } else { \ + ret = -1; \ + } \ + tCoderClear(&coder); \ + ret; \ + }) void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderClear(SCoder* pCoder); @@ -377,7 +406,9 @@ static FORCE_INLINE int32_t tDecodeBinary(SCoder* pDecoder, const void** val, ui if (tDecodeU64v(pDecoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; - *val = (void*)TD_CODER_CURRENT(pDecoder); + if (val) { + *val = (void*)TD_CODER_CURRENT(pDecoder); + } TD_CODER_MOVE_POS(pDecoder, *len); return 0; diff --git a/include/util/tlog.h b/include/util/tlog.h index 32421a59cc..dead25a4a8 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -59,6 +59,7 @@ extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; extern int32_t fsDebugFlag; +extern int32_t metaDebugFlag; extern int32_t fnDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); @@ -79,6 +80,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons #endif ; +// clang-format off #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} @@ -88,6 +90,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } +// clang-format on #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c0237a184a..41448a4391 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -310,6 +310,9 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v void hbMgrInitMqHbRspHandle(); SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery); +int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList); +int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); + #ifdef __cplusplus } diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 04e9c3be9a..061d757551 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -34,8 +34,7 @@ typedef enum { STMT_PREPARE, STMT_SETTBNAME, STMT_SETTAGS, - STMT_FETCH_TAG_FIELDS, - STMT_FETCH_COL_FIELDS, + STMT_FETCH_FIELDS, STMT_BIND, STMT_BIND_COL, STMT_ADD_BATCH, @@ -75,6 +74,8 @@ typedef struct SStmtSQLInfo { SQuery* pQuery; char* sqlStr; int32_t sqlLen; + SArray* nodeList; + SQueryPlan* pQueryPlan; } SStmtSQLInfo; typedef struct STscStmt { @@ -87,6 +88,8 @@ typedef struct STscStmt { SStmtBindInfo bInfo; } STscStmt; +#define STMT_STATUS_NE(S) (pStmt->sql.status != STMT_##S) +#define STMT_STATUS_EQ(S) (pStmt->sql.status == STMT_##S) #define STMT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) @@ -97,14 +100,15 @@ int stmtClose(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt); const char *stmtErrstr(TAOS_STMT *stmt); int stmtAffectedRows(TAOS_STMT *stmt); +int stmtAffectedRowsOnce(TAOS_STMT *stmt); int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length); int stmtSetTbName(TAOS_STMT *stmt, const char *tbName); -int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags); +int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags); int stmtIsInsert(TAOS_STMT *stmt, int *insert); int stmtGetParamNum(TAOS_STMT *stmt, int *nums); int stmtAddBatch(TAOS_STMT *stmt); TAOS_RES *stmtUseResult(TAOS_STMT *stmt); -int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx); +int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx); #ifdef __cplusplus diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a40edbb65a..a00fd4714e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -305,6 +305,11 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList return pRequest->code; } +int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) { + *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList); +} + SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) { if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index a9efcced40..0e7563bb13 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -603,7 +603,7 @@ int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { return stmtPrepare(stmt, sql, length); } -int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags) { +int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags) { if (stmt == NULL || name == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; @@ -636,7 +636,7 @@ int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { return taos_stmt_set_tbname(stmt, name); } -int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { +int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; @@ -652,7 +652,7 @@ int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { return stmtBindBatch(stmt, bind, -1); } -int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { +int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; @@ -665,10 +665,18 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { return terrno; } + int32_t insert = 0; + stmtIsInsert(stmt, &insert); + if (0 == insert && bind->num > 1) { + tscError("only one row data allowed for query"); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return stmtBindBatch(stmt, bind, -1); } -int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) { +int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx) { if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; @@ -680,6 +688,14 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int c terrno = TSDB_CODE_INVALID_PARA; return terrno; } + + int32_t insert = 0; + stmtIsInsert(stmt, &insert); + if (0 == insert && bind->num > 1) { + tscError("only one row data allowed for query"); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } return stmtBindBatch(stmt, bind, colIdx); } @@ -748,6 +764,16 @@ int taos_stmt_affected_rows(TAOS_STMT *stmt) { return stmtAffectedRows(stmt); } +int taos_stmt_affected_rows_once(TAOS_STMT *stmt) { + if (stmt == NULL) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return 0; + } + + return stmtAffectedRowsOnce(stmt); +} + int taos_stmt_close(TAOS_STMT *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 0972ff3477..20c318e212 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -5,14 +5,52 @@ #include "tdef.h" int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { + int32_t code = 0; + switch (newStatus) { + case STMT_PREPARE: + break; case STMT_SETTBNAME: + if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND) || STMT_STATUS_EQ(BIND_COL)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } + break; + case STMT_SETTAGS: + if (STMT_STATUS_NE(SETTBNAME)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } + break; + case STMT_FETCH_FIELDS: + if (STMT_STATUS_EQ(INIT)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } + break; + case STMT_BIND: + if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND_COL)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } + break; + case STMT_BIND_COL: + if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } + break; + case STMT_ADD_BATCH: + if (STMT_STATUS_NE(BIND) && STMT_STATUS_NE(BIND_COL) && STMT_STATUS_NE(FETCH_FIELDS)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } + break; + case STMT_EXECUTE: + if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } break; default: + code = TSDB_CODE_TSC_APP_ERROR; break; } - //STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERR_RET(code); pStmt->sql.status = newStatus; @@ -69,15 +107,10 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { return TSDB_CODE_SUCCESS; } - uint64_t uid; - if (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) { - uid = pStmt->bInfo.tbSuid; - } else { - ASSERT(TSDB_NORMAL_TABLE == pStmt->bInfo.tbType); - uid = pStmt->bInfo.tbUid; - } + uint64_t uid = pStmt->bInfo.tbUid; + uint64_t tuid = (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) ? pStmt->bInfo.tbSuid : uid; - if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) { + if (taosHashGet(pStmt->sql.pTableCache, &tuid, sizeof(tuid))) { return TSDB_CODE_SUCCESS; } @@ -91,7 +124,7 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { .boundTags = pStmt->bInfo.boundTags, }; - if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) { + if (taosHashPut(pStmt->sql.pTableCache, &tuid, sizeof(tuid), &cache, sizeof(cache))) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -149,9 +182,11 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) { - taos_free_result(pStmt->exec.pRequest); - pStmt->exec.pRequest = NULL; +int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { + if (STMT_TYPE_QUERY != pStmt->sql.type || freeRequest) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + } void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { @@ -186,7 +221,9 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) { int32_t stmtCleanSQLInfo(STscStmt* pStmt) { taosMemoryFree(pStmt->sql.sqlStr); qDestroyQuery(pStmt->sql.pQuery); - + qDestroyQueryPlan(pStmt->sql.pQueryPlan); + taosArrayDestroy(pStmt->sql.nodeList); + void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); while (pIter) { SStmtTableCache* pCache = (SStmtTableCache*)pIter; @@ -201,7 +238,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { memset(&pStmt->sql, 0, sizeof(pStmt->sql)); - STMT_ERR_RET(stmtCleanExecInfo(pStmt, false)); + STMT_ERR_RET(stmtCleanExecInfo(pStmt, false, true)); STMT_ERR_RET(stmtCleanBindInfo(pStmt)); return TSDB_CODE_SUCCESS; @@ -333,6 +370,13 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME)); + int32_t insert = 0; + stmtIsInsert(stmt, &insert); + if (0 == insert) { + tscError("set tb name not available for none insert statement"); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); + } + if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } @@ -349,7 +393,7 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { return TSDB_CODE_SUCCESS; } -int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { +int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) { STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); @@ -370,15 +414,7 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { } -int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { - STscStmt* pStmt = (STscStmt*)stmt; - - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_TAG_FIELDS)); - - if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); - } - +int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) { if (STMT_TYPE_QUERY == pStmt->sql.type) { tscError("invalid operation to get query tag fileds"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); @@ -395,15 +431,7 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel return TSDB_CODE_SUCCESS; } -int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { - STscStmt* pStmt = (STscStmt*)stmt; - - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_COL_FIELDS)); - - if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); - } - +int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) { if (STMT_TYPE_QUERY == pStmt->sql.type) { tscError("invalid operation to get query column fileds"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); @@ -420,7 +448,7 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel return TSDB_CODE_SUCCESS; } -int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) { +int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); @@ -429,6 +457,11 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) { pStmt->bInfo.needParse = false; } + if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + } + if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } @@ -437,6 +470,16 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) { STMT_ERR_RET(stmtParseSql(pStmt)); } + if (STMT_TYPE_QUERY == pStmt->sql.type) { + if (NULL == pStmt->sql.pQueryPlan) { + STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); + pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; + pStmt->exec.pRequest->body.pDag = NULL; + } + + STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx)); + } + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); @@ -480,10 +523,13 @@ int stmtExec(TAOS_STMT *stmt) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); - STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); - - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true); - + if (STMT_TYPE_QUERY == pStmt->sql.type) { + scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList); + } else { + STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true); + } + STMT_ERR_JRET(pStmt->exec.pRequest->code); pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest); @@ -491,7 +537,7 @@ int stmtExec(TAOS_STMT *stmt) { _return: - stmtCleanExecInfo(pStmt, (code ? false : true)); + stmtCleanExecInfo(pStmt, (code ? false : true), false); ++pStmt->sql.runTimes; @@ -523,6 +569,10 @@ int stmtAffectedRows(TAOS_STMT *stmt) { return ((STscStmt*)stmt)->affectedRows; } +int stmtAffectedRowsOnce(TAOS_STMT *stmt) { + return ((STscStmt*)stmt)->exec.affectedRows; +} + int stmtIsInsert(TAOS_STMT *stmt, int *insert) { STscStmt* pStmt = (STscStmt*)stmt; @@ -536,13 +586,38 @@ int stmtIsInsert(TAOS_STMT *stmt, int *insert) { } int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { - STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + if (STMT_TYPE_QUERY == pStmt->sql.type) { + if (NULL == pStmt->sql.pQueryPlan) { + STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); + pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; + pStmt->exec.pRequest->body.pDag = NULL; + } + + *nums = (pStmt->sql.pQueryPlan->pPlaceholderValues) ? pStmt->sql.pQueryPlan->pPlaceholderValues->length : 0; + } else { + STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); + } return TSDB_CODE_SUCCESS; } TAOS_RES *stmtUseResult(TAOS_STMT *stmt) { - return NULL; + STscStmt* pStmt = (STscStmt*)stmt; + + if (STMT_TYPE_QUERY != pStmt->sql.type) { + tscError("useResult only for query statement"); + return NULL; + } + + return pStmt->exec.pRequest; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 66e10f8c6a..e0fb46e122 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -25,7 +25,14 @@ #include "tref.h" #include "ttimer.h" -int32_t tmqAskEp(tmq_t* tmq, bool sync); +int32_t tmqAskEp(tmq_t* tmq, bool async); + +typedef struct { + int8_t inited; + tmr_h timer; +} SMqMgmt; + +static SMqMgmt tmqMgmt = {0}; typedef struct { int8_t tmqRspType; @@ -33,9 +40,9 @@ typedef struct { } SMqRspWrapper; typedef struct { - int8_t tmqRspType; - int32_t epoch; - SMqCMGetSubEpRsp msg; + int8_t tmqRspType; + int32_t epoch; + SMqAskEpRsp msg; } SMqAskEpRspWrapper; struct tmq_list_t { @@ -64,13 +71,6 @@ struct tmq_conf_t { tmq_commit_cb* commit_cb; }; -typedef struct { - int8_t inited; - tmr_h timer; -} SMqMgmt; - -static SMqMgmt tmqMgmt = {0}; - struct tmq_t { // conf char groupId[TSDB_CGROUP_LEN]; @@ -164,7 +164,7 @@ typedef struct { typedef struct { tmq_t* tmq; int32_t code; - int32_t sync; + int32_t async; tsem_t rspSem; } SMqAskEpCbParam; @@ -188,6 +188,7 @@ typedef struct { tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); conf->autoCommit = false; + conf->autoCommitInterval = 5000; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; return conf; } @@ -324,7 +325,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { if (pTaskType == NULL) break; if (*pTaskType == TMQ_DELAYED_TASK__HB) { - tmqAskEp(tmq, false); + tmqAskEp(tmq, true); taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { tmq_commit(tmq, NULL, true); @@ -472,8 +473,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); - /*pTmq->autoCommit = conf->autoCommit;*/ - pTmq->autoCommit = 0; + pTmq->autoCommit = conf->autoCommit; pTmq->autoCommitInterval = conf->autoCommitInterval; pTmq->commit_cb = conf->commit_cb; pTmq->resetOffsetCfg = conf->resetOffset; @@ -662,8 +662,8 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { if (code != 0) goto FAIL; // TODO: add max retry cnt - while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, true)) { - tscDebug("not ready, retry\n"); + while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + tscDebug("not ready, retry"); taosMsleep(500); } @@ -854,7 +854,7 @@ CREATE_MSG_FAIL: return -1; } -bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { +bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); @@ -936,7 +936,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tmq_t* tmq = pParam->tmq; pParam->code = code; if (code != 0) { - tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync); + tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); goto END; } @@ -950,15 +950,15 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { goto END; } - if (pParam->sync) { - SMqCMGetSubEpRsp rsp; - tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); + if (!pParam->async) { + SMqAskEpRsp rsp; + tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ if (tmqUpdateEp(tmq, head->epoch, &rsp)) { atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); } - tDeleteSMqCMGetSubEpRsp(&rsp); + tDeleteSMqAskEpRsp(&rsp); } else { SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper)); if (pWrapper == NULL) { @@ -969,7 +969,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; pWrapper->epoch = head->epoch; memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg); + tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg); taosWriteQitem(tmq->mqueue, pWrapper); /*tsem_post(&tmq->rspSem);*/ @@ -978,13 +978,13 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { END: /*atomic_store_8(&tmq->epStatus, 0);*/ - if (pParam->sync) { + if (!pParam->async) { tsem_post(&pParam->rspSem); } return code; } -int32_t tmqAskEp(tmq_t* tmq, bool sync) { +int32_t tmqAskEp(tmq_t* tmq, bool async) { int32_t code = 0; #if 0 int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); @@ -995,8 +995,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { } atomic_store_32(&tmq->epSkipCnt, 0); #endif - int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen); + int32_t tlen = sizeof(SMqAskEpReq); + SMqAskEpReq* req = taosMemoryMalloc(tlen); if (req == NULL) { tscError("failed to malloc get subscribe ep buf"); /*atomic_store_8(&tmq->epStatus, 0);*/ @@ -1014,7 +1014,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { return -1; } pParam->tmq = tmq; - pParam->sync = sync; + pParam->async = async; tsem_init(&pParam->rspSem, 0, 0); SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); @@ -1036,7 +1036,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqAskEpCb; - sendInfo->msgType = TDMT_MND_GET_SUB_EP; + sendInfo->msgType = TDMT_MND_MQ_ASK_EP; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -1045,7 +1045,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - if (sync) { + if (!async) { tsem_wait(&pParam->rspSem); code = pParam->code; taosMemoryFree(pParam); @@ -1209,7 +1209,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/ if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; - SMqCMGetSubEpRsp* rspMsg = &pEpRspWrapper->msg; + SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ *pReset = true; @@ -1271,15 +1271,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { SMqRspObj* rspObj; int64_t startTime = taosGetTimestampMs(); - // TODO: put into delayed queue -#if 0 - int8_t status = atomic_load_8(&tmq->status); - while (0 != tmqAskEp(tmq, status != TMQ_CONSUMER_STATUS__READY)) { - tscDebug("not ready, retry\n"); - taosSsleep(1); - } -#endif - rspObj = tmqHandleAllRsp(tmq, blocking_time, false); if (rspObj) { return (TAOS_RES*)rspObj; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 09452b584d..d6a6ba5b13 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -79,7 +79,11 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRo if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length; } else { - return pColumnInfoData->info.bytes * numOfRows; + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) { + return 0; + } else { + return pColumnInfoData->info.bytes * numOfRows; + } } } @@ -307,6 +311,8 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); } + pColumnInfoData->hasNull = pSource->hasNull; + pColumnInfoData->info = pSource->info; return 0; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 98d0bae427..dc52afb382 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -490,181 +490,6 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR return 0; } -int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { - int32_t tlen = 0; - - tlen += taosEncodeString(buf, pReq->name); - tlen += taosEncodeFixedU32(buf, pReq->ttl); - tlen += taosEncodeFixedU32(buf, pReq->keep); - tlen += taosEncodeFixedU8(buf, pReq->info); - - switch (pReq->type) { - case TD_SUPER_TABLE: - tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid); - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nCols); - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols); - for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) { - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].flags); - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); - tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); - } - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols); - for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) { - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].flags); - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); - tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); - } - if (pReq->rollup && pReq->stbCfg.pRSmaParam) { - SRSmaParam *param = pReq->stbCfg.pRSmaParam; - tlen += taosEncodeBinary(buf, (const void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); - tlen += taosEncodeFixedI32(buf, param->delay); - tlen += taosEncodeFixedI32(buf, param->qmsg1Len); - if (param->qmsg1Len > 0) { - tlen += taosEncodeString(buf, param->qmsg1); - } - - tlen += taosEncodeFixedI32(buf, param->qmsg2Len); - if (param->qmsg2Len > 0) { - tlen += taosEncodeString(buf, param->qmsg2); - } - } - break; - case TD_CHILD_TABLE: - tlen += taosEncodeFixedI64(buf, pReq->ctbCfg.suid); - tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); - break; - case TD_NORMAL_TABLE: - tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nCols); - tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols); - for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) { - tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); - tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].flags); - tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId); - tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); - tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); - } - if (pReq->rollup && pReq->ntbCfg.pRSmaParam) { - SRSmaParam *param = pReq->ntbCfg.pRSmaParam; - tlen += taosEncodeBinary(buf, (const void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); - tlen += taosEncodeFixedI32(buf, param->delay); - } - break; - default: - ASSERT(0); - } - - return tlen; -} - -void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { - buf = taosDecodeString(buf, &(pReq->name)); - buf = taosDecodeFixedU32(buf, &(pReq->ttl)); - buf = taosDecodeFixedU32(buf, &(pReq->keep)); - buf = taosDecodeFixedU8(buf, &(pReq->info)); - - switch (pReq->type) { - case TD_SUPER_TABLE: - buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid)); - buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols)); - buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); - pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema)); - for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) { - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].flags)); - buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId)); - buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); - buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); - } - buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols); - pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); - for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) { - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].flags)); - buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId); - buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); - buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); - } - if (pReq->rollup) { - pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam)); - SRSmaParam *param = pReq->stbCfg.pRSmaParam; - buf = taosDecodeBinaryTo(buf, (void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); - buf = taosDecodeFixedI32(buf, ¶m->delay); - buf = taosDecodeFixedI32(buf, ¶m->qmsg1Len); - if (param->qmsg1Len > 0) { - buf = taosDecodeString(buf, ¶m->qmsg1); - } - - buf = taosDecodeFixedI32(buf, ¶m->qmsg2Len); - if (param->qmsg2Len > 0) { - buf = taosDecodeString(buf, ¶m->qmsg2); - } - } else { - pReq->stbCfg.pRSmaParam = NULL; - } - break; - case TD_CHILD_TABLE: - buf = taosDecodeFixedI64(buf, &pReq->ctbCfg.suid); - buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag); - break; - case TD_NORMAL_TABLE: - buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols); - buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); - pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema)); - for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) { - buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); - buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].flags); - buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId); - buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); - buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); - } - if (pReq->rollup) { - pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); - SRSmaParam *param = pReq->ntbCfg.pRSmaParam; - buf = taosDecodeBinaryTo(buf, (void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); - buf = taosDecodeFixedI32(buf, ¶m->delay); - } else { - pReq->ntbCfg.pRSmaParam = NULL; - } - break; - default: - ASSERT(0); - } - - return buf; -} - -int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { - int32_t tlen = 0; - - tlen += taosEncodeFixedI64(buf, pReq->ver); - tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray)); - for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) { - SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i); - tlen += tSerializeSVCreateTbReq(buf, pCreateTbReq); - } - - return tlen; -} - -void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { - uint32_t nsize = 0; - - buf = taosDecodeFixedI64(buf, &pReq->ver); - buf = taosDecodeFixedU32(buf, &nsize); - pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq)); - for (size_t i = 0; i < nsize; i++) { - SVCreateTbReq req = {0}; - buf = tDeserializeSVCreateTbReq(buf, &req); - taosArrayPush(pReq->pArray, &req); - } - - return buf; -} - int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->ver); @@ -692,7 +517,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1; - if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1; if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1; if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1; if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1; @@ -702,6 +526,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeI8(&encoder, pField->type) < 0) return -1; if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1; + if (tEncodeI8(&encoder, pField->flags) < 0) return -1; } for (int32_t i = 0; i < pReq->numOfTags; ++i) { @@ -709,13 +534,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeI8(&encoder, pField->type) < 0) return -1; if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1; - } - - for (int32_t i = 0; i < pReq->numOfSmas; ++i) { - SField *pField = taosArrayGet(pReq->pSmas, i); - if (tEncodeI8(&encoder, pField->type) < 0) return -1; - if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; - if (tEncodeCStr(&encoder, pField->name) < 0) return -1; + if (tEncodeI8(&encoder, pField->flags) < 0) return -1; } if (pReq->commentLen > 0) { @@ -746,15 +565,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1; if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1; if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1; if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1; pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField)); pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField)); - pReq->pSmas = taosArrayInit(pReq->numOfSmas, sizeof(SField)); - if (pReq->pColumns == NULL || pReq->pTags == NULL || pReq->pSmas == NULL) { + if (pReq->pColumns == NULL || pReq->pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -764,6 +581,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeI8(&decoder, &field.type) < 0) return -1; if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; + if (tDecodeI8(&decoder, &field.flags) < 0) return -1; if (taosArrayPush(pReq->pColumns, &field) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -775,23 +593,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeI8(&decoder, &field.type) < 0) return -1; if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; + if (tDecodeI8(&decoder, &field.flags) < 0) return -1; if (taosArrayPush(pReq->pTags, &field) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } } - for (int32_t i = 0; i < pReq->numOfSmas; ++i) { - SField field = {0}; - if (tDecodeI8(&decoder, &field.type) < 0) return -1; - if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; - if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; - if (taosArrayPush(pReq->pSmas, &field) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - } - if (pReq->commentLen > 0) { pReq->comment = taosMemoryMalloc(pReq->commentLen); if (pReq->comment == NULL) return -1; @@ -819,13 +627,11 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { taosArrayDestroy(pReq->pColumns); taosArrayDestroy(pReq->pTags); - taosArrayDestroy(pReq->pSmas); taosMemoryFreeClear(pReq->comment); taosMemoryFreeClear(pReq->pAst1); taosMemoryFreeClear(pReq->pAst2); pReq->pColumns = NULL; pReq->pTags = NULL; - pReq->pSmas = NULL; } int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) { @@ -2141,7 +1947,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr)); if (NULL == pRsp->addrsList) return -1; } - + for (int32_t i = 0; i < num; ++i) { SQueryNodeAddr addr = {0}; if (tDecodeSQueryNodeAddr(&decoder, &addr) < 0) return -1; @@ -3420,7 +3226,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - TCODER_MALLOC(pReq->offsets, SMqOffset *, pReq->num * sizeof(SMqOffset), decoder); + pReq->offsets = (SMqOffset *)TCODER_MALLOC(decoder, sizeof(SMqOffset) * pReq->num); if (pReq->offsets == NULL) return -1; for (int32_t i = 0; i < pReq->num; i++) { tDecodeSMqOffset(decoder, &pReq->offsets[i]); @@ -3636,48 +3442,79 @@ int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pR } int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { - SCoder encoder = {0}; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + // SCoder encoder = {0}; + // tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - if (tStartEncode(&encoder) < 0) return -1; - if (pRsp->rspList) { - int32_t num = taosArrayGetSize(pRsp->rspList); - if (tEncodeI32(&encoder, num) < 0) return -1; - for (int32_t i = 0; i < num; ++i) { - SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); - if (tEncodeI32(&encoder, rsp->code) < 0) return -1; - } - } else { - if (tEncodeI32(&encoder, 0) < 0) return -1; - } - tEndEncode(&encoder); + // if (tStartEncode(&encoder) < 0) return -1; + // if (pRsp->rspList) { + // int32_t num = taosArrayGetSize(pRsp->rspList); + // if (tEncodeI32(&encoder, num) < 0) return -1; + // for (int32_t i = 0; i < num; ++i) { + // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); + // if (tEncodeI32(&encoder, rsp->code) < 0) return -1; + // } + // } else { + // if (tEncodeI32(&encoder, 0) < 0) return -1; + // } + // tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tCoderClear(&encoder); - return tlen; + // int32_t tlen = encoder.pos; + // tCoderClear(&encoder); + // reture tlen; + return 0; } int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { - SCoder decoder = {0}; - int32_t num = 0; - tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + // SCoder decoder = {0}; + // int32_t num = 0; + // tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &num) < 0) return -1; - if (num > 0) { - pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp)); - if (NULL == pRsp->rspList) return -1; - for (int32_t i = 0; i < num; ++i) { - SVCreateTbRsp rsp = {0}; - if (tDecodeI32(&decoder, &rsp.code) < 0) return -1; - if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1; - } - } else { - pRsp->rspList = NULL; + // if (tStartDecode(&decoder) < 0) return -1; + // if (tDecodeI32(&decoder, &num) < 0) return -1; + // if (num > 0) { + // pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp)); + // if (NULL == pRsp->rspList) return -1; + // for (int32_t i = 0; i < num; ++i) { + // SVCreateTbRsp rsp = {0}; + // if (tDecodeI32(&decoder, &rsp.code) < 0) return -1; + // if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1; + // } + // } else { + // pRsp->rspList = NULL; + // } + // tEndDecode(&decoder); + + // tCoderClear(&decoder); + return 0; +} + +int tEncodeSVCreateTbBatchRsp(SCoder *pCoder, const SVCreateTbBatchRsp *pRsp) { + int32_t nRsps = taosArrayGetSize(pRsp->pArray); + SVCreateTbRsp *pCreateRsp; + + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, nRsps) < 0) return -1; + for (int32_t i = 0; i < nRsps; i++) { + pCreateRsp = taosArrayGet(pRsp->pArray, i); + if (tEncodeSVCreateTbRsp(pCoder, pCreateRsp) < 0) return -1; } - tEndDecode(&decoder); - tCoderClear(&decoder); + tEndEncode(pCoder); + + return 0; +} + +int tDecodeSVCreateTbBatchRsp(SCoder *pCoder, SVCreateTbBatchRsp *pRsp) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pRsp->nRsps) < 0) return -1; + pRsp->pRsps = (SVCreateTbRsp *)TCODER_MALLOC(pCoder, sizeof(*pRsp->pRsps) * pRsp->nRsps); + for (int32_t i = 0; i < pRsp->nRsps; i++) { + if (tDecodeSVCreateTbRsp(pCoder, pRsp->pRsps + i) < 0) return -1; + } + + tEndDecode(pCoder); return 0; } @@ -3779,6 +3616,38 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { taosMemoryFreeClear(pReq->ast); } +int tEncodeSVCreateStbReq(SCoder *pCoder, const SVCreateStbReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeCStr(pCoder, pReq->name) < 0) return -1; + if (tEncodeI64(pCoder, pReq->suid) < 0) return -1; + if (tEncodeI8(pCoder, pReq->rollup) < 0) return -1; + if (tEncodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1; + if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; + // if (pReq->rollup) { + // if (tEncodeSRSmaParam(pCoder, pReq->pRSmaParam) < 0) return -1; + // } + + tEndEncode(pCoder); + return 0; +} + +int tDecodeSVCreateStbReq(SCoder *pCoder, SVCreateStbReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->rollup) < 0) return -1; + if (tDecodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1; + if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; + // if (pReq->rollup) { + // if (tDecodeSRSmaParam(pCoder, pReq->pRSmaParam) < 0) return -1; + // } + + tEndDecode(pCoder); + return 0; +} + STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { STSchemaBuilder schemaBuilder = {0}; if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) { @@ -3802,3 +3671,97 @@ STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { tdDestroyTSchemaBuilder(&schemaBuilder); return pNSchema; } + +int tEncodeSVCreateTbReq(SCoder *pCoder, const SVCreateTbReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pReq->uid) < 0) return -1; + if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1; + + if (tEncodeCStr(pCoder, pReq->name) < 0) return -1; + if (tEncodeI32(pCoder, pReq->ttl) < 0) return -1; + if (tEncodeI8(pCoder, pReq->type) < 0) return -1; + + if (pReq->type == TSDB_CHILD_TABLE) { + if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1; + if (tEncodeBinary(pCoder, pReq->ctb.pTag, kvRowLen(pReq->ctb.pTag)) < 0) return -1; + } else if (pReq->type == TSDB_NORMAL_TABLE) { + if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schema) < 0) return -1; + } else { + ASSERT(0); + } + + tEndEncode(pCoder); + return 0; +} + +int tDecodeSVCreateTbReq(SCoder *pCoder, SVCreateTbReq *pReq) { + uint64_t len; + + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1; + + if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1; + if (tDecodeI32(pCoder, &pReq->ttl) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->type) < 0) return -1; + + if (pReq->type == TSDB_CHILD_TABLE) { + if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1; + if (tDecodeBinary(pCoder, &pReq->ctb.pTag, &len) < 0) return -1; + } else if (pReq->type == TSDB_NORMAL_TABLE) { + if (tDecodeSSchemaWrapper(pCoder, &pReq->ntb.schema) < 0) return -1; + } else { + ASSERT(0); + } + + tEndDecode(pCoder); + return 0; +} + +int tEncodeSVCreateTbBatchReq(SCoder *pCoder, const SVCreateTbBatchReq *pReq) { + int32_t nReq = taosArrayGetSize(pReq->pArray); + + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, nReq) < 0) return -1; + for (int iReq = 0; iReq < nReq; iReq++) { + if (tEncodeSVCreateTbReq(pCoder, (SVCreateTbReq *)taosArrayGet(pReq->pArray, iReq)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int tDecodeSVCreateTbBatchReq(SCoder *pCoder, SVCreateTbBatchReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pReq->nReqs) < 0) return -1; + pReq->pReqs = (SVCreateTbReq *)TCODER_MALLOC(pCoder, sizeof(SVCreateTbReq) * pReq->nReqs); + if (pReq->pReqs == NULL) return -1; + for (int iReq = 0; iReq < pReq->nReqs; iReq++) { + if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int tEncodeSVCreateTbRsp(SCoder *pCoder, const SVCreateTbRsp *pRsp) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32(pCoder, pRsp->code) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +int tDecodeSVCreateTbRsp(SCoder *pCoder, SVCreateTbRsp *pRsp) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32(pCoder, &pRsp->code) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} \ No newline at end of file diff --git a/source/common/test/tschemaTest.cpp b/source/common/test/tschemaTest.cpp deleted file mode 100644 index acced6e09e..0000000000 --- a/source/common/test/tschemaTest.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include -#include "tschema.h" - -TEST(td_schema_test, build_schema_test) { - -} \ No newline at end of file diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 5983ba92ac..6d9ead4c87 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -68,6 +68,7 @@ static void dmSetSignalHandle() { static int32_t dmParseArgs(int32_t argc, char const *argv[]) { int32_t cmdEnvIndex = 0; + if (argc < 2) return 0; global.envCmd = taosMemoryMalloc(argc-1); memset(global.envCmd, 0, argc-1); for (int32_t i = 1; i < argc; ++i) { diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index afe57e3d8f..ed9384a869 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -211,7 +211,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_MND_MQ_ASK_EP, mmProcessReadMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 7fc263c93c..2da98a41df 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -107,14 +107,13 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->vgId = pCreate->vgId; strcpy(pCfg->dbname, pCreate->db); - pCfg->wsize = pCreate->cacheBlockSize * 1024 * 1024; - pCfg->ssize = 1024; - pCfg->lsize = 1024 * 1024; + pCfg->szBuf = pCreate->cacheBlockSize * 1024 * 1024; pCfg->streamMode = pCreate->streamMode; pCfg->isWeak = true; - pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; - pCfg->tsdbCfg.keep0 = pCreate->daysToKeep2; - pCfg->tsdbCfg.keep1 = pCreate->daysToKeep0; + pCfg->tsdbCfg.days = 10; + pCfg->tsdbCfg.keep2 = 3650; // pCreate->daysToKeep0; + pCfg->tsdbCfg.keep0 = 3650; // pCreate->daysToKeep2; + pCfg->tsdbCfg.keep1 = 3650; // pCreate->daysToKeep0; pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; pCfg->tsdbCfg.retentions = pCreate->pRetensions; pCfg->walCfg.vgId = pCreate->vgId; @@ -212,7 +211,6 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return code; } - code = vmWriteVnodesToFile(pMgmt); if (code != 0) { tFreeSCreateVnodeReq(&createReq); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 36a8173adf..65cc4353e8 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -358,13 +358,11 @@ typedef struct { int32_t ttl; int32_t numOfColumns; int32_t numOfTags; - int32_t numOfSmas; int32_t commentLen; int32_t ast1Len; int32_t ast2Len; SSchema* pColumns; SSchema* pTags; - SSchema* pSmas; char* comment; char* pAst1; char* pAst2; @@ -440,14 +438,12 @@ static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) } typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - int64_t uid; - // TODO: use subDbUid + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; int64_t dbUid; - int64_t subDbUid; int32_t version; int8_t subType; // db or table int8_t withTbName; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index b9ec1905e6..5083104039 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -40,12 +40,12 @@ extern "C" { #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); -typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct SQWorkerMgmt SQHandle; @@ -84,32 +84,32 @@ typedef struct { int64_t timeseriesAllowed; } SGrantInfo; -struct SMnode { - int32_t selfId; - int64_t clusterId; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; - tmr_h timer; - tmr_h transTimer; - tmr_h mqTimer; - tmr_h telemTimer; - char *path; - int64_t checkTime; - SSdb *pSdb; - SMgmtWrapper *pWrapper; - SArray *pSteps; - SQHandle *pQuery; - SShowMgmt showMgmt; - SProfileMgmt profileMgmt; - STelemMgmt telemMgmt; - SSyncMgmt syncMgmt; - SHashObj *infosMeta; - SHashObj *perfsMeta; - SGrantInfo grant; - MndMsgFp msgFp[TDMT_MAX]; - SMsgCb msgCb; -}; +typedef struct SMnode { + int32_t selfId; + int64_t clusterId; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; + tmr_h timer; + tmr_h transTimer; + tmr_h mqTimer; + tmr_h telemTimer; + char *path; + int64_t checkTime; + SSdb *pSdb; + SMgmtWrapper *pWrapper; + SArray *pSteps; + SQHandle *pQuery; + SShowMgmt showMgmt; + SProfileMgmt profileMgmt; + STelemMgmt telemMgmt; + SSyncMgmt syncMgmt; + SHashObj *infosMeta; + SHashObj *perfsMeta; + SGrantInfo grant; + MndMsgFp msgFp[TDMT_MAX]; + SMsgCb msgCb; +} SMnode; void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); int64_t mndGenerateUid(char *name, int32_t len); diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 25bf20fa0d..cf4c41ee36 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -17,8 +17,8 @@ #include "mndAcct.h" #include "mndShow.h" -#define TSDB_ACCT_VER_NUMBER 1 -#define TSDB_ACCT_RESERVE_SIZE 128 +#define ACCT_VER_NUMBER 1 +#define ACCT_RESERVE_SIZE 128 static int32_t mndCreateDefaultAcct(SMnode *pMnode); static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct); @@ -55,6 +55,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) { acctObj.createdTime = taosGetTimestampMs(); acctObj.updateTime = acctObj.createdTime; acctObj.acctId = 1; + acctObj.status = 0; acctObj.cfg = (SAcctCfg){.maxUsers = INT32_MAX, .maxDbs = INT32_MAX, .maxStbs = INT32_MAX, @@ -79,7 +80,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) { static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, TSDB_ACCT_VER_NUMBER, sizeof(SAcctObj) + TSDB_ACCT_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, ACCT_VER_NUMBER, sizeof(SAcctObj) + ACCT_RESERVE_SIZE); if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; @@ -100,7 +101,7 @@ static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxTopics, _OVER) SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage, _OVER) SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState, _OVER) - SDB_SET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE, _OVER) + SDB_SET_RESERVE(pRaw, dataPos, ACCT_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; @@ -122,7 +123,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_ACCT_VER_NUMBER) { + if (sver != ACCT_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -151,7 +152,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pAcct->cfg.maxTopics, _OVER) SDB_GET_INT64(pRaw, dataPos, &pAcct->cfg.maxStorage, _OVER) SDB_GET_INT32(pRaw, dataPos, &pAcct->cfg.accessState, _OVER) - SDB_GET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE, _OVER) + SDB_GET_RESERVE(pRaw, dataPos, ACCT_RESERVE_SIZE, _OVER) terrno = 0; @@ -178,7 +179,6 @@ static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { mTrace("acct:%s, perform update action, old row:%p new row:%p", pOld->acct, pOld, pNew); - pOld->updateTime = pNew->updateTime; pOld->status = pNew->status; memcpy(&pOld->cfg, &pNew->cfg, sizeof(SAcctCfg)); @@ -186,19 +186,19 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { } static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq) { - terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq) { - terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessDropAcctReq(SNodeMsg *pReq) { - terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7f5df5d356..ff7a757007 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -59,7 +59,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessAskEpReq); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg); return sdbSetTable(pMnode->pSdb, table); @@ -86,7 +86,7 @@ static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) { mndTransDrop(pTrans); return 0; FAIL: - // TODO delete consumer + tDeleteSMqConsumerObj(pConsumerNew); mndTransDrop(pTrans); return -1; } @@ -197,11 +197,11 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { } static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { - SMnode *pMnode = pMsg->pNode; - SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; - SMqCMGetSubEpRsp rsp = {0}; - int64_t consumerId = be64toh(pReq->consumerId); - int32_t epoch = ntohl(pReq->epoch); + SMnode *pMnode = pMsg->pNode; + SMqAskEpReq *pReq = (SMqAskEpReq *)pMsg->rpcMsg.pCont; + SMqAskEpRsp rsp = {0}; + int64_t consumerId = be64toh(pReq->consumerId); + int32_t epoch = ntohl(pReq->epoch); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId); if (pConsumer == NULL) { @@ -300,7 +300,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { taosRUnLockLatch(&pConsumer->lock); } // encode rsp - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, &rsp); void *buf = rpcMallocCont(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -311,10 +311,10 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); + tEncodeSMqAskEpRsp(&abuf, &rsp); // release consumer and free memory - tDeleteSMqCMGetSubEpRsp(&rsp); + tDeleteSMqAskEpRsp(&rsp); mndReleaseConsumer(pMnode, pConsumer); // send rsp @@ -322,7 +322,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { pMsg->rspLen = tlen; return 0; FAIL: - tDeleteSMqCMGetSubEpRsp(&rsp); + tDeleteSMqAskEpRsp(&rsp); mndReleaseConsumer(pMnode, pConsumer); return -1; } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 800a013c4f..12de1f5bbc 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -215,7 +215,7 @@ SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) { if (pSubNew == NULL) return NULL; memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN); taosInitRWLatch(&pSubNew->lock); - pSubNew->vgNum = -1; + pSubNew->vgNum = 0; pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); // TODO set free fp SMqConsumerEpInSub epInSub = { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 5fa5182079..3eeec61dbb 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -363,7 +363,7 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) { pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; } mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, tsVersion); - terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; + terrno = TSDB_CODE_VERSION_NOT_COMPATIBLE; goto PROCESS_STATUS_MSG_OVER; } diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index a0ecbe9ae4..cf1cb34115 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -41,29 +41,31 @@ static const SPerfsTableSchema queriesSchema[] = { static const SPerfsTableSchema topicSchema[] = { {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - /*{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},*/ + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - /*{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},*/ + // TODO config }; static const SPerfsTableSchema consumerSchema[] = { - {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - // ep - // up time - // topics + {.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; -static const SPerfsTableSchema subscribeSchema[] = { +static const SPerfsTableSchema subscriptionSchema[] = { {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "committed_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "current_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, }; static const SPerfsTableMeta perfsMeta[] = { @@ -71,7 +73,7 @@ static const SPerfsTableMeta perfsMeta[] = { {TSDB_PERFS_TABLE_QUERIES, queriesSchema, tListLen(queriesSchema)}, {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)}, {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)}, - {TSDB_PERFS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)}, + {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)}, }; // connection/application/ diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 3dff65866c..a106cf348d 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -478,6 +478,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib SVgObj* pVgroup = NULL; SQueryPlan* pPlan = NULL; SSubplan* plan = NULL; + if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) { pPlan = qStringToQueryPlan(pTopic->physicalPlan); if (pPlan == NULL) { @@ -485,10 +486,6 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib return -1; } - ASSERT(pSub->vgNum == -1); - - pSub->vgNum = 0; - int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { qDestroyQueryPlan(pPlan); @@ -529,7 +526,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib pVgEp->vgId = pVgroup->vgId; taosArrayPush(pEpInSub->vgs, &pVgEp); - mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId); + mDebug("init subscription %s, assign vg: %d", pSub->key, pVgEp->vgId); if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) { int32_t msgLen; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 6332ff8662..3b6b264d1b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -72,8 +72,8 @@ void mndCleanupStb(SMnode *pMnode) {} SSdbRaw *mndStbActionEncode(SStbObj *pStb) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) + - + pStb->commentLen + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE; + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen + + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; @@ -91,7 +91,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER) @@ -112,14 +111,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } - for (int32_t i = 0; i < pStb->numOfSmas; ++i) { - SSchema *pSchema = &pStb->pSmas[i]; - SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) - SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) - } - if (pStb->commentLen > 0) { SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER) } @@ -178,15 +169,13 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER) pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema)); - pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema)); - if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) { + if (pStb->pColumns == NULL || pStb->pTags == NULL) { goto _OVER; } @@ -206,14 +195,6 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } - for (int32_t i = 0; i < pStb->numOfSmas; ++i) { - SSchema *pSchema = &pStb->pSmas[i]; - SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER) - SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) - SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) - } - if (pStb->commentLen > 0) { pStb->comment = taosMemoryCalloc(pStb->commentLen, 1); if (pStb->comment == NULL) goto _OVER; @@ -291,18 +272,6 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } } - if (pOld->numOfSmas < pNew->numOfSmas) { - void *pSmas = taosMemoryMalloc(pNew->numOfSmas * sizeof(SSchema)); - if (pSmas != NULL) { - taosMemoryFree(pOld->pSmas); - pOld->pSmas = pSmas; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); - taosWUnLockLatch(&pOld->lock); - } - } - if (pOld->commentLen < pNew->commentLen) { void *comment = taosMemoryMalloc(pNew->commentLen); if (comment != NULL) { @@ -393,72 +362,49 @@ static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSch } static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { - SName name = {0}; + SCoder coder = {0}; + int32_t contLen; + SName name = {0}; + SVCreateStbReq req = {0}; + tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFName); - SVCreateTbReq req = {0}; req.name = (char *)tNameGetTableName(&name); - req.ttl = 0; - req.keep = 0; - req.rollup = pStb->pAst1 > 0 ? 1 : 0; - req.type = TD_SUPER_TABLE; - req.stbCfg.suid = pStb->uid; - req.stbCfg.nCols = pStb->numOfColumns; - req.stbCfg.nTagCols = pStb->numOfTags; - req.stbCfg.pTagSchema = pStb->pTags; - req.stbCfg.nBSmaCols = pStb->numOfSmas; - req.stbCfg.pSchema = (SSchema *)taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); - if (req.stbCfg.pSchema == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } + req.suid = pStb->uid; + req.rollup = pStb->ast1Len > 0 ? 1 : 0; + req.schema.nCols = pStb->numOfColumns; + req.schema.sver = 0; + req.schema.pSchema = pStb->pColumns; + req.schemaTag.nCols = pStb->numOfTags; + req.schemaTag.pSchema = pStb->pTags; - memcpy(req.stbCfg.pSchema, pStb->pColumns, sizeof(SSchema) * pStb->numOfColumns); - for (int i = 0; i < pStb->numOfColumns; i++) { - req.stbCfg.pSchema[i].flags = SCHEMA_SMA_ON; - } - - SRSmaParam *pRSmaParam = NULL; if (req.rollup) { - pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam)); - if (pRSmaParam == NULL) { - taosMemoryFreeClear(req.stbCfg.pSchema); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pRSmaParam->xFilesFactor = pStb->xFilesFactor; - pRSmaParam->delay = pStb->delay; + req.pRSmaParam.xFilesFactor = pStb->xFilesFactor; + req.pRSmaParam.delay = pStb->delay; if (pStb->ast1Len > 0) { - if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(req.stbCfg.pRSmaParam); - taosMemoryFreeClear(req.stbCfg.pSchema); + if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len) != TSDB_CODE_SUCCESS) { return NULL; } } if (pStb->ast2Len > 0) { - if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pRSmaParam->qmsg1); - taosMemoryFreeClear(req.stbCfg.pRSmaParam); - taosMemoryFreeClear(req.stbCfg.pSchema); + if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len) != TSDB_CODE_SUCCESS) { return NULL; } } - - req.stbCfg.pRSmaParam = pRSmaParam; + } + // get length + if (tEncodeSize(tEncodeSVCreateStbReq, &req, contLen) < 0) { + return NULL; } - int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + contLen += sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { - if (pRSmaParam) { - taosMemoryFreeClear(pRSmaParam->qmsg1); - taosMemoryFreeClear(pRSmaParam->qmsg2); - taosMemoryFreeClear(pRSmaParam); - } - taosMemoryFreeClear(req.stbCfg.pSchema); + taosMemoryFreeClear(req.pRSmaParam.qmsg1); + taosMemoryFreeClear(req.pRSmaParam.qmsg2); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -467,15 +413,15 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt pHead->vgId = htonl(pVgroup->vgId); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); - tSerializeSVCreateTbReq(&pBuf, &req); + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, contLen - sizeof(SMsgHead), TD_ENCODER); + if (tEncodeSVCreateStbReq(&coder, &req) < 0) { + return NULL; + } + tCoderClear(&coder); *pContLen = contLen; - if (pRSmaParam) { - taosMemoryFreeClear(pRSmaParam->qmsg1); - taosMemoryFreeClear(pRSmaParam->qmsg2); - taosMemoryFreeClear(pRSmaParam); - } - taosMemoryFreeClear(req.stbCfg.pSchema); + taosMemoryFreeClear(req.pRSmaParam.qmsg1); + taosMemoryFreeClear(req.pRSmaParam.qmsg2); return pHead; } @@ -692,7 +638,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.ttl = pCreate->ttl; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; - stbObj.numOfSmas = pCreate->numOfSmas; stbObj.commentLen = pCreate->commentLen; if (stbObj.commentLen > 0) { stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1); @@ -725,8 +670,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema)); stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema)); - stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema)); - if (stbObj.pColumns == NULL || stbObj.pTags == NULL || stbObj.pSmas == NULL) { + if (stbObj.pColumns == NULL || stbObj.pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -736,6 +680,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre SSchema *pSchema = &stbObj.pColumns[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; + pSchema->flags = pField->flags; memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); pSchema->colId = stbObj.nextColId; stbObj.nextColId++; @@ -751,18 +696,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.nextColId++; } - for (int32_t i = 0; i < stbObj.numOfSmas; ++i) { - SField *pField = taosArrayGet(pCreate->pSmas, i); - SSchema *pSchema = &stbObj.pSmas[i]; - SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name); - if (pColSchema == NULL) { - mError("stb:%s, sma:%s not found in columns", stbObj.name, pField->name); - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; - } - memcpy(pSchema, pColSchema, sizeof(SSchema)); - } - int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg); if (pTrans == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 15a70e3311..3dbe3241a7 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -22,13 +22,15 @@ static int32_t mndInitWal(SMnode *pMnode) { char path[PATH_MAX] = {0}; snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); - SWalCfg cfg = {.vgId = 1, - .fsyncPeriod = 0, - .rollPeriod = -1, - .segSize = -1, - .retentionPeriod = -1, - .retentionSize = -1, - .level = TAOS_WAL_FSYNC}; + SWalCfg cfg = { + .vgId = 1, + .fsyncPeriod = 0, + .rollPeriod = -1, + .segSize = -1, + .retentionPeriod = -1, + .retentionSize = -1, + .level = TAOS_WAL_FSYNC, + }; pMgmt->pWal = walOpen(path, &cfg); if (pMgmt->pWal == NULL) return -1; @@ -54,62 +56,62 @@ static int32_t mndRestoreWal(SMnode *pMnode) { int64_t first = walGetFirstVer(pWal); int64_t last = walGetLastVer(pWal); - mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); + mDebug("start to restore wal, sdbver:%" PRId64 ", first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); first = TMAX(lastSdbVer + 1, first); for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) { if (walReadWithHandle(pHandle, ver) < 0) { - mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver); - goto WAL_RESTORE_OVER; + mError("ver:%" PRId64 ", failed to read from wal since %s", ver, terrstr()); + goto _OVER; } SWalHead *pHead = pHandle->pHead; int64_t sdbVer = sdbUpdateVer(pSdb, 0); if (sdbVer + 1 != ver) { terrno = TSDB_CODE_SDB_INVALID_WAl_VER; - mError("failed to read wal from sdb, sdbVer:%" PRId64 " inconsistent with ver:%" PRId64, sdbVer, ver); - goto WAL_RESTORE_OVER; + mError("ver:%" PRId64 ", failed to write to sdb, since inconsistent with sdbver:%" PRId64, ver, sdbVer); + goto _OVER; } - mTrace("wal:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body); - if (sdbWriteNotFree(pSdb, (void *)pHead->head.body) < 0) { - mError("failed to read wal from sdb since %s, ver:%" PRId64, terrstr(), ver); - goto WAL_RESTORE_OVER; + mTrace("ver:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body); + if (sdbWriteWithoutFree(pSdb, (void *)pHead->head.body) < 0) { + mError("ver:%" PRId64 ", failed to write to sdb since %s", ver, terrstr()); + goto _OVER; } sdbUpdateVer(pSdb, 1); - mDebug("wal:%" PRId64 ", is restored", ver); + mDebug("ver:%" PRId64 ", is restored", ver); } int64_t sdbVer = sdbUpdateVer(pSdb, 0); - mDebug("restore sdb wal finished, sdb ver:%" PRId64, sdbVer); + mDebug("restore wal finished, sdbver:%" PRId64, sdbVer); mndTransPullup(pMnode); sdbVer = sdbUpdateVer(pSdb, 0); - mDebug("pullup trans finished, sdb ver:%" PRId64, sdbVer); + mDebug("pullup trans finished, sdbver:%" PRId64, sdbVer); if (sdbVer != lastSdbVer) { mInfo("sdb restored from %" PRId64 " to %" PRId64 ", write file", lastSdbVer, sdbVer); if (sdbWriteFile(pSdb) != 0) { - goto WAL_RESTORE_OVER; + goto _OVER; } if (walCommit(pWal, sdbVer) != 0) { - goto WAL_RESTORE_OVER; + goto _OVER; } if (walBeginSnapshot(pWal, sdbVer) < 0) { - goto WAL_RESTORE_OVER; + goto _OVER; } if (walEndSnapshot(pWal) < 0) { - goto WAL_RESTORE_OVER; + goto _OVER; } } code = 0; -WAL_RESTORE_OVER: +_OVER: walCloseReadHandle(pHandle); return code; } @@ -158,11 +160,11 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) { int64_t ver = sdbUpdateVer(pSdb, 1); if (walWrite(pWal, ver, 1, pRaw, sdbGetRawTotalSize(pRaw)) < 0) { sdbUpdateVer(pSdb, -1); - mError("failed to write raw:%p since %s, ver:%" PRId64, pRaw, terrstr(), ver); + mError("ver:%" PRId64 ", failed to write raw:%p to wal since %s", ver, pRaw, terrstr()); return -1; } - mTrace("raw:%p, write to wal, ver:%" PRId64, pRaw, ver); + mTrace("ver:%" PRId64 ", write to wal, raw:%p", ver, pRaw); walCommit(pWal, ver); walFsync(pWal, true); diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index e445022548..e5067c7027 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -146,7 +146,6 @@ int32_t mndInitTelem(SMnode* pMnode) { taosGetEmail(pMgmt->email, sizeof(pMgmt->email)); mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); - mDebug("mnode telemetry is initialized"); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 7c4e51298f..886ba028de 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -76,7 +76,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER); - SDB_SET_INT64(pRaw, dataPos, pTopic->subDbUid, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER); @@ -139,7 +138,6 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER); - SDB_GET_INT64(pRaw, dataPos, &pTopic->subDbUid, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER); @@ -520,29 +518,33 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic); if (pShow->pIter == NULL) break; - int32_t cols = 0; + SColumnInfoData *pColInfo; + SName n; + int32_t cols = 0; char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - - SName n; - tNameFromString(&n, pTopic->name, T_NAME_ACCT|T_NAME_DB); + tNameFromString(&n, pTopic->name, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&n, varDataVal(topicName)); varDataSetLen(topicName, strlen(varDataVal(topicName))); - - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); + char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&n, varDataVal(dbName)); + varDataSetLen(dbName, strlen(varDataVal(dbName))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)dbName, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN); varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); -// taosMemoryFree(sql); - numOfRows++; sdbRelease(pSdb, pTopic); } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6e221fb3df..582b60f7b2 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -844,7 +844,7 @@ static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { for (int32_t i = 0; i < arraySize; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); - int32_t code = sdbWriteNotFree(pSdb, pRaw); + int32_t code = sdbWriteWithoutFree(pSdb, pRaw); if (code != 0) { return code; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 054bff466c..d6876782f6 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -21,8 +21,8 @@ #include "mndTrans.h" #include "tbase64.h" -#define TSDB_USER_VER_NUMBER 1 -#define TSDB_USER_RESERVE_SIZE 64 +#define USER_VER_NUMBER 1 +#define USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); static SSdbRaw *mndUserActionEncode(SUserObj *pUser); @@ -35,7 +35,7 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq); static int32_t mndProcessAlterUserReq(SNodeMsg *pReq); static int32_t mndProcessDropUserReq(SNodeMsg *pReq); static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq); -static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); int32_t mndInitUser(SMnode *pMnode) { @@ -93,9 +93,9 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); - int32_t size = sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN; + int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN; - SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, size); + SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) goto USER_ENCODE_OVER; int32_t dataPos = 0; @@ -120,7 +120,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { db = taosHashIterate(pUser->writeDbs, db); } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, USER_ENCODE_OVER) terrno = 0; @@ -142,7 +142,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto USER_DECODE_OVER; - if (sver != TSDB_USER_VER_NUMBER) { + if (sver != USER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto USER_DECODE_OVER; } @@ -184,7 +184,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN); } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_DECODE_OVER) terrno = 0; @@ -639,7 +639,7 @@ GET_AUTH_OVER: return code; } -static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -652,29 +652,29 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pB if (pShow->pIter == NULL) break; cols = 0; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->bytes[cols]); - colDataAppend(pColInfo, numOfRows, (const char*) name, false); + colDataAppend(pColInfo, numOfRows, (const char *)name, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - const char* src = pUser->superUser? "super":"normal"; - char b[10+VARSTR_HEADER_SIZE] = {0}; + const char *src = pUser->superUser ? "super" : "normal"; + char b[10 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src)); - colDataAppend(pColInfo, numOfRows, (const char*) b, false); + colDataAppend(pColInfo, numOfRows, (const char *)b, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataAppend(pColInfo, numOfRows, (const char*) &pUser->createdTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->acct, pShow->bytes[cols]); - colDataAppend(pColInfo, numOfRows, (const char*) name, false); + colDataAppend(pColInfo, numOfRows, (const char *)name, false); numOfRows++; sdbRelease(pSdb, pUser); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 75caef2336..41a309e941 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -368,7 +368,7 @@ int32_t mndProcessMsg(SNodeMsg *pMsg) { } if (isReq && (pRpc->contLen == 0 || pRpc->pCont == NULL)) { - terrno = TSDB_CODE_MND_INVALID_MSG_LEN; + terrno = TSDB_CODE_INVALID_MSG_LEN; mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); return -1; } diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 0260b5f59e..b143594ec0 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -32,7 +32,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); } TEST_F(MndTestAcct, 02_Alter_Acct) { @@ -42,7 +42,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); } TEST_F(MndTestAcct, 03_Drop_Acct) { @@ -52,5 +52,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); } diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index 0d41d5de20..54a154e684 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -114,18 +114,15 @@ void* MndTestSma::BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen) SMCreateStbReq createReq = {0}; createReq.numOfColumns = 3; createReq.numOfTags = 1; - createReq.numOfSmas = 1; createReq.igExists = 0; createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); - createReq.pSmas = taosArrayInit(createReq.numOfSmas, sizeof(SField)); strcpy(createReq.name, stbname); PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts"); PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1"); PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2"); PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1"); - PushField(createReq.pSmas, 2, TSDB_DATA_TYPE_TINYINT, "col1"); int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); void* pHead = rpcMallocCont(tlen); @@ -190,7 +187,7 @@ void* MndTestSma::BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int3 } TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) { - #if 0 +#if 0 const char* dbname = "1.d1"; const char* stbname = "1.d1.stb"; const char* smaname = "1.d1.sma"; @@ -244,7 +241,7 @@ TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) { test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); } -#endif +#endif } TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { @@ -264,7 +261,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { pReq = BuildCreateBSmaStbReq(stbname, &contLen); pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -280,7 +277,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { pReq = BuildDropStbReq(stbname, &contLen); pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index c645dae5b5..bc878d7b31 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -162,7 +162,4 @@ static int32_t sdbCreateDir(SSdb *pSdb) { return 0; } -int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) { - pSdb->curVer += val; - return pSdb->curVer; -} \ No newline at end of file +int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) { return atomic_add_fetch_64(&pSdb->curVer, val); } \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index f61899766e..98ac04684e 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -28,7 +28,7 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) { if (fp == NULL) continue; if ((*fp)(pSdb->pMnode) != 0) { - mError("failed to deploy sdb:%d since %s", i, terrstr()); + mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr()); return -1; } } @@ -202,7 +202,7 @@ int32_t sdbReadFile(SSdb *pSdb) { break; } - code = sdbWriteNotFree(pSdb, pRaw); + code = sdbWriteWithoutFree(pSdb, pRaw); if (code != 0) { mError("failed to read file:%s since %s", file, terrstr()); goto PARSE_SDB_DATA_ERROR; @@ -263,7 +263,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { continue; } - sdbPrintOper(pSdb, pRow, "writeFile"); + sdbPrintOper(pSdb, pRow, "write"); SSdbRaw *pRaw = (*encodeFp)(pRow->pObj); if (pRaw != NULL) { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index dc2c12a2c4..1158bedc21 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -51,7 +51,9 @@ const char *sdbTableName(ESdbType type) { case SDB_TOPIC: return "topic"; case SDB_VGROUP: - return "vgId"; + return "vgroup"; + case SDB_SMA: + return "sma"; case SDB_STB: return "stb"; case SDB_DB: @@ -86,13 +88,13 @@ void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) { EKeyType keyType = pSdb->keyTypes[pRow->type]; if (keyType == SDB_KEY_BINARY) { - mTrace("%s:%s, refCount:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), (char *)pRow->pObj, pRow->refCount, - oper, pRow->pObj, sdbStatusStr(pRow->status)); + mTrace("%s:%s, ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), (char *)pRow->pObj, pRow->refCount, oper, + pRow->pObj, sdbStatusStr(pRow->status)); } else if (keyType == SDB_KEY_INT32) { - mTrace("%s:%d, refCount:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, - pRow->refCount, oper, pRow->pObj, sdbStatusStr(pRow->status)); + mTrace("%s:%d, ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, pRow->refCount, + oper, pRow->pObj, sdbStatusStr(pRow->status)); } else if (keyType == SDB_KEY_INT64) { - mTrace("%s:%" PRId64 ", refCount:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, + mTrace("%s:%" PRId64 ", ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, pRow->refCount, oper, pRow->pObj, sdbStatusStr(pRow->status)); } else { } @@ -142,7 +144,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * pRow->refCount = 0; pRow->status = pRaw->status; - sdbPrintOper(pSdb, pRow, "insertRow"); + sdbPrintOper(pSdb, pRow, "insert"); if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { taosWUnLockLatch(pLock); @@ -191,7 +193,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SSdbRow *pOldRow = *ppOldRow; pOldRow->status = pRaw->status; - sdbPrintOper(pSdb, pOldRow, "updateRow"); + sdbPrintOper(pSdb, pOldRow, "update"); taosRUnLockLatch(pLock); int32_t code = 0; @@ -220,7 +222,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SSdbRow *pOldRow = *ppOldRow; pOldRow->status = pRaw->status; - sdbPrintOper(pSdb, pOldRow, "deleteRow"); + sdbPrintOper(pSdb, pOldRow, "delete"); taosHashRemove(hash, pOldRow->pObj, keySize); taosWUnLockLatch(pLock); @@ -233,7 +235,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * return 0; } -int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) { +int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) { SHashObj *hash = sdbGetHash(pSdb, pRaw->type); if (hash == NULL) return terrno; @@ -266,7 +268,7 @@ int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) { } int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw) { - int32_t code = sdbWriteNotFree(pSdb, pRaw); + int32_t code = sdbWriteWithoutFree(pSdb, pRaw); sdbFreeRaw(pRaw); return code; } @@ -296,7 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { case SDB_STATUS_UPDATING: atomic_add_fetch_32(&pRow->refCount, 1); pRet = pRow->pObj; - sdbPrintOper(pSdb, pRow, "acquireRow"); + sdbPrintOper(pSdb, pRow, "acquire"); break; case SDB_STATUS_CREATING: terrno = TSDB_CODE_SDB_OBJ_CREATING; @@ -318,7 +320,7 @@ static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) { taosRLockLatch(pLock); int32_t ref = atomic_load_32(&pRow->refCount); - sdbPrintOper(pSdb, pRow, "checkRow"); + sdbPrintOper(pSdb, pRow, "check"); if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { sdbFreeRow(pSdb, pRow); } @@ -330,13 +332,13 @@ void sdbRelease(SSdb *pSdb, void *pObj) { if (pObj == NULL) return; SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow)); - if (pRow->type >= SDB_MAX ) return; + if (pRow->type >= SDB_MAX) return; SRWLatch *pLock = &pSdb->locks[pRow->type]; taosRLockLatch(pLock); int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); - sdbPrintOper(pSdb, pRow, "releaseRow"); + sdbPrintOper(pSdb, pRow, "release"); if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { sdbFreeRow(pSdb, pRow); } @@ -372,7 +374,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { } atomic_add_fetch_32(&pRow->refCount, 1); - sdbPrintOper(pSdb, pRow, "fetchRow"); + sdbPrintOper(pSdb, pRow, "fetch"); *ppObj = pRow->pObj; break; } diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index c3aaf562be..326fe53fc7 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -27,7 +27,7 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { pRaw->sver = sver; pRaw->dataLen = dataLen; - mTrace("raw:%p, is created, len:%d", pRaw, dataLen); + mTrace("raw:%p, is created, len:%d table:%s", pRaw, dataLen, sdbTableName(type)); return pRaw; } diff --git a/source/dnode/mnode/sdb/src/sdbRow.c b/source/dnode/mnode/sdb/src/sdbRow.c index ac86a72155..94f87cb350 100644 --- a/source/dnode/mnode/sdb/src/sdbRow.c +++ b/source/dnode/mnode/sdb/src/sdbRow.c @@ -43,7 +43,7 @@ void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) { (*deleteFp)(pSdb, pRow->pObj); } - sdbPrintOper(pSdb, pRow, "freeRow"); + sdbPrintOper(pSdb, pRow, "free"); mTrace("row:%p, is freed", pRow->pObj); taosMemoryFreeClear(pRow); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 7ebfc6e230..2a784c112e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -5,9 +5,7 @@ target_sources( PRIVATE # vnode "src/vnd/vnodeOpen.c" - "src/vnd/vnodeArenaMAImpl.c" - "src/vnd/vnodeBufferPool.c" - # "src/vnd/vnodeBufferPool2.c" + "src/vnd/vnodeBufPool.c" "src/vnd/vnodeCfg.c" "src/vnd/vnodeCommit.c" "src/vnd/vnodeInt.c" @@ -21,18 +19,19 @@ target_sources( "src/meta/metaOpen.c" "src/meta/metaIdx.c" "src/meta/metaTable.c" - "src/meta/metaTDBImpl.c" - # "src/meta/metaBDBImpl.c" + "src/meta/metaQuery.c" + "src/meta/metaCommit.c" + "src/meta/metaEntry.c" # tsdb "src/tsdb/tsdbTDBImpl.c" "src/tsdb/tsdbCommit.c" + "src/tsdb/tsdbCommit2.c" "src/tsdb/tsdbCompact.c" "src/tsdb/tsdbFile.c" "src/tsdb/tsdbFS.c" "src/tsdb/tsdbMain.c" "src/tsdb/tsdbMemTable.c" - "src/tsdb/tsdbOptions.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbScan.c" @@ -71,6 +70,7 @@ target_link_libraries( PUBLIC transport PUBLIC stream ) +target_compile_definitions(vnode PUBLIC -DMETA_REFACT) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 9eab4d3376..3314d2f264 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -32,6 +32,8 @@ #include "tmsg.h" #include "trow.h" +#include "tdbInt.h" + #ifdef __cplusplus extern "C" { #endif @@ -59,7 +61,7 @@ int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg); int32_t vnodeCompact(SVnode *pVnode); int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); -int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName); +int vnodeValidateTableHash(SVnode *pVnode, char *tableFName); int32_t vnodeStart(SVnode *pVnode); void vnodeStop(SVnode *pVnode); @@ -68,28 +70,36 @@ int64_t vnodeGetSyncHandle(SVnode *pVnode); void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); // meta -typedef struct SMeta SMeta; // todo: remove -typedef struct SMTbCursor SMTbCursor; +typedef struct SMeta SMeta; // todo: remove +typedef struct SMetaReader SMetaReader; +typedef struct SMetaEntry SMetaEntry; +void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void metaReaderClear(SMetaReader *pReader); +int metaReadNext(SMetaReader *pReader); + +#if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; typedef SVCreateTSmaReq SSmaCfg; +typedef struct SMTbCursor SMTbCursor; + SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); -char *metaTbCursorNext(SMTbCursor *pTbCur); +int metaTbCursorNext(SMTbCursor *pTbCur); +#endif // tsdb -typedef struct STsdb STsdb; -typedef struct STsdbQueryCond STsdbQueryCond; -typedef void *tsdbReaderT; +typedef struct STsdb STsdb; +typedef void *tsdbReaderT; #define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 #define BLOCK_LOAD_TABLE_SEQ_ORDER 2 #define BLOCK_LOAD_TABLE_RR_ORDER 3 -tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, +tsdbReaderT *tsdbQueryTables(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); -tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, +tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, void *pMemRef); int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo); bool isTsdbCacheLastRow(tsdbReaderT *pReader); @@ -101,6 +111,7 @@ bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis); SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList); +void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond); void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); @@ -130,6 +141,7 @@ struct STsdbCfg { int8_t precision; int8_t update; int8_t compression; + int8_t slLevel; int32_t days; int32_t minRows; int32_t maxRows; @@ -144,10 +156,10 @@ struct SVnodeCfg { int32_t vgId; char dbname[TSDB_DB_NAME_LEN]; uint64_t dbId; - uint64_t wsize; - uint64_t ssize; - uint64_t lsize; - bool isHeapAllocator; + int32_t szPage; + int32_t szCache; + uint64_t szBuf; + bool isHeap; uint32_t ttl; uint32_t keep; int8_t streamMode; @@ -160,20 +172,53 @@ struct SVnodeCfg { int8_t hashMethod; }; -struct STsdbQueryCond { - STimeWindow twindow; - int32_t order; // desc|asc order to iterate the data block - int32_t numOfCols; - SColumnInfo *colList; - bool loadExternalRows; // load external rows or not - int32_t type; // data block load type: -}; - typedef struct { TSKEY lastKey; uint64_t uid; } STableKeyInfo; +struct SMetaEntry { + int64_t version; + int8_t type; + tb_uid_t uid; + const char *name; + union { + struct { + SSchemaWrapper schema; + SSchemaWrapper schemaTag; + } stbEntry; + struct { + int64_t ctime; + int32_t ttlDays; + tb_uid_t suid; + const void *pTags; + } ctbEntry; + struct { + int64_t ctime; + int32_t ttlDays; + SSchemaWrapper schema; + } ntbEntry; + }; +}; + +struct SMetaReader { + int32_t flags; + SMeta *pMeta; + SCoder coder; + SMetaEntry me; + void *pBuf; + int szBuf; +}; + +struct SMTbCursor { + TDBC *pDbc; + void *pKey; + void *pVal; + int kLen; + int vLen; + SMetaReader mr; +}; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 7647929246..9a0b1f4578 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -16,13 +16,14 @@ #ifndef _TD_VNODE_META_H_ #define _TD_VNODE_META_H_ +#include "vnodeInt.h" + #ifdef __cplusplus extern "C" { #endif typedef struct SMetaIdx SMetaIdx; typedef struct SMetaDB SMetaDB; -typedef struct SMCtbCursor SMCtbCursor; typedef struct SMSmaCursor SMSmaCursor; // metaDebug ================== @@ -36,8 +37,16 @@ typedef struct SMSmaCursor SMSmaCursor; // clang-format on // metaOpen ================== -int metaOpen(SVnode* pVnode, SMeta** ppMeta); -int metaClose(SMeta* pMeta); + +// metaEntry ================== +int metaEncodeEntry(SCoder* pCoder, const SMetaEntry* pME); +int metaDecodeEntry(SCoder* pCoder, SMetaEntry* pME); + +// metaTable ================== +int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); + +// metaQuery ================== +int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid); // metaIdx ================== int metaOpenIdx(SMeta* pMeta); @@ -45,32 +54,59 @@ void metaCloseIdx(SMeta* pMeta); int metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions); int metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid); +// metaCommit ================== static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); } -#define META_SUPER_TABLE TD_SUPER_TABLE -#define META_CHILD_TABLE TD_CHILD_TABLE -#define META_NORMAL_TABLE TD_NORMAL_TABLE +struct SMeta { + char* path; + SVnode* pVnode; + TENV* pEnv; + TXN txn; + TDB* pTbDb; + TDB* pSkmDb; + TDB* pUidIdx; + TDB* pNameIdx; + TDB* pCtbIdx; + TDB* pTagIdx; + TDB* pTtlIdx; + SMetaIdx* pIdx; +}; -int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaDropTable(SMeta* pMeta, tb_uid_t uid); -int metaCommit(SMeta* pMeta); -int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); -int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); -STbCfg* metaGetTbInfoByUid(SMeta* pMeta, tb_uid_t uid); -STbCfg* metaGetTbInfoByName(SMeta* pMeta, char* tbname, tb_uid_t* uid); -SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); -STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); -void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode); -STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid); -SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup); -int metaGetTbNum(SMeta* pMeta); -SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); -void metaCloseSmaCursor(SMSmaCursor* pSmaCur); -int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); -SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); -void metaCloseCtbCurosr(SMCtbCursor* pCtbCur); -tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); +typedef struct { + int64_t version; + tb_uid_t uid; +} STbDbKey; +typedef struct __attribute__((__packed__)) { + tb_uid_t uid; + int32_t sver; +} SSkmDbKey; + +typedef struct { + tb_uid_t suid; + tb_uid_t uid; +} SCtbIdxKey; + +typedef struct __attribute__((__packed__)) { + tb_uid_t suid; + int16_t cid; + char data[]; +} STagIdxKey; + +typedef struct { + int64_t dtime; + tb_uid_t uid; +} STtlIdxKey; + +#if 1 + +// int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); +int metaDropTable(SMeta* pMeta, tb_uid_t uid); +SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); +void metaCloseSmaCursor(SMSmaCursor* pSmaCur); +int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); + +#ifndef META_REFACT // SMetaDB int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); @@ -78,17 +114,9 @@ int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); +#endif -// SMetaIdx - -tb_uid_t metaGenerateUid(SMeta* pMeta); - -struct SMeta { - char* path; - SVnode* pVnode; - SMetaDB* pDB; - SMetaIdx* pIdx; -}; +#endif #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 347d28f1ea..b0461067e1 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -16,6 +16,8 @@ #ifndef _TD_VNODE_TQ_H_ #define _TD_VNODE_TQ_H_ +#include "vnodeInt.h" + #include "executor.h" #include "os.h" #include "tcache.h" @@ -237,17 +239,7 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); -void tqClose(STQ*); // required by vnode -int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); -int tqCommit(STQ*); - -int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); -int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); -int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**); int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 8af3068b8d..3349accb88 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -16,6 +16,8 @@ #ifndef _TD_VNODE_TSDB_H_ #define _TD_VNODE_TSDB_H_ +#include "vnodeInt.h" + #ifdef __cplusplus extern "C" { #endif @@ -30,29 +32,38 @@ extern "C" { #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +// tsdbMemTable ================ +typedef struct STbData STbData; +typedef struct STsdbMemTable STsdbMemTable; +typedef struct SMergeInfo SMergeInfo; +typedef struct STable STable; + +int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable); +void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable); +int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows); +int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, + TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); + +// tsdbCommit ================ + +#if 1 + typedef struct SSmaStat SSmaStat; typedef struct SSmaEnv SSmaEnv; typedef struct SSmaEnvs SSmaEnvs; -typedef struct STable { +struct STable { uint64_t tid; uint64_t uid; STSchema *pSchema; -} STable; +}; #define TABLE_TID(t) (t)->tid #define TABLE_UID(t) (t)->uid -STsdb *tsdbOpen(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); -void tsdbClose(STsdb *); -int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp); int tsdbPrepareCommit(STsdb *pTsdb); -int tsdbCommit(STsdb *pTsdb); int32_t tsdbInitSma(STsdb *pTsdb); -int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg); int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg); -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version); -int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg); int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); void tsdbCleanupReadHandle(tsdbReaderT queryHandle); @@ -108,25 +119,28 @@ typedef struct { TSKEY minKey; } SRtn; -typedef struct STbData { +struct STbData { tb_uid_t uid; TSKEY keyMin; TSKEY keyMax; + int64_t minVer; + int64_t maxVer; int64_t nrows; SSkipList *pData; -} STbData; +}; -typedef struct STsdbMemTable { +struct STsdbMemTable { + SVBufPool *pPool; T_REF_DECLARE() - SRWLatch latch; - TSKEY keyMin; - TSKEY keyMax; - uint64_t nRow; - SMemAllocator *pMA; - // Container + SRWLatch latch; + TSKEY keyMin; + TSKEY keyMax; + int64_t minVer; + int64_t maxVer; + int64_t nRow; SSkipList *pSlIdx; // SSkiplist SHashObj *pHashIdx; -} STsdbMemTable; +}; typedef struct { uint32_t version; // Commit version from 0 to increase @@ -152,18 +166,17 @@ typedef struct { } STsdbFS; struct STsdb { - int32_t vgId; - SVnode *pVnode; - bool repoLocked; - TdThreadMutex mutex; - char *path; - STsdbCfg config; - STsdbMemTable *mem; - STsdbMemTable *imem; - SRtn rtn; - SMemAllocatorFactory *pmaf; - STsdbFS *fs; - SSmaEnvs smaEnvs; + char *path; + SVnode *pVnode; + int32_t vgId; + bool repoLocked; + TdThreadMutex mutex; + STsdbCfg config; + STsdbMemTable *mem; + STsdbMemTable *imem; + SRtn rtn; + STsdbFS *fs; + SSmaEnvs smaEnvs; }; #define REPO_ID(r) ((r)->vgId) @@ -185,7 +198,7 @@ static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, } // tsdbMemTable.h -typedef struct { +struct SMergeInfo { int rowsInserted; int rowsUpdated; int rowsDeleteSucceed; @@ -193,7 +206,7 @@ typedef struct { int nOperations; TSKEY keyFirst; TSKEY keyLast; -} SMergeInfo; +}; static void *taosTMalloc(size_t size); static void *taosTCalloc(size_t nmemb, size_t size); @@ -202,12 +215,6 @@ static void *taosTZfree(void *ptr); static size_t taosTSizeof(void *ptr); static void taosTMemset(void *ptr, int c); -STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb); -void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp); -int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); - static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) { if (pIter == NULL) return NULL; @@ -224,12 +231,6 @@ static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { return TD_ROW_KEY(row); } -// tsdbOptions -extern const STsdbCfg defautlTsdbOptions; - -int tsdbValidateOptions(const STsdbCfg *); -void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc); - // tsdbReadImpl typedef struct SReadH SReadH; @@ -987,6 +988,8 @@ static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void return len; } +#endif + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index e20d2989b9..8fb18ddfea 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -16,9 +16,7 @@ #ifndef _TD_VNODE_TSDB_SMA_H_ #define _TD_VNODE_TSDB_SMA_H_ -#include "os.h" -#include "thash.h" -#include "tmsg.h" +#include "tsdb.h" #ifdef __cplusplus extern "C" { @@ -32,12 +30,6 @@ struct STbDdlH { __tb_ddl_fn_t fp; }; -typedef struct { - tb_uid_t suid; - SArray *tbUids; - SHashObj *uidHash; -} STbUidStore; - static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) { ASSERT(*pStore == NULL); *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index fa3cf65e60..3f5435ee47 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -16,6 +16,10 @@ #ifndef _TD_VND_H_ #define _TD_VND_H_ +#include "sync.h" +#include "syncTools.h" +#include "vnodeInt.h" + #ifdef __cplusplus extern "C" { #endif @@ -31,69 +35,65 @@ extern "C" { // clang-format on // vnodeCfg ==================== +extern const SVnodeCfg vnodeCfgDefault; + +int vnodeCheckCfg(const SVnodeCfg*); int vnodeEncodeConfig(const void* pObj, SJson* pJson); int vnodeDecodeConfig(const SJson* pJson, void* pObj); // vnodeModule ==================== int vnodeScheduleTask(int (*execute)(void*), void* arg); +// vnodeBufPool ==================== +typedef struct SVBufPoolNode SVBufPoolNode; +struct SVBufPoolNode { + SVBufPoolNode* prev; + SVBufPoolNode** pnext; + int64_t size; + uint8_t data[]; +}; + +struct SVBufPool { + SVBufPool* next; + int64_t nRef; + int64_t size; + uint8_t* ptr; + SVBufPoolNode* pTail; + SVBufPoolNode node; +}; + +int vnodeOpenBufPool(SVnode* pVnode, int64_t size); +int vnodeCloseBufPool(SVnode* pVnode); +void vnodeBufPoolReset(SVBufPool* pPool); + // vnodeQuery ==================== int vnodeQueryOpen(SVnode* pVnode); void vnodeQueryClose(SVnode* pVnode); int vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg); // vnodeCommit ==================== +int vnodeBegin(SVnode* pVnode); +int vnodeShouldCommit(SVnode* pVnode); +int vnodeCommit(SVnode* pVnode); int vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg); int vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo); int vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo); -int vnodeBegin(SVnode* pVnode, int option); int vnodeSyncCommit(SVnode* pVnode); int vnodeAsyncCommit(SVnode* pVnode); -#define vnodeShouldCommit vnodeBufPoolIsFull - -#if 1 -// SVBufPool -int vnodeOpenBufPool(SVnode* pVnode); -void vnodeCloseBufPool(SVnode* pVnode); -int vnodeBufPoolSwitch(SVnode* pVnode); -int vnodeBufPoolRecycle(SVnode* pVnode); -void* vnodeMalloc(SVnode* pVnode, uint64_t size); -bool vnodeBufPoolIsFull(SVnode* pVnode); - -SMemAllocatorFactory* vBufPoolGetMAF(SVnode* pVnode); - -// SVMemAllocator -typedef struct SVArenaNode { - TD_SLIST_NODE(SVArenaNode); - uint64_t size; // current node size - void* ptr; - char data[]; -} SVArenaNode; - -typedef struct SVMemAllocator { - T_REF_DECLARE() - TD_DLIST_NODE(SVMemAllocator); - uint64_t capacity; - uint64_t ssize; - uint64_t lsize; - SVArenaNode* pNode; - TD_SLIST(SVArenaNode) nlist; -} SVMemAllocator; - -SVMemAllocator* vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); -void vmaDestroy(SVMemAllocator* pVMA); -void vmaReset(SVMemAllocator* pVMA); -void* vmaMalloc(SVMemAllocator* pVMA, uint64_t size); -void vmaFree(SVMemAllocator* pVMA, void* ptr); -bool vmaIsFull(SVMemAllocator* pVMA); - -// vnodeCfg.h -extern const SVnodeCfg vnodeCfgDefault; - -int vnodeCheckCfg(const SVnodeCfg*); - -#endif +// vnodeCommit ==================== +int32_t vnodeSyncOpen(SVnode* pVnode, char* path); +int32_t vnodeSyncStart(SVnode* pVnode); +void vnodeSyncClose(SVnode* pVnode); +void vnodeSyncSetQ(SVnode* pVnode, void* qHandle); +void vnodeSyncSetRpc(SVnode* pVnode, void* rpcHandle); +int32_t vnodeSyncEqMsg(void* qHandle, SRpcMsg* pMsg); +int32_t vnodeSendMsg(void* rpcHandle, const SEpSet* pEpSet, SRpcMsg* pMsg); +void vnodeSyncCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); +void vnodeSyncPreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); +void vnodeSyncRollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); +int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot); +SSyncFSM* syncVnodeMakeFsm(); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0f354401bf..2d4cee3cad 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -22,9 +22,11 @@ #include "sync.h" #include "tchecksum.h" #include "tcoding.h" +#include "tcompare.h" #include "tcompression.h" #include "tdatablock.h" #include "tdbInt.h" +#include "tencode.h" #include "tfs.h" #include "tglobal.h" #include "tjson.h" @@ -57,6 +59,55 @@ typedef struct SQWorkerMgmt SQHandle; #define VNODE_TQ_DIR "tq" #define VNODE_WAL_DIR "wal" +// vnd.h +void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); +void vnodeBufPoolFree(SVBufPool* pPool, void* p); + +// meta +typedef struct SMCtbCursor SMCtbCursor; +typedef struct STbUidStore STbUidStore; + +int metaOpen(SVnode* pVnode, SMeta** ppMeta); +int metaClose(SMeta* pMeta); +int metaBegin(SMeta* pMeta); +int metaCommit(SMeta* pMeta); +int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); +SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); +STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); +int metaGetTableEntryByUid(SMetaReader* pReader, tb_uid_t uid); +int metaGetTableEntryByName(SMetaReader* pReader, const char* name); +int metaGetTbNum(SMeta* pMeta); +SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); +void metaCloseCtbCurosr(SMCtbCursor* pCtbCur); +tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); +SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup); +void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode); +STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid); +int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); +int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); + +// tsdb +int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb); +int tsdbClose(STsdb* pTsdb); +int tsdbBegin(STsdb* pTsdb); +int tsdbCommit(STsdb* pTsdb); +int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version); +int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); +int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); +int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); + +// tq +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); +void tqClose(STQ*); +int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); +int tqCommit(STQ*); +int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); +int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); + typedef struct { int8_t streamType; // sma or other int8_t dstType; @@ -68,8 +119,8 @@ typedef struct { } SStreamSinkInfo; typedef struct { - SVnode *pVnode; - SHashObj *pHash; // streamId -> SStreamSinkInfo + SVnode* pVnode; + SHashObj* pHash; // streamId -> SStreamSinkInfo } SSink; // SVState @@ -85,20 +136,29 @@ struct SVnodeInfo { }; struct SVnode { - char *path; + char* path; SVnodeCfg config; SVState state; - STfs *pTfs; + STfs* pTfs; SMsgCb msgCb; - SVBufPool *pBufPool; - SMeta *pMeta; - STsdb *pTsdb; - SWal *pWal; - STQ *pTq; - SSink *pSink; - int64_t sync; // sync integration + SVBufPool* pPool; + SVBufPool* inUse; + SVBufPool* onCommit; + SVBufPool* onRecycle; + SMeta* pMeta; + STsdb* pTsdb; + SWal* pWal; + STQ* pTq; + SSink* pSink; + int64_t sync; tsem_t canCommit; - SQHandle *pQuery; + SQHandle* pQuery; +}; + +struct STbUidStore { + tb_uid_t suid; + SArray* tbUids; + SHashObj* uidHash; }; #define TD_VID(PVNODE) (PVNODE)->config.vgId @@ -106,19 +166,7 @@ struct SVnode { typedef struct STbDdlH STbDdlH; // sma -void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data); - -#include "vnd.h" - -#include "meta.h" - -#include "tsdb.h" - -#include "tq.h" - -#include "vnodeSync.h" - -#include "tsdbSma.h" +void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h deleted file mode 100644 index fea94c4607..0000000000 --- a/source/dnode/vnode/src/inc/vnodeSync.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_SYNC_H_ -#define _TD_VNODE_SYNC_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t vnodeSyncOpen(SVnode *pVnode, char *path); -int32_t vnodeSyncStart(SVnode *pVnode); -void vnodeSyncClose(SVnode *pVnode); - -void vnodeSyncSetQ(SVnode *pVnode, void *qHandle); -void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle); - -int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg); -int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg); - -void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta); -void vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta); -void vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta); -int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot); - -SSyncFSM *syncVnodeMakeFsm(); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_SYNC_H_*/ diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c deleted file mode 100644 index 249e489029..0000000000 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ /dev/null @@ -1,1048 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define ALLOW_FORBID_FUNC -#include "db.h" - -#include "vnodeInt.h" - -#define IMPL_WITH_LOCK 1 -// #if IMPL_WITH_LOCK -// #endif - -typedef struct { - tb_uid_t uid; - int32_t sver; - int32_t padding; -} SSchemaKey; - -struct SMetaDB { -#if IMPL_WITH_LOCK - TdThreadRwlock rwlock; -#endif - // DB - DB *pTbDB; - DB *pSchemaDB; - DB *pSmaDB; - - // IDX - DB *pNameIdx; - DB *pStbIdx; - DB *pNtbIdx; - DB *pCtbIdx; - DB *pSmaIdx; - // ENV - DB_ENV *pEvn; -}; - -typedef int (*bdbIdxCbPtr)(DB *, const DBT *, const DBT *, DBT *); - -static SMetaDB *metaNewDB(); -static void metaFreeDB(SMetaDB *pDB); -static int metaOpenBDBEnv(DB_ENV **ppEnv, const char *path); -static void metaCloseBDBEnv(DB_ENV *pEnv); -static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup); -static void metaCloseBDBDb(DB *pDB); -static int metaOpenBDBIdx(DB **ppIdx, DB_ENV *pEnv, const char *pFName, DB *pDB, bdbIdxCbPtr cbf, bool isDup); -static void metaCloseBDBIdx(DB *pIdx); -static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); -static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); -static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); -static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); -static int metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); -static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); -static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); -static void metaClearTbCfg(STbCfg *pTbCfg); -static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); -static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW); -static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW); -static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx); -static void metaDBWLock(SMetaDB *pDB); -static void metaDBRLock(SMetaDB *pDB); -static void metaDBULock(SMetaDB *pDB); -static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx); - -#define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) - -int metaOpenDB(SMeta *pMeta) { - SMetaDB *pDB; - - // Create DB object - pDB = metaNewDB(); - if (pDB == NULL) { - return -1; - } - - pMeta->pDB = pDB; - - // Open DB Env - if (metaOpenBDBEnv(&(pDB->pEvn), pMeta->path) < 0) { - metaCloseDB(pMeta); - return -1; - } - - // Open DBs - if (metaOpenBDBDb(&(pDB->pTbDB), pDB->pEvn, "meta.db", false) < 0) { - metaCloseDB(pMeta); - return -1; - } - - if (metaOpenBDBDb(&(pDB->pSchemaDB), pDB->pEvn, "schema.db", false) < 0) { - metaCloseDB(pMeta); - return -1; - } - - if (metaOpenBDBDb(&(pDB->pSmaDB), pDB->pEvn, "sma.db", false) < 0) { - metaCloseDB(pMeta); - return -1; - } - - // Open Indices - if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "name.index", pDB->pTbDB, &metaNameIdxCb, false) < 0) { - metaCloseDB(pMeta); - return -1; - } - - if (metaOpenBDBIdx(&(pDB->pStbIdx), pDB->pEvn, "stb.index", pDB->pTbDB, &metaStbIdxCb, false) < 0) { - metaCloseDB(pMeta); - return -1; - } - - if (metaOpenBDBIdx(&(pDB->pNtbIdx), pDB->pEvn, "ntb.index", pDB->pTbDB, &metaNtbIdxCb, false) < 0) { - metaCloseDB(pMeta); - return -1; - } - - if (metaOpenBDBIdx(&(pDB->pCtbIdx), pDB->pEvn, "ctb.index", pDB->pTbDB, &metaCtbIdxCb, true) < 0) { - metaCloseDB(pMeta); - return -1; - } - - if (metaOpenBDBIdx(&(pDB->pSmaIdx), pDB->pEvn, "sma.index", pDB->pSmaDB, &metaSmaIdxCb, true) < 0) { - metaCloseDB(pMeta); - return -1; - } - - return 0; -} - -void metaCloseDB(SMeta *pMeta) { - if (pMeta->pDB) { - metaCloseBDBIdx(pMeta->pDB->pSmaIdx); - metaCloseBDBIdx(pMeta->pDB->pCtbIdx); - metaCloseBDBIdx(pMeta->pDB->pNtbIdx); - metaCloseBDBIdx(pMeta->pDB->pStbIdx); - metaCloseBDBIdx(pMeta->pDB->pNameIdx); - metaCloseBDBDb(pMeta->pDB->pSmaDB); - metaCloseBDBDb(pMeta->pDB->pSchemaDB); - metaCloseBDBDb(pMeta->pDB->pTbDB); - metaCloseBDBEnv(pMeta->pDB->pEvn); - metaFreeDB(pMeta->pDB); - pMeta->pDB = NULL; - } -} - -int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - tb_uid_t uid; - char buf[512]; - char buf1[512]; - void *pBuf; - DBT key1, value1; - DBT key2, value2; - SSchemaEx *pSchema = NULL; - - if (pTbCfg->type == META_SUPER_TABLE) { - uid = pTbCfg->stbCfg.suid; - } else { - uid = metaGenerateUid(pMeta); - } - - { - // save table info - pBuf = buf; - memset(&key1, 0, sizeof(key1)); - memset(&value1, 0, sizeof(key1)); - - key1.data = &uid; - key1.size = sizeof(uid); - - metaEncodeTbInfo(&pBuf, pTbCfg); - - value1.data = buf; - value1.size = POINTER_DISTANCE(pBuf, buf); - value1.app_data = pTbCfg; - } - - // save schema - uint32_t ncols; - if (pTbCfg->type == META_SUPER_TABLE) { - ncols = pTbCfg->stbCfg.nCols; - pSchema = pTbCfg->stbCfg.pSchema; - } else if (pTbCfg->type == META_NORMAL_TABLE) { - ncols = pTbCfg->ntbCfg.nCols; - pSchema = pTbCfg->ntbCfg.pSchema; - } - - if (pSchema) { - pBuf = buf1; - memset(&key2, 0, sizeof(key2)); - memset(&value2, 0, sizeof(key2)); - SSchemaKey schemaKey = {uid, 0 /*TODO*/, 0}; - - key2.data = &schemaKey; - key2.size = sizeof(schemaKey); - - SSchemaWrapper sw = {.nCols = ncols, .pSchemaEx = pSchema}; - metaEncodeSchemaEx(&pBuf, &sw); - - value2.data = buf1; - value2.size = POINTER_DISTANCE(pBuf, buf1); - } - - metaDBWLock(pMeta->pDB); - pMeta->pDB->pTbDB->put(pMeta->pDB->pTbDB, NULL, &key1, &value1, 0); - if (pSchema) { - pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key2, &value2, 0); - } - metaDBULock(pMeta->pDB); - - return 0; -} - -int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { - // TODO - return 0; -} - -int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { - // char buf[512] = {0}; // TODO: may overflow - void *pBuf = NULL, *qBuf = NULL; - DBT key1 = {0}, value1 = {0}; - - // save sma info - int32_t len = tEncodeTSma(NULL, pSmaCfg); - pBuf = taosMemoryCalloc(1, len); - if (pBuf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - key1.data = (void *)&pSmaCfg->indexUid; - key1.size = sizeof(pSmaCfg->indexUid); - - qBuf = pBuf; - tEncodeTSma(&qBuf, pSmaCfg); - - value1.data = pBuf; - value1.size = POINTER_DISTANCE(qBuf, pBuf); - value1.app_data = pSmaCfg; - - metaDBWLock(pMeta->pDB); - pMeta->pDB->pSmaDB->put(pMeta->pDB->pSmaDB, NULL, &key1, &value1, 0); - metaDBULock(pMeta->pDB); - - // release - taosMemoryFreeClear(pBuf); - - return 0; -} - -int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) { - // TODO -#if 0 - DBT key = {0}; - - key.data = (void *)indexName; - key.size = strlen(indexName); - - metaDBWLock(pMeta->pDB); - // TODO: No guarantee of consistence. - // Use transaction or DB->sync() for some guarantee. - pMeta->pDB->pSmaDB->del(pMeta->pDB->pSmaDB, NULL, &key, 0); - metaDBULock(pMeta->pDB); -#endif - return 0; -} - -/* ------------------------ STATIC METHODS ------------------------ */ -static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { - int tlen = 0; - SSchema *pSchema; - - tlen += taosEncodeFixedU32(buf, pSW->nCols); - for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchema + i; - tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI16(buf, pSchema->colId); - tlen += taosEncodeFixedI32(buf, pSchema->bytes); - tlen += taosEncodeString(buf, pSchema->name); - } - - return tlen; -} - -static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { - SSchema *pSchema; - - buf = taosDecodeFixedU32(buf, &pSW->nCols); - pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); - - int8_t dummy; - for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchema + i; - buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosDecodeFixedI16(buf, &pSchema->colId); - buf = taosDecodeFixedI32(buf, &pSchema->bytes); - buf = taosDecodeStringTo(buf, pSchema->name); - } - - return buf; -} - -static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW) { - int tlen = 0; - SSchemaEx *pSchema; - - tlen += taosEncodeFixedU32(buf, pSW->nCols); - for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchemaEx + i; - tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI8(buf, pSchema->sma); - tlen += taosEncodeFixedI16(buf, pSchema->colId); - tlen += taosEncodeFixedI32(buf, pSchema->bytes); - tlen += taosEncodeString(buf, pSchema->name); - } - - return tlen; -} - -static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx) { - buf = taosDecodeFixedU32(buf, &pSW->nCols); - if (isGetEx) { - pSW->pSchemaEx = (SSchemaEx *)taosMemoryMalloc(sizeof(SSchemaEx) * pSW->nCols); - for (int i = 0; i < pSW->nCols; i++) { - SSchemaEx *pSchema = pSW->pSchemaEx + i; - buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosDecodeFixedI8(buf, &pSchema->sma); - buf = taosDecodeFixedI16(buf, &pSchema->colId); - buf = taosDecodeFixedI32(buf, &pSchema->bytes); - buf = taosDecodeStringTo(buf, pSchema->name); - } - } else { - pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); - for (int i = 0; i < pSW->nCols; i++) { - SSchema *pSchema = pSW->pSchema + i; - buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosSkipFixedLen(buf, sizeof(int8_t)); - buf = taosDecodeFixedI16(buf, &pSchema->colId); - buf = taosDecodeFixedI32(buf, &pSchema->bytes); - buf = taosDecodeStringTo(buf, pSchema->name); - } - } - - return buf; -} - -static SMetaDB *metaNewDB() { - SMetaDB *pDB = NULL; - pDB = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDB)); - if (pDB == NULL) { - return NULL; - } - -#if IMPL_WITH_LOCK - taosThreadRwlockInit(&pDB->rwlock, NULL); -#endif - - return pDB; -} - -static void metaFreeDB(SMetaDB *pDB) { - if (pDB) { -#if IMPL_WITH_LOCK - taosThreadRwlockDestroy(&pDB->rwlock); -#endif - taosMemoryFree(pDB); - } -} - -static int metaOpenBDBEnv(DB_ENV **ppEnv, const char *path) { - int ret; - DB_ENV *pEnv; - - if (path == NULL) return 0; - - ret = db_env_create(&pEnv, 0); - if (ret != 0) { - BDB_PERR("Failed to create META env", ret); - return -1; - } - - ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_CDB | DB_INIT_MPOOL, 0); - if (ret != 0) { - BDB_PERR("Failed to open META env", ret); - return -1; - } - - *ppEnv = pEnv; - - return 0; -} - -static void metaCloseBDBEnv(DB_ENV *pEnv) { - if (pEnv) { - pEnv->close(pEnv, 0); - } -} - -static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup) { - int ret; - DB *pDB; - - ret = db_create(&(pDB), pEnv, 0); - if (ret != 0) { - BDB_PERR("Failed to create META DB", ret); - return -1; - } - - if (isDup) { - ret = pDB->set_flags(pDB, DB_DUPSORT); - if (ret != 0) { - BDB_PERR("Failed to set DB flags", ret); - return -1; - } - } - - ret = pDB->open(pDB, NULL, pFName, NULL, DB_BTREE, DB_CREATE, 0); - if (ret) { - BDB_PERR("Failed to open META DB", ret); - return -1; - } - - *ppDB = pDB; - - return 0; -} - -static void metaCloseBDBDb(DB *pDB) { - if (pDB) { - pDB->close(pDB, 0); - } -} - -static int metaOpenBDBIdx(DB **ppIdx, DB_ENV *pEnv, const char *pFName, DB *pDB, bdbIdxCbPtr cbf, bool isDup) { - DB *pIdx; - int ret; - - if (metaOpenBDBDb(ppIdx, pEnv, pFName, isDup) < 0) { - return -1; - } - - pIdx = *ppIdx; - ret = pDB->associate(pDB, NULL, pIdx, cbf, 0); - if (ret) { - BDB_PERR("Failed to associate META DB and Index", ret); - } - - return 0; -} - -static void metaCloseBDBIdx(DB *pIdx) { - if (pIdx) { - pIdx->close(pIdx, 0); - } -} - -static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { - STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); - - memset(pSKey, 0, sizeof(*pSKey)); - - pSKey->data = pTbCfg->name; - pSKey->size = strlen(pTbCfg->name); - - return 0; -} - -static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { - STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); - - if (pTbCfg->type == META_SUPER_TABLE) { - memset(pSKey, 0, sizeof(*pSKey)); - pSKey->data = pKey->data; - pSKey->size = pKey->size; - - return 0; - } else { - return DB_DONOTINDEX; - } -} - -static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { - STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); - - if (pTbCfg->type == META_NORMAL_TABLE) { - memset(pSKey, 0, sizeof(*pSKey)); - pSKey->data = pKey->data; - pSKey->size = pKey->size; - - return 0; - } else { - return DB_DONOTINDEX; - } -} - -static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { - STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); - DBT *pDbt; - - if (pTbCfg->type == META_CHILD_TABLE) { - // pDbt = taosMemoryCalloc(2, sizeof(DBT)); - - // // First key is suid - // pDbt[0].data = &(pTbCfg->ctbCfg.suid); - // pDbt[0].size = sizeof(pTbCfg->ctbCfg.suid); - - // // Second key is the first tag - // void *pTagVal = tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); - // pDbt[1].data = pTagVal; - // pDbt[1].size = sizeof(int32_t); - - // Set index key - memset(pSKey, 0, sizeof(*pSKey)); -#if 0 - pSKey->flags = DB_DBT_MULTIPLE | DB_DBT_APPMALLOC; - pSKey->data = pDbt; - pSKey->size = 2; -#else - pSKey->data = &(pTbCfg->ctbCfg.suid); - pSKey->size = sizeof(pTbCfg->ctbCfg.suid); -#endif - - return 0; - } else { - return DB_DONOTINDEX; - } -} - -static int metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { - STSma *pSmaCfg = (STSma *)(pValue->app_data); - - memset(pSKey, 0, sizeof(*pSKey)); - pSKey->data = &(pSmaCfg->tableUid); - pSKey->size = sizeof(pSmaCfg->tableUid); - - return 0; -} - -static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { - int tsize = 0; - - tsize += taosEncodeString(buf, pTbCfg->name); - tsize += taosEncodeFixedU32(buf, pTbCfg->ttl); - tsize += taosEncodeFixedU32(buf, pTbCfg->keep); - tsize += taosEncodeFixedU8(buf, pTbCfg->info); - - if (pTbCfg->type == META_SUPER_TABLE) { - SSchemaWrapper sw = {.nCols = pTbCfg->stbCfg.nTagCols, .pSchema = pTbCfg->stbCfg.pTagSchema}; - tsize += metaEncodeSchema(buf, &sw); - } else if (pTbCfg->type == META_CHILD_TABLE) { - tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid); - tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag); - } else if (pTbCfg->type == META_NORMAL_TABLE) { - // TODO - } else { - ASSERT(0); - } - - return tsize; -} - -static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { - buf = taosDecodeString(buf, &(pTbCfg->name)); - buf = taosDecodeFixedU32(buf, &(pTbCfg->ttl)); - buf = taosDecodeFixedU32(buf, &(pTbCfg->keep)); - buf = taosDecodeFixedU8(buf, &(pTbCfg->info)); - - if (pTbCfg->type == META_SUPER_TABLE) { - SSchemaWrapper sw; - buf = metaDecodeSchema(buf, &sw); - pTbCfg->stbCfg.nTagCols = sw.nCols; - pTbCfg->stbCfg.pTagSchema = sw.pSchema; - } else if (pTbCfg->type == META_CHILD_TABLE) { - buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); - buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag)); - } else if (pTbCfg->type == META_NORMAL_TABLE) { - // TODO - } else { - ASSERT(0); - } - return buf; -} - -static void metaClearTbCfg(STbCfg *pTbCfg) { - taosMemoryFreeClear(pTbCfg->name); - if (pTbCfg->type == META_SUPER_TABLE) { - tdFreeSchema(pTbCfg->stbCfg.pTagSchema); - } else if (pTbCfg->type == META_CHILD_TABLE) { - taosMemoryFreeClear(pTbCfg->ctbCfg.pTag); - } -} - -/* ------------------------ FOR QUERY ------------------------ */ -STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { - STbCfg *pTbCfg = NULL; - SMetaDB *pDB = pMeta->pDB; - DBT key = {0}; - DBT value = {0}; - int ret; - - // Set key/value - key.data = &uid; - key.size = sizeof(uid); - - // Query - metaDBRLock(pDB); - ret = pDB->pTbDB->get(pDB->pTbDB, NULL, &key, &value, 0); - metaDBULock(pDB); - if (ret != 0) { - return NULL; - } - - // Decode - pTbCfg = (STbCfg *)taosMemoryMalloc(sizeof(*pTbCfg)); - if (pTbCfg == NULL) { - return NULL; - } - - metaDecodeTbInfo(value.data, pTbCfg); - - return pTbCfg; -} - -STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { - STbCfg *pTbCfg = NULL; - SMetaDB *pDB = pMeta->pDB; - DBT key = {0}; - DBT pkey = {0}; - DBT pvalue = {0}; - int ret; - - // Set key/value - key.data = tbname; - key.size = strlen(tbname); - - // Query - metaDBRLock(pDB); - ret = pDB->pNameIdx->pget(pDB->pNameIdx, NULL, &key, &pkey, &pvalue, 0); - metaDBULock(pDB); - if (ret != 0) { - return NULL; - } - - // Decode - *uid = *(tb_uid_t *)(pkey.data); - pTbCfg = (STbCfg *)taosMemoryMalloc(sizeof(*pTbCfg)); - if (pTbCfg == NULL) { - return NULL; - } - - metaDecodeTbInfo(pvalue.data, pTbCfg); - - return pTbCfg; -} - -void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { - STSma *pCfg = NULL; - SMetaDB *pDB = pMeta->pDB; - DBT key = {0}; - DBT value = {0}; - int ret; - - // Set key/value - key.data = (void *)&indexUid; - key.size = sizeof(indexUid); - - // Query - metaDBRLock(pDB); - ret = pDB->pTbDB->get(pDB->pSmaDB, NULL, &key, &value, 0); - metaDBULock(pDB); - if (ret != 0) { - return NULL; - } - - // Decode - pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma)); - if (pCfg == NULL) { - return NULL; - } - - if (tDecodeTSma(value.data, pCfg) == NULL) { - taosMemoryFreeClear(pCfg); - return NULL; - } - - return pCfg; -} - -SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { - return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false); -} - -static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) { - uint32_t nCols; - SSchemaWrapper *pSW = NULL; - SMetaDB *pDB = pMeta->pDB; - int ret; - void *pBuf; - // SSchema *pSchema; - SSchemaKey schemaKey = {uid, sver, 0}; - DBT key = {0}; - DBT value = {0}; - - // Set key/value properties - key.data = &schemaKey; - key.size = sizeof(schemaKey); - - // Query - metaDBRLock(pDB); - ret = pDB->pSchemaDB->get(pDB->pSchemaDB, NULL, &key, &value, 0); - metaDBULock(pDB); - if (ret != 0) { - printf("failed to query schema DB since %s================\n", db_strerror(ret)); - return NULL; - } - - // Decode the schema - pBuf = value.data; - pSW = taosMemoryMalloc(sizeof(*pSW)); - metaDecodeSchemaEx(pBuf, pSW, isGetEx); - - return pSW; -} - -struct SMTbCursor { - DBC *pCur; -}; - -SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { - SMTbCursor *pTbCur = NULL; - SMetaDB *pDB = pMeta->pDB; - - pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); - if (pTbCur == NULL) { - return NULL; - } - - pDB->pTbDB->cursor(pDB->pTbDB, NULL, &(pTbCur->pCur), 0); - -#if 0 - DB_BTREE_STAT *sp; - pDB->pTbDB->stat(pDB->pTbDB, NULL, &sp, 0); - printf("**************** %ld\n", sp->bt_nkeys); -#endif - - return pTbCur; -} - -int metaGetTbNum(SMeta *pMeta) { - SMetaDB *pDB = pMeta->pDB; - - DB_BTREE_STAT *sp1; - pDB->pTbDB->stat(pDB->pNtbIdx, NULL, &sp1, 0); - - DB_BTREE_STAT *sp2; - pDB->pTbDB->stat(pDB->pCtbIdx, NULL, &sp2, 0); - - return sp1->bt_nkeys + sp2->bt_nkeys; -} - -void metaCloseTbCursor(SMTbCursor *pTbCur) { - if (pTbCur) { - if (pTbCur->pCur) { - pTbCur->pCur->close(pTbCur->pCur); - } - taosMemoryFree(pTbCur); - } -} - -char *metaTbCursorNext(SMTbCursor *pTbCur) { - DBT key = {0}; - DBT value = {0}; - STbCfg tbCfg; - void *pBuf; - - for (;;) { - if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) { - pBuf = value.data; - metaDecodeTbInfo(pBuf, &tbCfg); - if (tbCfg.type == META_SUPER_TABLE) { - taosMemoryFree(tbCfg.name); - taosMemoryFree(tbCfg.stbCfg.pTagSchema); - continue; - } else if (tbCfg.type == META_CHILD_TABLE) { - kvRowFree(tbCfg.ctbCfg.pTag); - } - return tbCfg.name; - } else { - return NULL; - } - } -} - -STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { - STSchemaBuilder sb; - STSchema *pTSchema = NULL; - SSchemaEx *pSchema; - SSchemaWrapper *pSW; - STbCfg *pTbCfg; - tb_uid_t quid; - - pTbCfg = metaGetTbInfoByUid(pMeta, uid); - if (pTbCfg->type == META_CHILD_TABLE) { - quid = pTbCfg->ctbCfg.suid; - } else { - quid = uid; - } - - pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true); - if (pSW == NULL) { - return NULL; - } - - // Rebuild a schema - tdInitTSchemaBuilder(&sb, 0); - for (int32_t i = 0; i < pSW->nCols; ++i) { - pSchema = pSW->pSchemaEx + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->sma, pSchema->colId, pSchema->bytes); - } - pTSchema = tdGetSchemaFromBuilder(&sb); - tdDestroyTSchemaBuilder(&sb); - - return pTSchema; -} - -struct SMCtbCursor { - DBC *pCur; - tb_uid_t suid; -}; - -SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { - SMCtbCursor *pCtbCur = NULL; - SMetaDB *pDB = pMeta->pDB; - int ret; - - pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); - if (pCtbCur == NULL) { - return NULL; - } - - pCtbCur->suid = uid; - ret = pDB->pCtbIdx->cursor(pDB->pCtbIdx, NULL, &(pCtbCur->pCur), 0); - if (ret != 0) { - taosMemoryFree(pCtbCur); - return NULL; - } - - return pCtbCur; -} - -void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { - if (pCtbCur) { - if (pCtbCur->pCur) { - pCtbCur->pCur->close(pCtbCur->pCur); - } - - taosMemoryFree(pCtbCur); - } -} - -tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { - DBT skey = {0}; - DBT pkey = {0}; - DBT pval = {0}; - void *pBuf; - STbCfg tbCfg; - - // Set key - skey.data = &(pCtbCur->suid); - skey.size = sizeof(pCtbCur->suid); - - if (pCtbCur->pCur->pget(pCtbCur->pCur, &skey, &pkey, &pval, DB_NEXT) == 0) { - tb_uid_t id = *(tb_uid_t *)pkey.data; - assert(id != 0); - return id; - // metaDecodeTbInfo(pBuf, &tbCfg); - // return tbCfg.; - } else { - return 0; - } -} - -struct SMSmaCursor { - DBC *pCur; - tb_uid_t uid; -}; - -SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { - SMSmaCursor *pCur = NULL; - SMetaDB *pDB = pMeta->pDB; - int ret; - - pCur = (SMSmaCursor *)taosMemoryCalloc(1, sizeof(*pCur)); - if (pCur == NULL) { - return NULL; - } - - pCur->uid = uid; - // TODO: lock? - ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0); - if ((ret != 0) || (pCur->pCur == NULL)) { - taosMemoryFree(pCur); - return NULL; - } - - return pCur; -} - -void metaCloseSmaCursor(SMSmaCursor *pCur) { - if (pCur) { - if (pCur->pCur) { - pCur->pCur->close(pCur->pCur); - } - - taosMemoryFree(pCur); - } -} - -int64_t metaSmaCursorNext(SMSmaCursor *pCur) { -#if 0 - DBT skey = {0}; - DBT pkey = {0}; - DBT pval = {0}; - - // Set key - skey.data = &(pCur->uid); - skey.size = sizeof(pCur->uid); - // TODO: lock? - if (pCur->pCur->pget(pCur->pCur, &skey, &pkey, &pval, DB_NEXT) == 0) { - const char *indexName = (const char *)pkey.data; - assert(indexName != NULL); - return indexName; - } else { - return NULL; - } -#endif - return 0; -} - -STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { - STSmaWrapper *pSW = NULL; - - pSW = taosMemoryCalloc(1, sizeof(*pSW)); - if (pSW == NULL) { - return NULL; - } - - SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); - if (pCur == NULL) { - taosMemoryFree(pSW); - return NULL; - } - - DBT skey = {.data = &(pCur->uid), .size = sizeof(pCur->uid)}; - DBT pval = {0}; - void *pBuf = NULL; - - while (true) { - // TODO: lock? - if (pCur->pCur->pget(pCur->pCur, &skey, NULL, &pval, DB_NEXT) == 0) { - ++pSW->number; - STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); - if (tptr == NULL) { - metaCloseSmaCursor(pCur); - tdDestroyTSmaWrapper(pSW); - taosMemoryFreeClear(pSW); - return NULL; - } - pSW->tSma = tptr; - pBuf = pval.data; - if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { - metaCloseSmaCursor(pCur); - tdDestroyTSmaWrapper(pSW); - taosMemoryFreeClear(pSW); - return NULL; - } - continue; - } - break; - } - - metaCloseSmaCursor(pCur); - - return pSW; -} - -SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { - SArray *pUids = NULL; - SMetaDB *pDB = pMeta->pDB; - DBC *pCur = NULL; - DBT pkey = {0}, pval = {0}; - uint32_t mode = isDup ? DB_NEXT_DUP : DB_NEXT_NODUP; - int ret; - - // TODO: lock? - ret = pDB->pSmaIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0); - if (ret != 0) { - return NULL; - } - // TODO: lock? - - while ((ret = pCur->get(pCur, &pkey, &pval, mode)) == 0) { - if (!pUids) { - pUids = taosArrayInit(16, sizeof(tb_uid_t)); - if (!pUids) { - return NULL; - } - } - - taosArrayPush(pUids, pkey.data); - } - // TODO: lock? - - if (pCur) { - pCur->close(pCur); - } - - return pUids; -} - -static void metaDBWLock(SMetaDB *pDB) { -#if IMPL_WITH_LOCK - taosThreadRwlockWrlock(&(pDB->rwlock)); -#endif -} - -static void metaDBRLock(SMetaDB *pDB) { -#if IMPL_WITH_LOCK - taosThreadRwlockRdlock(&(pDB->rwlock)); -#endif -} - -static void metaDBULock(SMetaDB *pDB) { -#if IMPL_WITH_LOCK - taosThreadRwlockUnlock(&(pDB->rwlock)); -#endif -} diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c new file mode 100644 index 0000000000..456c4fd7ee --- /dev/null +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "meta.h" + +static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBufPoolMalloc((SVBufPool *)pPool, size); } +static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } + +int metaBegin(SMeta *pMeta) { + tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + if (tdbBegin(pMeta->pEnv, &pMeta->txn) < 0) { + return -1; + } + + return 0; +} + +int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, &pMeta->txn); } diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c new file mode 100644 index 0000000000..e71a748f97 --- /dev/null +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "meta.h" + +int metaEncodeEntry(SCoder *pCoder, const SMetaEntry *pME) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pME->version) < 0) return -1; + if (tEncodeI8(pCoder, pME->type) < 0) return -1; + if (tEncodeI64(pCoder, pME->uid) < 0) return -1; + if (tEncodeCStr(pCoder, pME->name) < 0) return -1; + + if (pME->type == TSDB_SUPER_TABLE) { + if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1; + if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + } else if (pME->type == TSDB_CHILD_TABLE) { + if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1; + if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; + if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; + if (tEncodeBinary(pCoder, pME->ctbEntry.pTags, kvRowLen(pME->ctbEntry.pTags)) < 0) return -1; + } else if (pME->type == TSDB_NORMAL_TABLE) { + if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; + if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; + if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; + } else { + ASSERT(0); + } + + tEndEncode(pCoder); + return 0; +} + +int metaDecodeEntry(SCoder *pCoder, SMetaEntry *pME) { + uint64_t len; + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pME->version) < 0) return -1; + if (tDecodeI8(pCoder, &pME->type) < 0) return -1; + if (tDecodeI64(pCoder, &pME->uid) < 0) return -1; + if (tDecodeCStr(pCoder, &pME->name) < 0) return -1; + + if (pME->type == TSDB_SUPER_TABLE) { + if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1; + if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + } else if (pME->type == TSDB_CHILD_TABLE) { + if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1; + if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; + if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; + if (tDecodeBinary(pCoder, &pME->ctbEntry.pTags, &len) < 0) return -1; // (TODO) + } else if (pME->type == TSDB_NORMAL_TABLE) { + if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; + if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; + if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; + } else { + ASSERT(0); + } + + tEndDecode(pCoder); + return 0; +} diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 36057ae806..853b2ecefb 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -16,7 +16,7 @@ #ifdef USE_INVERTED_INDEX #include "index.h" #endif -#include "vnodeInt.h" +#include "meta.h" struct SMetaIdx { #ifdef USE_INVERTED_INDEX @@ -54,7 +54,9 @@ int metaOpenIdx(SMeta *pMeta) { #ifdef USE_INVERTED_INDEX SIndexOpts opts; - if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { return -1; } + if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { + return -1; + } #endif return 0; @@ -70,7 +72,9 @@ void metaCloseIdx(SMeta *pMeta) { /* TODO */ #ifdef USE_INVERTED_INDEX SIndexOpts opts; - if (indexClose(pMeta->pIdx->pIdx) != 0) { return -1; } + if (indexClose(pMeta->pIdx->pIdx) != 0) { + return -1; + } #endif } @@ -87,7 +91,7 @@ int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) { tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id tb_uid_t tuid = 0; // child table uid SIndexMultiTerm *terms = indexMultiTermCreate(); - SIndexTerm * term = + SIndexTerm *term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid); indexMultiTermAdd(terms, term); @@ -117,10 +121,13 @@ int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) { // TODO: add atomicity +#ifdef META_REFACT +#else if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) { // TODO: handle error return -1; } +#endif return TSDB_CODE_SUCCESS; } @@ -128,9 +135,12 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { // TODO: Validate the cfg // TODO: add atomicity +#ifdef META_REFACT +#else if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) { // TODO: handle error return -1; } +#endif return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 4419420e59..e65f5229af 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -13,10 +13,18 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "meta.h" + +static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); int metaOpen(SVnode *pVnode, SMeta **ppMeta) { SMeta *pMeta = NULL; + int ret; int slen; *ppMeta = NULL; @@ -36,31 +44,240 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { // create path if not created yet taosMkDir(pMeta->path); - // open meta - if (metaOpenDB(pMeta) < 0) { + // open env + ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); + if (ret < 0) { + metaError("vgId: %d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } - if (metaOpenIdx(pMeta) < 0) { + // open pTbDb + ret = tdbDbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); + if (ret < 0) { + metaError("vgId: %d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } + // open pSkmDb + ret = tdbDbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); + if (ret < 0) { + metaError("vgId: %d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + // open pUidIdx + ret = tdbDbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); + if (ret < 0) { + metaError("vgId: %d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + // open pNameIdx + ret = tdbDbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); + if (ret < 0) { + metaError("vgId: %d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + // open pCtbIdx + ret = tdbDbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); + if (ret < 0) { + metaError("vgId: %d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + // open pTagIdx + ret = tdbDbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); + if (ret < 0) { + metaError("vgId: %d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + // open pTtlIdx + ret = tdbDbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); + if (ret < 0) { + metaError("vgId: %d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + // open index + if (metaOpenIdx(pMeta) < 0) { + metaError("vgId: %d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + metaDebug("vgId: %d meta is opened", TD_VID(pVnode)); + *ppMeta = pMeta; return 0; _err: if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pDB) metaCloseDB(pMeta); + if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); + if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); taosMemoryFree(pMeta); return -1; } int metaClose(SMeta *pMeta) { if (pMeta) { - metaCloseIdx(pMeta); - metaCloseDB(pMeta); + if (pMeta->pIdx) metaCloseIdx(pMeta); + if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); + if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); taosMemoryFree(pMeta); } return 0; -} \ No newline at end of file +} + +static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + STbDbKey *pTbDbKey1 = (STbDbKey *)pKey1; + STbDbKey *pTbDbKey2 = (STbDbKey *)pKey2; + + if (pTbDbKey1->version > pTbDbKey2->version) { + return 1; + } else if (pTbDbKey1->version < pTbDbKey2->version) { + return -1; + } + + if (pTbDbKey1->uid > pTbDbKey2->uid) { + return 1; + } else if (pTbDbKey1->uid < pTbDbKey2->uid) { + return -1; + } + + return 0; +} + +static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + SSkmDbKey *pSkmDbKey1 = (SSkmDbKey *)pKey1; + SSkmDbKey *pSkmDbKey2 = (SSkmDbKey *)pKey2; + + if (pSkmDbKey1->uid > pSkmDbKey2->uid) { + return 1; + } else if (pSkmDbKey1->uid < pSkmDbKey2->uid) { + return -1; + } + + if (pSkmDbKey1->sver > pSkmDbKey2->sver) { + return 1; + } else if (pSkmDbKey1->sver < pSkmDbKey2->sver) { + return -1; + } + + return 0; +} + +static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + tb_uid_t uid1 = *(tb_uid_t *)pKey1; + tb_uid_t uid2 = *(tb_uid_t *)pKey2; + + if (uid1 > uid2) { + return 1; + } else if (uid1 < uid2) { + return -1; + } + + return 0; +} + +static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + SCtbIdxKey *pCtbIdxKey1 = (SCtbIdxKey *)pKey1; + SCtbIdxKey *pCtbIdxKey2 = (SCtbIdxKey *)pKey2; + + if (pCtbIdxKey1->suid > pCtbIdxKey2->suid) { + return 1; + } else if (pCtbIdxKey1->suid < pCtbIdxKey2->suid) { + return -1; + } + + if (pCtbIdxKey1->uid > pCtbIdxKey2->uid) { + return 1; + } else if (pCtbIdxKey1->uid < pCtbIdxKey2->uid) { + return -1; + } + + return 0; +} + +static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + STagIdxKey *pTagIdxKey1 = (STagIdxKey *)pKey1; + STagIdxKey *pTagIdxKey2 = (STagIdxKey *)pKey2; + int8_t *p1, *p2; + int8_t type; + int c; + + // compare suid + if (pTagIdxKey1->suid > pTagIdxKey2->suid) { + return 1; + } else if (pTagIdxKey1->suid < pTagIdxKey2->suid) { + return -1; + } + + // compare column id + if (pTagIdxKey1->cid > pTagIdxKey2->cid) { + return 1; + } else if (pTagIdxKey1->cid < pTagIdxKey2->cid) { + return -1; + } + + // compare value + p1 = pTagIdxKey1->data; + p2 = pTagIdxKey2->data; + ASSERT(p1[0] == p2[0]); + type = p1[0]; + + p1++; + p2++; + + c = doCompare(p1, p2, type, 0); + if (c) return c; + + if (IS_VAR_DATA_TYPE(type)) { + p1 = p1 + varDataTLen(p1); + p2 = p2 + varDataTLen(p2); + } else { + p1 = p1 + tDataTypes[type].bytes; + p2 = p2 + tDataTypes[type].bytes; + } + + // compare suid + if (*(tb_uid_t *)p1 > *(tb_uid_t *)p2) { + return 1; + } else if (*(tb_uid_t *)p1 < *(tb_uid_t *)p2) { + return -1; + } + + return 0; +} + +static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1; + STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2; + + if (pTtlIdxKey1->dtime > pTtlIdxKey2->dtime) { + return 1; + } else if (pTtlIdxKey1->dtime < pTtlIdxKey2->dtime) { + return -1; + } + + if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) { + return 1; + } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) { + return -1; + } + + return 0; +} diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c new file mode 100644 index 0000000000..dbfa8fe9ed --- /dev/null +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -0,0 +1,436 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "meta.h" + +void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { + memset(pReader, 0, sizeof(*pReader)); + pReader->flags = flags; + pReader->pMeta = pMeta; +} + +void metaReaderClear(SMetaReader *pReader) { + tCoderClear(&pReader->coder); + TDB_FREE(pReader->pBuf); +} + +int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t uid) { + SMeta *pMeta = pReader->pMeta; + STbDbKey tbDbKey = {.version = version, .uid = uid}; + + // query table.db + if (tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { + goto _err; + } + + // decode the entry + tCoderInit(&pReader->coder, TD_LITTLE_ENDIAN, pReader->pBuf, pReader->szBuf, TD_DECODER); + + if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { + goto _err; + } + + return 0; + +_err: + return -1; +} + +int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { + SMeta *pMeta = pReader->pMeta; + int64_t version; + + // query uid.idx + if (tdbDbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { + return -1; + } + + version = *(int64_t *)pReader->pBuf; + return metaGetTableEntryByVersion(pReader, version, uid); +} + +int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { + SMeta *pMeta = pReader->pMeta; + tb_uid_t uid; + + // query name.idx + if (tdbDbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + return -1; + } + + uid = *(tb_uid_t *)pReader->pBuf; + return metaGetTableEntryByUid(pReader, uid); +} + +int metaReadNext(SMetaReader *pReader) { + SMeta *pMeta = pReader->pMeta; + + // TODO + + return 0; +} + +#if 1 // =================================================== +SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { + SMTbCursor *pTbCur = NULL; + + pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); + if (pTbCur == NULL) { + return NULL; + } + + metaReaderInit(&pTbCur->mr, pMeta, 0); + + tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc); + + return pTbCur; +} + +void metaCloseTbCursor(SMTbCursor *pTbCur) { + if (pTbCur) { + TDB_FREE(pTbCur->pKey); + TDB_FREE(pTbCur->pVal); + metaReaderClear(&pTbCur->mr); + if (pTbCur->pDbc) { + tdbDbcClose(pTbCur->pDbc); + } + taosMemoryFree(pTbCur); + } +} + +int metaTbCursorNext(SMTbCursor *pTbCur) { + int ret; + void *pBuf; + STbCfg tbCfg; + + for (;;) { + ret = tdbDbNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + if (ret < 0) { + return -1; + } + + metaGetTableEntryByVersion(&pTbCur->mr, *(int64_t *)pTbCur->pVal, *(tb_uid_t *)pTbCur->pKey); + if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) { + continue; + } + + break; + } + + return 0; +} + +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { + void *pKey = NULL; + void *pVal = NULL; + int kLen = 0; + int vLen = 0; + int ret; + SSkmDbKey skmDbKey; + SSchemaWrapper *pSW = NULL; + SSchema *pSchema = NULL; + void *pBuf; + SCoder coder = {0}; + + // fetch + skmDbKey.uid = uid; + skmDbKey.sver = sver; + pKey = &skmDbKey; + kLen = sizeof(skmDbKey); + ret = tdbDbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); + if (ret < 0) { + return NULL; + } + + // decode + pBuf = pVal; + pSW = taosMemoryMalloc(sizeof(pSW)); + + tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_DECODER); + tDecodeSSchemaWrapper(&coder, pSW); + pSchema = taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); + memcpy(pSchema, pSW->pSchema, sizeof(SSchema) * pSW->nCols); + tCoderClear(&coder); + + pSW->pSchema = pSchema; + + TDB_FREE(pVal); + + return pSW; +} + +struct SMCtbCursor { + TDBC *pCur; + tb_uid_t suid; + void *pKey; + void *pVal; + int kLen; + int vLen; +}; + +SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { + SMCtbCursor *pCtbCur = NULL; + int ret; + + pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); + if (pCtbCur == NULL) { + return NULL; + } + + pCtbCur->suid = uid; + ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur); + if (ret < 0) { + taosMemoryFree(pCtbCur); + return NULL; + } + + return pCtbCur; +} + +void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { + if (pCtbCur) { + if (pCtbCur->pCur) { + tdbDbcClose(pCtbCur->pCur); + + TDB_FREE(pCtbCur->pKey); + TDB_FREE(pCtbCur->pVal); + } + + taosMemoryFree(pCtbCur); + } +} + +tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { + int ret; + SCtbIdxKey *pCtbIdxKey; + + ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); + if (ret < 0) { + return 0; + } + + pCtbIdxKey = pCtbCur->pKey; + + return pCtbIdxKey->uid; +} + +STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { + tb_uid_t quid; + SMetaReader mr = {0}; + STSchema *pTSchema = NULL; + SSchemaWrapper *pSW = NULL; + STSchemaBuilder sb = {0}; + SSchema *pSchema; + + metaReaderInit(&mr, pMeta, 0); + metaGetTableEntryByUid(&mr, uid); + + if (mr.me.type == TSDB_CHILD_TABLE) { + quid = mr.me.ctbEntry.suid; + } else { + quid = uid; + } + + metaReaderClear(&mr); + + pSW = metaGetTableSchema(pMeta, quid, sver, 0); + tdInitTSchemaBuilder(&sb, 0); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchema + i; + tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); + } + pTSchema = tdGetSchemaFromBuilder(&sb); + tdDestroyTSchemaBuilder(&sb); + + taosMemoryFree(pSW->pSchema); + taosMemoryFree(pSW); + return pTSchema; +} + +STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { +#if 0 +#ifdef META_TDB_SMA_TEST + STSmaWrapper *pSW = NULL; + + pSW = taosMemoryCalloc(1, sizeof(*pSW)); + if (pSW == NULL) { + return NULL; + } + + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); + if (pCur == NULL) { + taosMemoryFree(pSW); + return NULL; + } + + void *pBuf = NULL; + SSmaIdxKey *pSmaIdxKey = NULL; + + while (true) { + // TODO: lock during iterate? + if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + pSmaIdxKey = pCur->pKey; + ASSERT(pSmaIdxKey != NULL); + + void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false); + + if (pSmaVal == NULL) { + tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid); + continue; + } + + ++pSW->number; + STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); + if (tptr == NULL) { + TDB_FREE(pSmaVal); + metaCloseSmaCursor(pCur); + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); + return NULL; + } + pSW->tSma = tptr; + pBuf = pSmaVal; + if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { + TDB_FREE(pSmaVal); + metaCloseSmaCursor(pCur); + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); + return NULL; + } + TDB_FREE(pSmaVal); + continue; + } + break; + } + + metaCloseSmaCursor(pCur); + + return pSW; + +#endif +#endif + return NULL; +} + +int metaGetTbNum(SMeta *pMeta) { + // TODO + // ASSERT(0); + return 0; +} + +SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { +#if 0 + // TODO + // ASSERT(0); // comment this line to pass CI + // return NULL: +#ifdef META_TDB_SMA_TEST + SArray *pUids = NULL; + SMetaDB *pDB = pMeta->pDB; + void *pKey; + + // TODO: lock? + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0); + if (pCur == NULL) { + return NULL; + } + // TODO: lock? + + SSmaIdxKey *pSmaIdxKey = NULL; + tb_uid_t uid = 0; + while (true) { + // TODO: lock during iterate? + if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + ASSERT(pSmaIdxKey != NULL); + pSmaIdxKey = pCur->pKey; + + if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) { + continue; + } + uid = pSmaIdxKey->uid; + + if (!pUids) { + pUids = taosArrayInit(16, sizeof(tb_uid_t)); + if (!pUids) { + metaCloseSmaCursor(pCur); + return NULL; + } + } + + taosArrayPush(pUids, &uid); + + continue; + } + break; + } + + metaCloseSmaCursor(pCur); + + return pUids; +#endif +#endif + return NULL; +} + +void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { +#if 0 + // TODO + // ASSERT(0); + // return NULL; +#ifdef META_TDB_SMA_TEST + SMetaDB *pDB = pMeta->pDB; + void *pKey = NULL; + void *pVal = NULL; + int kLen = 0; + int vLen = 0; + int ret = -1; + + // Set key + pKey = (void *)&indexUid; + kLen = sizeof(indexUid); + + // Query + ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen); + if (ret != 0 || !pVal) { + return NULL; + } + + if (!isDecode) { + // return raw value + return pVal; + } + + // Decode + STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma)); + if (pCfg == NULL) { + taosMemoryFree(pVal); + return NULL; + } + + void *pBuf = pVal; + if (tDecodeTSma(pBuf, pCfg) == NULL) { + tdDestroyTSma(pCfg); + taosMemoryFree(pCfg); + TDB_FREE(pVal); + return NULL; + } + + TDB_FREE(pVal); + return pCfg; +#endif +#endif + return NULL; +} + +#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index d61d3346a2..fd5f679d0b 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -15,6 +15,8 @@ #include "vnodeInt.h" +#ifndef META_REFACT + typedef struct SPoolMem { int64_t size; struct SPoolMem *prev; @@ -45,7 +47,7 @@ struct SMetaDB { #endif }; -#pragma pack(push,1) +#pragma pack(push, 1) typedef struct { tb_uid_t uid; int32_t sver; @@ -379,64 +381,6 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { return 0; } -STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { - int ret; - SMetaDB *pMetaDb = pMeta->pDB; - void *pKey; - void *pVal; - int kLen; - int vLen; - STbCfg *pTbCfg; - - // Fetch - pKey = &uid; - kLen = sizeof(uid); - pVal = NULL; - ret = tdbDbGet(pMetaDb->pTbDB, pKey, kLen, &pVal, &vLen); - if (ret < 0) { - return NULL; - } - - // Decode - pTbCfg = taosMemoryMalloc(sizeof(*pTbCfg)); - metaDecodeTbInfo(pVal, pTbCfg); - - TDB_FREE(pVal); - - return pTbCfg; -} - -STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { - void *pKey; - void *pVal; - void *ppKey; - int pkLen; - int kLen; - int vLen; - int ret; - - pKey = tbname; - kLen = strlen(tbname) + 1; - pVal = NULL; - ppKey = NULL; - ret = tdbDbPGet(pMeta->pDB->pNameIdx, pKey, kLen, &ppKey, &pkLen, &pVal, &vLen); - if (ret < 0) { - return NULL; - } - - ASSERT(pkLen == kLen + sizeof(uid)); - - *uid = *(tb_uid_t *)POINTER_SHIFT(ppKey, kLen); - TDB_FREE(ppKey); - TDB_FREE(pVal); - - return metaGetTbInfoByUid(pMeta, *uid); -} - -SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { - return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false); -} - static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) { void *pKey; void *pVal; @@ -468,156 +412,6 @@ static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_ return pSchemaWrapper; } -STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { - tb_uid_t quid; - SSchemaWrapper *pSW; - STSchemaBuilder sb; - SSchema *pSchema; - STSchema *pTSchema; - STbCfg *pTbCfg; - - pTbCfg = metaGetTbInfoByUid(pMeta, uid); - if (pTbCfg->type == META_CHILD_TABLE) { - quid = pTbCfg->ctbCfg.suid; - } else { - quid = uid; - } - - pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true); - if (pSW == NULL) { - return NULL; - } - - tdInitTSchemaBuilder(&sb, 0); - for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); - } - pTSchema = tdGetSchemaFromBuilder(&sb); - tdDestroyTSchemaBuilder(&sb); - - return pTSchema; -} - -struct SMTbCursor { - TDBC *pDbc; -}; - -SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { - SMTbCursor *pTbCur = NULL; - SMetaDB *pDB = pMeta->pDB; - - pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); - if (pTbCur == NULL) { - return NULL; - } - - tdbDbcOpen(pDB->pTbDB, &pTbCur->pDbc); - - return pTbCur; -} - -void metaCloseTbCursor(SMTbCursor *pTbCur) { - if (pTbCur) { - if (pTbCur->pDbc) { - tdbDbcClose(pTbCur->pDbc); - } - taosMemoryFree(pTbCur); - } -} - -char *metaTbCursorNext(SMTbCursor *pTbCur) { - void *pKey = NULL; - void *pVal = NULL; - int kLen; - int vLen; - int ret; - void *pBuf; - STbCfg tbCfg; - - for (;;) { - ret = tdbDbNext(pTbCur->pDbc, &pKey, &kLen, &pVal, &vLen); - if (ret < 0) break; - pBuf = pVal; - metaDecodeTbInfo(pBuf, &tbCfg); - if (tbCfg.type == META_SUPER_TABLE) { - taosMemoryFree(tbCfg.name); - taosMemoryFree(tbCfg.stbCfg.pTagSchema); - continue; - } else if (tbCfg.type == META_CHILD_TABLE) { - kvRowFree(tbCfg.ctbCfg.pTag); - } - - return tbCfg.name; - } - - return NULL; -} - -struct SMCtbCursor { - TDBC *pCur; - tb_uid_t suid; - void *pKey; - void *pVal; - int kLen; - int vLen; -}; - -SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { - SMCtbCursor *pCtbCur = NULL; - SMetaDB *pDB = pMeta->pDB; - int ret; - - pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); - if (pCtbCur == NULL) { - return NULL; - } - - pCtbCur->suid = uid; - ret = tdbDbcOpen(pDB->pCtbIdx, &pCtbCur->pCur); - if (ret < 0) { - taosMemoryFree(pCtbCur); - return NULL; - } - - // TODO: move the cursor to the suid there - - return pCtbCur; -} - -void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { - if (pCtbCur) { - if (pCtbCur->pCur) { - tdbDbcClose(pCtbCur->pCur); - - TDB_FREE(pCtbCur->pKey); - TDB_FREE(pCtbCur->pVal); - } - - taosMemoryFree(pCtbCur); - } -} - -tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { - int ret; - SCtbIdxKey *pCtbIdxKey; - - ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); - if (ret < 0) { - return 0; - } - - pCtbIdxKey = pCtbCur->pKey; - - return pCtbIdxKey->uid; -} - -int metaGetTbNum(SMeta *pMeta) { - // TODO - // ASSERT(0); - return 0; -} - struct SMSmaCursor { TDBC *pCur; tb_uid_t uid; @@ -768,53 +562,6 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { return 0; } -void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { - // TODO - // ASSERT(0); - // return NULL; -#ifdef META_TDB_SMA_TEST - SMetaDB *pDB = pMeta->pDB; - void *pKey = NULL; - void *pVal = NULL; - int kLen = 0; - int vLen = 0; - int ret = -1; - - // Set key - pKey = (void *)&indexUid; - kLen = sizeof(indexUid); - - // Query - ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen); - if (ret != 0 || !pVal) { - return NULL; - } - - if (!isDecode) { - // return raw value - return pVal; - } - - // Decode - STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma)); - if (pCfg == NULL) { - taosMemoryFree(pVal); - return NULL; - } - - void *pBuf = pVal; - if (tDecodeTSma(pBuf, pCfg) == NULL) { - tdDestroyTSma(pCfg); - taosMemoryFree(pCfg); - TDB_FREE(pVal); - return NULL; - } - - TDB_FREE(pVal); - return pCfg; -#endif -} - /** * @brief * @@ -889,56 +636,6 @@ void metaCloseSmaCursor(SMSmaCursor *pCur) { #endif } -SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { - // TODO - // ASSERT(0); // comment this line to pass CI - // return NULL: -#ifdef META_TDB_SMA_TEST - SArray *pUids = NULL; - SMetaDB *pDB = pMeta->pDB; - void *pKey; - - // TODO: lock? - SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0); - if (pCur == NULL) { - return NULL; - } - // TODO: lock? - - SSmaIdxKey *pSmaIdxKey = NULL; - tb_uid_t uid = 0; - while (true) { - // TODO: lock during iterate? - if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { - ASSERT(pSmaIdxKey != NULL); - pSmaIdxKey = pCur->pKey; - - if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) { - continue; - } - uid = pSmaIdxKey->uid; - - if (!pUids) { - pUids = taosArrayInit(16, sizeof(tb_uid_t)); - if (!pUids) { - metaCloseSmaCursor(pCur); - return NULL; - } - } - - taosArrayPush(pUids, &uid); - - continue; - } - break; - } - - metaCloseSmaCursor(pCur); - - return pUids; -#endif -} - static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { int tlen = 0; SSchema *pSchema; @@ -1142,3 +839,5 @@ static void poolFree(void *arg, void *ptr) { tdbOsFree(pMem); } + +#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7f8ec3a656..452d52ce5e 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -13,31 +13,122 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "meta.h" -int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { - // Validate the tbOptions - // if (metaValidateTbCfg(pMeta, pTbCfg) < 0) { - // // TODO: handle error - // return -1; - // } +static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); +static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME); - // TODO: add atomicity +int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + SMetaEntry me = {0}; + int kLen = 0; + int vLen = 0; + const void *pKey = NULL; + const void *pVal = NULL; + void *pBuf = NULL; + int32_t szBuf = 0; + void *p = NULL; + SCoder coder = {0}; + SMetaReader mr = {0}; - if (metaSaveTableToDB(pMeta, pTbCfg, pHandle) < 0) { - // TODO: handle error + // validate req + metaReaderInit(&mr, pMeta, 0); + if (metaGetTableEntryByName(&mr, pReq->name) == 0) { +// TODO: just for pass case +#if 0 + terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + metaReaderClear(&mr); return -1; +#else + metaReaderClear(&mr); + return 0; +#endif } + metaReaderClear(&mr); - if (metaSaveTableToIdx(pMeta, pTbCfg) < 0) { - // TODO: handle error - return -1; - } + // set structs + me.version = version; + me.type = TSDB_SUPER_TABLE; + me.uid = pReq->suid; + me.name = pReq->name; + me.stbEntry.schema = pReq->schema; + me.stbEntry.schemaTag = pReq->schemaTag; + if (metaHandleEntry(pMeta, &me) < 0) goto _err; + + metaDebug("vgId: %d super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + + return 0; + +_err: + metaError("vgId: %d failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(terrno)); + return -1; +} + +int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { + // TODO return 0; } +int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { + SMetaEntry me = {0}; + SMetaReader mr = {0}; + + // validate message + if (pReq->type != TSDB_CHILD_TABLE && pReq->type != TSDB_NORMAL_TABLE) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + + // preprocess req + pReq->uid = tGenIdPI64(); + pReq->ctime = taosGetTimestampSec(); + + // validate req + metaReaderInit(&mr, pMeta, 0); + if (metaGetTableEntryByName(&mr, pReq->name) == 0) { + terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + metaReaderClear(&mr); + return -1; + } + metaReaderClear(&mr); + + // build SMetaEntry + me.version = version; + me.type = pReq->type; + me.uid = pReq->uid; + me.name = pReq->name; + if (me.type == TSDB_CHILD_TABLE) { + me.ctbEntry.ctime = pReq->ctime; + me.ctbEntry.ttlDays = pReq->ttl; + me.ctbEntry.suid = pReq->ctb.suid; + me.ctbEntry.pTags = pReq->ctb.pTag; + } else { + me.ntbEntry.ctime = pReq->ctime; + me.ntbEntry.ttlDays = pReq->ttl; + me.ntbEntry.schema = pReq->ntb.schema; + } + + if (metaHandleEntry(pMeta, &me) < 0) goto _err; + + metaDebug("vgId:%d table %s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, + pReq->type); + return 0; + +_err: + metaError("vgId:%d failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); + return -1; +} + int metaDropTable(SMeta *pMeta, tb_uid_t uid) { +#if 0 if (metaRemoveTableFromIdx(pMeta, uid) < 0) { // TODO: handle error return -1; @@ -47,7 +138,164 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid) { // TODO return -1; } +#endif return 0; } +static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { + STbDbKey tbDbKey; + void *pKey = NULL; + void *pVal = NULL; + int kLen = 0; + int vLen = 0; + SCoder coder = {0}; + + // set key and value + tbDbKey.version = pME->version; + tbDbKey.uid = pME->uid; + + pKey = &tbDbKey; + kLen = sizeof(tbDbKey); + + if (tEncodeSize(metaEncodeEntry, pME, vLen) < 0) { + goto _err; + } + + pVal = taosMemoryMalloc(vLen); + if (pVal == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER); + + if (metaEncodeEntry(&coder, pME) < 0) { + goto _err; + } + + tCoderClear(&coder); + + // write to table.db + if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + goto _err; + } + + taosMemoryFree(pVal); + return 0; + +_err: + taosMemoryFree(pVal); + return -1; +} + +static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); +} + +static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); +} + +static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { + int32_t ttlDays; + int64_t ctime; + STtlIdxKey ttlKey; + + if (pME->type == TSDB_CHILD_TABLE) { + ctime = pME->ctbEntry.ctime; + ttlDays = pME->ctbEntry.ttlDays; + } else if (pME->type == TSDB_NORMAL_TABLE) { + ctime = pME->ntbEntry.ctime; + ttlDays = pME->ntbEntry.ttlDays; + } else { + ASSERT(0); + } + + if (ttlDays <= 0) return 0; + + ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; + ttlKey.uid = pME->uid; + + return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); +} + +static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { + SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; + return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); +} + +static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) { + // TODO + return 0; +} + +static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { + SCoder coder = {0}; + void *pVal = NULL; + int vLen = 0; + int rcode = 0; + SSkmDbKey skmDbKey = {0}; + const SSchemaWrapper *pSW; + + if (pME->type == TSDB_SUPER_TABLE) { + pSW = &pME->stbEntry.schema; + } else if (pME->type == TSDB_NORMAL_TABLE) { + pSW = &pME->ntbEntry.schema; + } else { + ASSERT(0); + } + + skmDbKey.uid = pME->uid; + skmDbKey.sver = pSW->sver; + + // encode schema + if (tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen) < 0) return -1; + pVal = taosMemoryMalloc(vLen); + if (pVal == NULL) { + rcode = -1; + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER); + tEncodeSSchemaWrapper(&coder, pSW); + + if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + rcode = -1; + goto _exit; + } + +_exit: + taosMemoryFree(pVal); + tCoderClear(&coder); + return rcode; +} + +static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { + // save to table.db + if (metaSaveToTbDb(pMeta, pME) < 0) return -1; + + // update uid.idx + if (metaUpdateUidIdx(pMeta, pME) < 0) return -1; + + // update name.idx + if (metaUpdateNameIdx(pMeta, pME) < 0) return -1; + + if (pME->type == TSDB_CHILD_TABLE) { + // update ctb.idx + if (metaUpdateCtbIdx(pMeta, pME) < 0) return -1; + + // update tag.idx + if (metaUpdateTagIdx(pMeta, pME) < 0) return -1; + } else { + // update schema.db + if (metaSaveToSkmDb(pMeta, pME) < 0) return -1; + } + + if (pME->type != TSDB_SUPER_TABLE) { + if (metaUpdateTtlIdx(pMeta, pME) < 0) return -1; + } + + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8cabada511..28a489752a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tq.h" int32_t tqInit() { // @@ -176,9 +176,9 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ atomic_store_ptr(&pExec->pushHandle.handle, NULL); taosWUnLockLatch(&pExec->pushHandle.lock); - vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", - TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum, - rsp.reqOffset, rsp.rspOffset); + tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum, + rsp.reqOffset, rsp.rspOffset); // TODO destroy taosArrayDestroy(rsp.blockData); @@ -390,8 +390,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { fetchOffset = pReq->currentOffset + 1; } - vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); + tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); STqExec* pExec = taosHashGet(pTq->execs, pReq->subKey, strlen(pReq->subKey)); ASSERT(pExec); @@ -418,16 +418,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { while (1) { consumerEpoch = atomic_load_32(&pExec->epoch); if (consumerEpoch > reqEpoch) { - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", - consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", + consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); break; } taosThreadMutexLock(&pExec->pWalReader->mutex); if (walFetchHead(pExec->pWalReader, fetchOffset, pHeadWithCkSum) < 0) { - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), fetchOffset); + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), fetchOffset); taosThreadMutexUnlock(&pExec->pWalReader->mutex); break; } @@ -448,7 +448,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); #if 0 @@ -476,8 +476,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } #endif - vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); + tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; @@ -591,8 +591,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pMsg->code = 0; tmsgSendRsp(pMsg); - vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", - TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); + tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); // TODO destroy taosArrayDestroy(rsp.blockData); @@ -618,7 +618,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { fetchOffset = pReq->currentOffset + 1; } - vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, + tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); SMqPollRspV2 rspV2 = {0}; @@ -660,7 +660,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; } - vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, + tqDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, TD_VID(pTq->pVnode)); rspV2.reqOffset = pReq->currentOffset; @@ -671,7 +671,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO consumerEpoch = atomic_load_32(&pConsumer->epoch); if (consumerEpoch > reqEpoch) { - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); break; } @@ -680,11 +680,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); break; } - vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, + tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/ /*pHead = pTopic->pReadhandle->pHead;*/ @@ -709,7 +709,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } if (taosArrayGetSize(pRes) == 0) { - vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId, + tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); fetchOffset++; rspV2.skipLogNum++; @@ -770,7 +770,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pMsg->pCont = buf; pMsg->contLen = msgLen; pMsg->code = 0; - vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset, + tqDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset, pHead->msgType, consumerId, pReq->epoch); tmsgSendRsp(pMsg); taosMemoryFree(pHead); @@ -804,7 +804,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pMsg->contLen = tlen; pMsg->code = 0; tmsgSendRsp(pMsg); - vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId, + tqDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch); /*}*/ diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c index 8e59243a9c..e31566f3fa 100644 --- a/source/dnode/vnode/src/tq/tqCommit.c +++ b/source/dnode/vnode/src/tq/tqCommit.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tq.h" diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 72469b74ac..ca09cc1dc1 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tq.h" // #include // #include // #include @@ -86,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } strcpy(pMeta->dirPath, path); - char *name = taosMemoryMalloc(pathLen + 10) ; + char* name = taosMemoryMalloc(pathLen + 10); strcpy(name, path); if (!taosDirExist(name) && taosMkDir(name) != 0) { diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 3cff87340d..90f512611b 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "vnodeInt.h" +#include "tq.h" enum ETqOffsetPersist { TQ_OFFSET_PERSIST__LAZY = 1, diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index aeb9f27eab..cf1154071d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tq.h" STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle)); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 28ba50c3df..29f148b64d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" #define TSDB_MAX_SUBBLOCKS 8 @@ -142,12 +142,13 @@ int tsdbPrepareCommit(STsdb *pTsdb) { } int tsdbCommit(STsdb *pRepo) { - STsdbMemTable *pMem = pRepo->imem; - SCommitH commith = {0}; - SDFileSet *pSet = NULL; - int fid; + SCommitH commith = {0}; + SDFileSet *pSet = NULL; + int fid; - if (pRepo->imem == NULL) return 0; + // if (pRepo->imem == NULL) return 0; + pRepo->imem = pRepo->mem; + pRepo->mem = NULL; tsdbStartCommit(pRepo); // Resource initialization @@ -239,7 +240,7 @@ static void tsdbStartCommit(STsdb *pRepo) { static void tsdbEndCommit(STsdb *pTsdb, int eno) { tsdbEndFSTxn(pTsdb); - tsdbFreeMemTable(pTsdb, pTsdb->imem); + tsdbMemTableDestroy(pTsdb, pTsdb->imem); pTsdb->imem = NULL; tsdbInfo("vgId:%d commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c new file mode 100644 index 0000000000..585ef63531 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +int tsdbBegin(STsdb *pTsdb) { + STsdbMemTable *pMem; + + if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) { + return -1; + } + + return 0; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c index e24d5974af..9e0721815a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ #if 0 -#include "tsdbint.h" +#include "tsdb.h" typedef struct { STable * pTable; @@ -33,15 +33,15 @@ typedef struct { SDataCols *pDataCols; } SCompactH; -#define TSDB_COMPACT_WSET(pComph) (&((pComph)->wSet)) -#define TSDB_COMPACT_REPO(pComph) TSDB_READ_REPO(&((pComph)->readh)) +#define TSDB_COMPACT_WSET(pComph) (&((pComph)->wSet)) +#define TSDB_COMPACT_REPO(pComph) TSDB_READ_REPO(&((pComph)->readh)) #define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD) #define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA) #define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST) #define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD) #define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL) -#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh)) -#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh)) +#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh)) +#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh)) static int tsdbAsyncCompact(STsdbRepo *pRepo); static void tsdbStartCompact(STsdbRepo *pRepo); @@ -531,4 +531,3 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { } #endif - diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 0012a50ccf..91a7e1cd54 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 2fbe819476..74e3c66a9d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" static const char *TSDB_FNAME_SUFFIX[] = { "head", // TSDB_FILE_HEAD diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 8691920c8f..de5ff9ac91 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -13,96 +13,62 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" -static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); -static void tsdbFree(STsdb *pTsdb); -static int tsdbOpenImpl(STsdb *pTsdb); -static void tsdbCloseImpl(STsdb *pTsdb); - -STsdb *tsdbOpen(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { +int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) { STsdb *pTsdb = NULL; + int slen = 0; - // Set default TSDB Options - // if (pTsdbCfg == NULL) { - pTsdbCfg = &defautlTsdbOptions; - // } + *ppTsdb = NULL; + slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_TSDB_DIR) + 3; - // Validate the options - if (tsdbValidateOptions(pTsdbCfg) < 0) { - // TODO: handle error - return NULL; - } - - // Create the handle - pTsdb = tsdbNew(path, pVnode, pTsdbCfg, pMAF); + // create handle + pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); if (pTsdb == NULL) { - // TODO: handle error - return NULL; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - taosMkDir(path); - - // Open the TSDB - if (tsdbOpenImpl(pTsdb) < 0) { - // TODO: handle error - return NULL; - } - - return pTsdb; -} - -void tsdbClose(STsdb *pTsdb) { - if (pTsdb) { - tsdbCloseImpl(pTsdb); - tsdbFree(pTsdb); - } -} - -/* ------------------------ STATIC METHODS ------------------------ */ -static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { - STsdb *pTsdb = NULL; - - pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); - if (pTsdb == NULL) { - // TODO: handle error - return NULL; - } - - pTsdb->path = strdup(path); - pTsdb->vgId = TD_VID(pVnode); + pTsdb->path = (char *)&pTsdb[1]; + sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, + VNODE_TSDB_DIR); pTsdb->pVnode = pVnode; - tsdbOptionsCopy(&(pTsdb->config), pTsdbCfg); - pTsdb->pmaf = pMAF; - pTsdb->fs = tsdbNewFS(pTsdbCfg); + pTsdb->vgId = TD_VID(pVnode); + pTsdb->repoLocked = false; + tdbMutexInit(&pTsdb->mutex, NULL); + pTsdb->config = pVnode->config.tsdbCfg; + pTsdb->fs = tsdbNewFS(&pTsdb->config); - return pTsdb; + // create dir (TODO: use tfsMkdir) + taosMkDir(pTsdb->path); + + // open tsdb + if (tsdbOpenFS(pTsdb) < 0) { + goto _err; + } + + tsdbDebug("vgId: %d tsdb is opened", TD_VID(pVnode)); + + *ppTsdb = pTsdb; + return 0; + +_err: + taosMemoryFree(pTsdb); + return -1; } -static void tsdbFree(STsdb *pTsdb) { +int tsdbClose(STsdb *pTsdb) { if (pTsdb) { - tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb)); - tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb)); + tsdbCloseFS(pTsdb); + // tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb)); + // tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb)); tsdbFreeFS(pTsdb->fs); - taosMemoryFreeClear(pTsdb->path); + // taosMemoryFreeClear(pTsdb->path); taosMemoryFree(pTsdb); } -} - -static int tsdbOpenImpl(STsdb *pTsdb) { - tsdbOpenFS(pTsdb); - - // tsdbInitSma(pTsdb); - // TODO - return 0; } -static void tsdbCloseImpl(STsdb *pTsdb) { - tsdbCloseFS(pTsdb); - // TODO -} - int tsdbLockRepo(STsdb *pTsdb) { int code = taosThreadMutexLock(&pTsdb->mutex); if (code != 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index bea2b66af8..323fc7970b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); -static int tsdbMemTableInsertTbData(STsdb *pRepo, SSubmitBlk *pBlock, int32_t *pAffectedRows); static STbData *tsdbNewTbData(tb_uid_t uid); static void tsdbFreeTbData(STbData *pTbData); static char *tsdbGetTsTupleKey(const void *data); @@ -24,86 +22,51 @@ static int tsdbTbDataComp(const void *arg1, const void *arg2); static char *tsdbTbDataGetUid(const void *arg); static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row); -STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { - STsdbMemTable *pMemTable = (STsdbMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable)); +int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) { + STsdbMemTable *pMemTable; + SVnode *pVnode; + + *ppMemTable = NULL; + pVnode = pTsdb->pVnode; + + // alloc handle + pMemTable = (STsdbMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable)); if (pMemTable == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } + pMemTable->pPool = pTsdb->pVnode->inUse; T_REF_INIT_VAL(pMemTable, 1); - taosInitRWLatch(&(pMemTable->latch)); - pMemTable->keyMax = TSKEY_MIN; + taosInitRWLatch(&pMemTable->latch); pMemTable->keyMin = TSKEY_MAX; + pMemTable->keyMax = TSKEY_MIN; pMemTable->nRow = 0; - pMemTable->pMA = pTsdb->pmaf->create(pTsdb->pmaf); - if (pMemTable->pMA == NULL) { - taosMemoryFree(pMemTable); - return NULL; - } - - // Initialize the container - pMemTable->pSlIdx = - tSkipListCreate(5, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t), tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid); + pMemTable->pSlIdx = tSkipListCreate(pVnode->config.tsdbCfg.slLevel, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t), + tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid); if (pMemTable->pSlIdx == NULL) { - pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); taosMemoryFree(pMemTable); - return NULL; + return -1; } pMemTable->pHashIdx = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (pMemTable->pHashIdx == NULL) { - pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); tSkipListDestroy(pMemTable->pSlIdx); taosMemoryFree(pMemTable); - return NULL; - } - - return pMemTable; -} - -void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) { - if (pMemTable) { - taosHashCleanup(pMemTable->pHashIdx); - tSkipListDestroy(pMemTable->pSlIdx); - if (pMemTable->pMA) { - pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); - } - taosMemoryFree(pMemTable); - } -} - -int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp) { - SSubmitBlk *pBlock = NULL; - SSubmitMsgIter msgIter = {0}; - int32_t affectedrows = 0, numOfRows = 0; - - if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { - if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { - tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } return -1; } - tInitSubmitMsgIter(pMsg, &msgIter); - while (true) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; - if (tsdbMemTableInsertTbData(pTsdb, pBlock, &affectedrows) < 0) { - return -1; - } - - numOfRows += pBlock->numOfRows; - } - - if (pRsp != NULL) { - pRsp->affectedRows = affectedrows; - pRsp->numOfRows = numOfRows; - } - + *ppMemTable = pMemTable; return 0; } +void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable) { + if (pMemTable) { + taosHashCleanup(pMemTable->pHashIdx); + tSkipListDestroy(pMemTable->pSlIdx); + taosMemoryFree(pMemTable); + } +} + /** * This is an important function to load data or try to load data from memory skiplist iterator. * @@ -255,78 +218,7 @@ int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) { return 0; } -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { - ASSERT(pMsg != NULL); - // STsdbMeta * pMeta = pTsdb->tsdbMeta; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; - TSKEY now = taosGetTimestamp(pTsdb->config.precision); - TSKEY minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep2; - TSKEY maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.days; - - terrno = TSDB_CODE_SUCCESS; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; - - pBlock->uid = htobe64(pBlock->uid); - pBlock->suid = htobe64(pBlock->suid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->numOfRows = htons(pBlock->numOfRows); - -#if 0 - if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { - tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - STable *pTable = pMeta->tables[pBlock->tid]; - if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) { - tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable)); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - // Check schema version and update schema if needed - if (tsdbCheckTableSchema(pTsdb, pBlock, pTable) < 0) { - if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - continue; - } else { - return -1; - } - } - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) { - return -1; - } - } -#endif - } - - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; -} - -static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) { +int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) { // STsdbMeta *pMeta = pRepo->tsdbMeta; // int32_t points = 0; // STable *pTable = NULL; @@ -337,11 +229,9 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p STSRow *row; TSKEY keyMin; TSKEY keyMax; + SSubmitBlk *pBlkCopy; - // SMemTable *pMemTable = NULL; - // STableData *pTableData = NULL; - // STsdbCfg *pCfg = &(pRepo->config); - + // create container is nedd tptr = taosHashGet(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid)); if (tptr == NULL) { pTbData = tsdbNewTbData(pBlock->uid); @@ -358,7 +248,11 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p pTbData = *(STbData **)tptr; } - tInitSubmitBlkIter(pBlock, &blkIter); + // copy data to buffer pool + pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pBlock->dataLen + sizeof(*pBlock)); + memcpy(pBlkCopy, pBlock, pBlock->dataLen + sizeof(*pBlock)); + + tInitSubmitBlkIter(pBlkCopy, &blkIter); if (blkIter.row == NULL) return 0; keyMin = TD_ROW_KEY(blkIter.row); @@ -377,31 +271,6 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p (*pAffectedRows) += pBlock->numOfRows; - // STSRow* lastRow = NULL; - // int64_t osize = SL_SIZE(pTableData->pData); - // tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow); - // tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext); - // int64_t dsize = SL_SIZE(pTableData->pData) - osize; - // (*pAffectedRows) += points; - - // if(lastRow != NULL) { - // TSKEY lastRowKey = TD_ROW_KEY(lastRow); - // if (pMemTable->keyFirst > firstRowKey) pMemTable->keyFirst = firstRowKey; - // pMemTable->numOfRows += dsize; - - // if (pTableData->keyFirst > firstRowKey) pTableData->keyFirst = firstRowKey; - // pTableData->numOfRows += dsize; - // if (pMemTable->keyLast < lastRowKey) pMemTable->keyLast = lastRowKey; - // if (pTableData->keyLast < lastRowKey) pTableData->keyLast = lastRowKey; - // if (tsdbUpdateTableLatestInfo(pRepo, pTable, lastRow) < 0) { - // return -1; - // } - // } - - // STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion, -1); - // pRepo->stat.pointsWritten += points * schemaNCols(pSchema); - // pRepo->stat.totalStorage += points * schemaVLen(pSchema); - return 0; } @@ -526,7 +395,6 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow* row); static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); static STSRow* tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); -static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitReq *pMsg); static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows); static int tsdbInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter); static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock); diff --git a/source/dnode/vnode/src/tsdb/tsdbOptions.c b/source/dnode/vnode/src/tsdb/tsdbOptions.c deleted file mode 100644 index 3560c9feaa..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbOptions.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeInt.h" - -const STsdbCfg defautlTsdbOptions = {.precision = 0, - .lruCacheSize = 0, - .days = 10, - .minRows = 100, - .maxRows = 4096, - .keep2 = 3650, - .keep0 = 3650, - .keep1 = 3650, - .update = 0, - .compression = TWO_STAGE_COMP}; - -int tsdbValidateOptions(const STsdbCfg *pTsdbOptions) { - // TODO - return 0; -} - -void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc) { memcpy(pDest, pSrc, sizeof(STsdbCfg)); } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 483475601b..302ee89e51 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -98,24 +98,24 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct STsdbReadHandle { - STsdb* pTsdb; - SQueryFilePos cur; // current position - int16_t order; - STimeWindow window; // the primary query time window that applies to all queries + STsdb* pTsdb; + SQueryFilePos cur; // current position + int16_t order; + STimeWindow window; // the primary query time window that applies to all queries SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time - int32_t numOfBlocks; - SArray* pColumns; // column list, SColumnInfoData array list - bool locateStart; - int32_t outputCapacity; - int32_t realNumOfRows; - SArray* pTableCheckInfo; // SArray - int32_t activeIndex; - bool checkFiles; // check file stage - int8_t cachelastrow; // check if last row cached - bool loadExternalRow; // load time window external data rows - bool currentLoadExternalRows; // current load external rows - int32_t loadType; // block load type - char* idStr; // query info handle, for debug purpose + int32_t numOfBlocks; + SArray* pColumns; // column list, SColumnInfoData array list + bool locateStart; + int32_t outputCapacity; + int32_t realNumOfRows; + SArray* pTableCheckInfo; // SArray + int32_t activeIndex; + bool checkFiles; // check file stage + int8_t cachelastrow; // check if last row cached + bool loadExternalRow; // load time window external data rows + bool currentLoadExternalRows; // current load external rows + int32_t loadType; // block load type + char* idStr; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows SDFileSet* pFileGroup; SFSIter fileIter; @@ -254,7 +254,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S assert(info.lastKey >= pTsdbReadHandle->window.skey && info.lastKey <= pTsdbReadHandle->window.ekey); } else { - assert(info.lastKey >= pTsdbReadHandle->window.ekey && info.lastKey <= pTsdbReadHandle->window.skey); + info.lastKey = pTsdbReadHandle->window.skey; } taosArrayPush(pTableCheckInfo, &info); @@ -317,7 +317,7 @@ static int64_t getEarliestValidTimestamp(STsdb* pTsdb) { return now - (tsTickPerDay[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick } -static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* pCond) { +static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableDataCond* pCond) { pTsdbReadHandle->window = pCond->twindow; bool updateTs = false; @@ -343,7 +343,7 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* } } -static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, uint64_t taskId) { +static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) { STsdbReadHandle* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReadHandle)); if (pReadHandle == NULL) { goto _end; @@ -422,7 +422,7 @@ _end: return NULL; } -tsdbReaderT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, +tsdbReaderT* tsdbQueryTables(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { @@ -448,7 +448,7 @@ tsdbReaderT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo return (tsdbReaderT)pTsdbReadHandle; } -void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond* pCond) { +void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond) { STsdbReadHandle* pTsdbReadHandle = queryHandle; if (emptyQueryTimewindow(pTsdbReadHandle)) { @@ -485,7 +485,7 @@ void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond* pCond) { resetCheckInfo(pTsdbReadHandle); } -void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond* pCond, STableGroupInfo* groupList) { +void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, SQueryTableDataCond* pCond, STableGroupInfo* groupList) { STsdbReadHandle* pTsdbReadHandle = queryHandle; pTsdbReadHandle->order = pCond->order; @@ -526,7 +526,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond* pC // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, +tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { pCond->twindow = updateLastrowForEachGroup(groupList); @@ -555,7 +555,7 @@ tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo } #if 0 -tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { +tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); if (pTsdbReadHandle == NULL) { return NULL; @@ -618,8 +618,8 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, - uint64_t taskId) { +tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, + uint64_t qId, uint64_t taskId) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); if (pNew->numOfTables == 0) { @@ -1185,10 +1185,12 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* tsdbDebug("%p no data in mem, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); } - if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || - (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { - if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || - (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { + bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order); + + if ((ascScan && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || + (!ascScan && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { + if ((ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || + (!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { // do not load file block into buffer int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; @@ -1225,8 +1227,8 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* assert(pTsdbReadHandle->outputCapacity >= binfo.rows); int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &binfo); - if ((cur->pos == 0 && endPos == binfo.rows - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)))) { + if ((cur->pos == 0 && endPos == binfo.rows - 1 && ascScan) || + (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ascScan))) { pTsdbReadHandle->realNumOfRows = binfo.rows; cur->rows = binfo.rows; @@ -1234,7 +1236,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* cur->mixBlock = false; cur->blockCompleted = true; - if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (ascScan) { cur->lastKey = binfo.window.ekey + 1; cur->pos = binfo.rows; } else { @@ -1382,8 +1384,6 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { - int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; - SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; @@ -1394,6 +1394,11 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t return numOfRows; } + bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order); + int32_t trueStart = ascScan ? start : end; + int32_t trueEnd = ascScan ? end : start; + int32_t step = ascScan ? 1 : -1; + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); // data in buffer has greater timestamp, copy data in file block @@ -1411,7 +1416,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t if (!IS_VAR_DATA_TYPE(pColInfo->info.type)) { // todo opt performance // memmove(pData, (char*)src->pData + bytes * start, bytes * num); int32_t rowIndex = numOfRows; - for (int32_t k = start; k <= end; ++k, ++rowIndex) { + for (int32_t k = trueStart; ((ascScan && k <= trueEnd) || (!ascScan && k >= trueEnd)); k += step, ++rowIndex) { SCellVal sVal = {0}; if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) { TASSERT(0); @@ -1427,7 +1432,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t int32_t rowIndex = numOfRows; // todo refactor, only copy one-by-one - for (int32_t k = start; k < num + start; ++k, ++rowIndex) { + for (int32_t k = trueStart; ((ascScan && k <= trueEnd) || (!ascScan && k >= trueEnd)); k += step, ++rowIndex) { SCellVal sVal = {0}; if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) { TASSERT(0); @@ -1443,27 +1448,20 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t j++; i++; - } else { // pColInfo->info.colId < src->colId, it is a NULL data - int32_t rowIndex = numOfRows; - for (int32_t k = start; k < num + start; ++k, ++rowIndex) { // TODO opt performance - colDataAppend(pColInfo, rowIndex, NULL, true); - } + } else { // pColInfo->info.colId < src->colId, it is a NULL data + colDataAppendNNULL(pColInfo, numOfRows, num); i++; } } while (i < requiredNumOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - int32_t rowIndex = numOfRows; - - for (int32_t k = start; k < num + start; ++k, ++rowIndex) { - colDataAppend(pColInfo, rowIndex, NULL, true); // TODO add a fast version to set a number of consecutive NULL value. - } + colDataAppendNNULL(pColInfo, numOfRows, num); i++; } - pTsdbReadHandle->cur.win.ekey = tsArray[end]; - pTsdbReadHandle->cur.lastKey = tsArray[end] + step; + pTsdbReadHandle->cur.win.ekey = tsArray[trueEnd]; + pTsdbReadHandle->cur.lastKey = tsArray[trueEnd] + step; return numOfRows + num; } @@ -1777,7 +1775,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf STable* pTable = NULL; int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); - tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64 " rows:%d, start:%d, end:%d, %s", + tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64 + " rows:%d, start:%d, end:%d, %s", pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->idStr); @@ -2966,7 +2965,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // } // // // load the previous row -// STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; +// SQueryTableDataCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; // if (type == TSDB_PREV_ROW) { // cond.order = TSDB_ORDER_DESC; // cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MIN}; @@ -3223,7 +3222,7 @@ void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDa tsdbDebug("data block generated, uid:%" PRIu64 " numOfRows:%d, tsrange:%" PRId64 " - %" PRId64 " %s", uid, cur->rows, cur->win.skey, cur->win.ekey, pHandle->idStr); - pDataBlockInfo->uid = uid; + pDataBlockInfo->uid = uid; #if 0 // for multi-group data query processing test purpose @@ -3330,21 +3329,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { return NULL; } - // todo refactor int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1); - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pHandle->order) && numOfRows < pHandle->outputCapacity) { - int32_t emptySize = pHandle->outputCapacity - numOfRows; - int32_t reqNumOfCols = (int32_t)taosArrayGetSize(pHandle->pColumns); - - for (int32_t i = 0; i < reqNumOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i); - memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, - numOfRows * pColInfo->info.bytes); - } - } - return pHandle->pColumns; } } @@ -3631,8 +3616,11 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) { - STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid); - if (pTbCfg == NULL) { + SMetaReader mr = {0}; + + metaReaderInit(&mr, (SMeta*)pMeta, 0); + + if (metaGetTableEntryByUid(&mr, uid) < 0) { tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; goto _error; @@ -3640,13 +3628,15 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); } - if (pTbCfg->type != META_SUPER_TABLE) { + if (mr.me.type != TSDB_SUPER_TABLE) { tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); terrno = TSDB_CODE_OPS_NOT_SUPPORT; // basically, this error is caused by invalid sql issued by client goto _error; } + metaReaderClear(&mr); + // NOTE: not add ref count for super table SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true); @@ -3697,12 +3687,17 @@ int32_t tsdbQueryTableList(void* pMeta, SArray* pRes, void* filterInfo) { return TSDB_CODE_SUCCESS; } int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { - STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid); - if (pTbCfg == NULL) { + SMetaReader mr = {0}; + + metaReaderInit(&mr, (SMeta*)pMeta, 0); + + if (metaGetTableEntryByUid(&mr, uid) < 0) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; goto _error; } + metaReaderClear(&mr); + pGroupInfo->numOfTables = 1; pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); @@ -3715,6 +3710,7 @@ int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGr return TSDB_CODE_SUCCESS; _error: + metaReaderClear(&mr); return terrno; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index f18f36e07b..c39780372b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" #define TSDB_KEY_COL_OFFSET 0 diff --git a/source/dnode/vnode/src/tsdb/tsdbScan.c b/source/dnode/vnode/src/tsdb/tsdbScan.c index c0e468e640..cc6fb473d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbScan.c +++ b/source/dnode/vnode/src/tsdb/tsdbScan.c @@ -13,9 +13,8 @@ * along with this program. If not, see . */ - #if 0 -#include "tsdbint.h" +#include "tsdb.h" #ifndef _TSDB_PLUGINS int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 806512f07d..c9e33cefc7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -13,7 +13,8 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdbSma.h" +#include "tsdb.h" static const char *TSDB_SMA_DNAME[] = { "", // TSDB_SMA_TYPE_BLOCK @@ -678,7 +679,6 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers return TSDB_CODE_FAILED; } - if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; @@ -1693,6 +1693,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { * @return int32_t */ int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) { +#if 0 SRSmaParam *param = pReq->stbCfg.pRSmaParam; if (!param) { @@ -1764,6 +1765,7 @@ int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) { tsdbDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, REPO_ID(pTsdb), pReq->stbCfg.suid); } +#endif return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index ebfa1ecaeb..ac3baf8c3a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -15,14 +15,14 @@ #define ALLOW_FORBID_FUNC -#include "vnodeInt.h" +#include "tsdb.h" int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { - int ret = 0; + int ret = 0; if (path == NULL) return -1; - ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param if (ret != 0) { tsdbError("Failed to create tsdb db env, ret = %d", ret); diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 910b5adc96..82fa6b9ae5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -13,23 +13,113 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" -/** - * @brief insert TS data - * - * @param pTsdb - * @param pMsg - * @param pRsp - * @return int - */ -int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { - // Check if mem is there. If not, create one. - if (pTsdb->mem == NULL) { - pTsdb->mem = tsdbNewMemTable(pTsdb); - if (pTsdb->mem == NULL) { +static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); + +int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + int32_t affectedrows = 0; + int32_t numOfRows = 0; + + ASSERT(pTsdb->mem != NULL); + + // scan and convert + if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { + if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { + tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return -1; + } + + // loop to insert + tInitSubmitMsgIter(pMsg, &msgIter); + while (true) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (pBlock == NULL) break; + if (tsdbInsertTableData(pTsdb, pBlock, &affectedrows) < 0) { return -1; } + + numOfRows += pBlock->numOfRows; } - return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, pRsp); + + if (pRsp != NULL) { + pRsp->affectedRows = affectedrows; + pRsp->numOfRows = numOfRows; + } + + return 0; +} + +static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { + ASSERT(pMsg != NULL); + // STsdbMeta * pMeta = pTsdb->tsdbMeta; + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SSubmitBlkIter blkIter = {0}; + STSRow *row = NULL; + TSKEY now = taosGetTimestamp(pTsdb->config.precision); + TSKEY minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep2; + TSKEY maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.days; + + terrno = TSDB_CODE_SUCCESS; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; + while (true) { + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + if (pBlock == NULL) break; + + pBlock->uid = htobe64(pBlock->uid); + pBlock->suid = htobe64(pBlock->suid); + pBlock->sversion = htonl(pBlock->sversion); + pBlock->dataLen = htonl(pBlock->dataLen); + pBlock->schemaLen = htonl(pBlock->schemaLen); + pBlock->numOfRows = htons(pBlock->numOfRows); + +#if 0 + if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { + tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, + pBlock->tid); + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + return -1; + } + + STable *pTable = pMeta->tables[pBlock->tid]; + if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) { + tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, + pBlock->tid); + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + return -1; + } + + if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { + tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable)); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + return -1; + } + + // Check schema version and update schema if needed + if (tsdbCheckTableSchema(pTsdb, pBlock, pTable) < 0) { + if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + continue; + } else { + return -1; + } + } + + tsdbInitSubmitBlkIter(pBlock, &blkIter); + while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { + if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) { + return -1; + } + } +#endif + } + + if (terrno != TSDB_CODE_SUCCESS) return -1; + return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c deleted file mode 100644 index 7b7f6c9157..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeInt.h" - -static SVArenaNode *vArenaNodeNew(uint64_t capacity); -static void vArenaNodeFree(SVArenaNode *pNode); - -SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { - SVMemAllocator *pVMA = (SVMemAllocator *)taosMemoryMalloc(sizeof(*pVMA)); - if (pVMA == NULL) { - return NULL; - } - - pVMA->capacity = capacity; - pVMA->ssize = ssize; - pVMA->lsize = lsize; - TD_SLIST_INIT(&(pVMA->nlist)); - - pVMA->pNode = vArenaNodeNew(capacity); - if (pVMA->pNode == NULL) { - taosMemoryFree(pVMA); - return NULL; - } - - TD_SLIST_PUSH(&(pVMA->nlist), pVMA->pNode); - - return pVMA; -} - -void vmaDestroy(SVMemAllocator *pVMA) { - if (pVMA) { - while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { - SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - TD_SLIST_POP(&(pVMA->nlist)); - vArenaNodeFree(pNode); - } - - taosMemoryFree(pVMA); - } -} - -void vmaReset(SVMemAllocator *pVMA) { - while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { - SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - TD_SLIST_POP(&(pVMA->nlist)); - vArenaNodeFree(pNode); - } - - SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - pNode->ptr = pNode->data; -} - -void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { - SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - void * ptr; - - if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { - uint64_t capacity = TMAX(pVMA->ssize, size); - pNode = vArenaNodeNew(capacity); - if (pNode == NULL) { - // TODO: handle error - return NULL; - } - - TD_SLIST_PUSH(&(pVMA->nlist), pNode); - } - - ptr = pNode->ptr; - pNode->ptr = POINTER_SHIFT(ptr, size); - - return ptr; -} - -void vmaFree(SVMemAllocator *pVMA, void *ptr) { - // TODO -} - -bool vmaIsFull(SVMemAllocator *pVMA) { - SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - - return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) || - (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); -} - -/* ------------------------ STATIC METHODS ------------------------ */ -static SVArenaNode *vArenaNodeNew(uint64_t capacity) { - SVArenaNode *pNode = NULL; - - pNode = (SVArenaNode *)taosMemoryMalloc(sizeof(*pNode) + capacity); - if (pNode == NULL) { - return NULL; - } - - pNode->size = capacity; - pNode->ptr = pNode->data; - - return pNode; -} - -static void vArenaNodeFree(SVArenaNode *pNode) { - if (pNode) { - taosMemoryFree(pNode); - } -} diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool2.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c similarity index 90% rename from source/dnode/vnode/src/vnd/vnodeBufferPool2.c rename to source/dnode/vnode/src/vnd/vnodeBufPool.c index d63c86734a..9122913cda 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool2.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" /* ------------------------ STRUCTURES ------------------------ */ -static int vnodeBufPoolCreate(int size, SVBufPool **ppPool); +static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool); static int vnodeBufPoolDestroy(SVBufPool *pPool); int vnodeOpenBufPool(SVnode *pVnode, int64_t size) { @@ -30,17 +30,17 @@ int vnodeOpenBufPool(SVnode *pVnode, int64_t size) { // create pool ret = vnodeBufPoolCreate(size, &pPool); if (ret < 0) { - vError("vgId:%d failed to open vnode buffer pool since %s", TD_VNODE_ID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); vnodeCloseBufPool(pVnode); return -1; } - // add pool to queue + // add pool to vnode pPool->next = pVnode->pPool; pVnode->pPool = pPool; } - vDebug("vgId:%d vnode buffer pool is opened, pool size: %" PRId64, TD_VNODE_ID(pVnode), size); + vDebug("vgId:%d vnode buffer pool is opened, pool size: %" PRId64, TD_VID(pVnode), size); return 0; } @@ -53,7 +53,7 @@ int vnodeCloseBufPool(SVnode *pVnode) { vnodeBufPoolDestroy(pPool); } - vDebug("vgId:%d vnode buffer pool is closed", TD_VNODE_ID(pVnode)); + vDebug("vgId:%d vnode buffer pool is closed", TD_VID(pVnode)); return 0; } @@ -75,7 +75,7 @@ void vnodeBufPoolReset(SVBufPool *pPool) { pPool->ptr = pPool->node.data; } -void *vnodeBufPoolMalloc(SVBufPool *pPool, size_t size) { +void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { SVBufPoolNode *pNode; void *p; @@ -120,7 +120,7 @@ void vnodeBufPoolFree(SVBufPool *pPool, void *p) { } // STATIC METHODS ------------------- -static int vnodeBufPoolCreate(int size, SVBufPool **ppPool) { +static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool) { SVBufPool *pPool; pPool = taosMemoryMalloc(sizeof(SVBufPool) + size); diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c deleted file mode 100644 index 8764950f27..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeInt.h" - -/* ------------------------ STRUCTURES ------------------------ */ -#define VNODE_BUF_POOL_SHARDS 3 - -struct SVBufPool { - TdThreadMutex mutex; - TdThreadCond hasFree; - TD_DLIST(SVMemAllocator) free; - TD_DLIST(SVMemAllocator) incycle; - SVMemAllocator *inuse; - // MAF for submodules to use - SMemAllocatorFactory *pMAF; -}; - -static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF); -static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA); - -int vnodeOpenBufPool(SVnode *pVnode) { - uint64_t capacity; - - if ((pVnode->pBufPool = (SVBufPool *)taosMemoryCalloc(1, sizeof(SVBufPool))) == NULL) { - /* TODO */ - return -1; - } - - TD_DLIST_INIT(&(pVnode->pBufPool->free)); - TD_DLIST_INIT(&(pVnode->pBufPool->incycle)); - - pVnode->pBufPool->inuse = NULL; - - // TODO - capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; - - for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { - SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize); - if (pVMA == NULL) { - // TODO: handle error - return -1; - } - - TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA); - } - - pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)taosMemoryMalloc(sizeof(SMemAllocatorFactory)); - if (pVnode->pBufPool->pMAF == NULL) { - // TODO: handle error - return -1; - } - pVnode->pBufPool->pMAF->impl = pVnode; - pVnode->pBufPool->pMAF->create = vBufPoolCreateMA; - pVnode->pBufPool->pMAF->destroy = vBufPoolDestroyMA; - - return 0; -} - -void vnodeCloseBufPool(SVnode *pVnode) { - if (pVnode->pBufPool) { - taosMemoryFreeClear(pVnode->pBufPool->pMAF); - vmaDestroy(pVnode->pBufPool->inuse); - - while (true) { - SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle)); - if (pVMA == NULL) break; - TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA); - vmaDestroy(pVMA); - } - - while (true) { - SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free)); - if (pVMA == NULL) break; - TD_DLIST_POP(&(pVnode->pBufPool->free), pVMA); - vmaDestroy(pVMA); - } - - taosMemoryFree(pVnode->pBufPool); - pVnode->pBufPool = NULL; - } -} - -int vnodeBufPoolSwitch(SVnode *pVnode) { - SVMemAllocator *pvma = pVnode->pBufPool->inuse; - - pVnode->pBufPool->inuse = NULL; - - if (pvma) { - TD_DLIST_APPEND(&(pVnode->pBufPool->incycle), pvma); - } - return 0; -} - -int vnodeBufPoolRecycle(SVnode *pVnode) { - SVBufPool * pBufPool = pVnode->pBufPool; - SVMemAllocator *pvma = TD_DLIST_HEAD(&(pBufPool->incycle)); - if (pvma == NULL) return 0; - // ASSERT(pvma != NULL); - - TD_DLIST_POP(&(pBufPool->incycle), pvma); - vmaReset(pvma); - TD_DLIST_APPEND(&(pBufPool->free), pvma); - - return 0; -} - -void *vnodeMalloc(SVnode *pVnode, uint64_t size) { - SVBufPool *pBufPool = pVnode->pBufPool; - - if (pBufPool->inuse == NULL) { - while (true) { - // TODO: add sem_wait and sem_post - pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free)); - if (pBufPool->inuse) { - TD_DLIST_POP(&(pBufPool->free), pBufPool->inuse); - break; - } else { - // tsem_wait(&(pBufPool->hasFree)); - } - } - } - - return vmaMalloc(pBufPool->inuse, size); -} - -bool vnodeBufPoolIsFull(SVnode *pVnode) { - if (pVnode->pBufPool->inuse == NULL) return false; - return vmaIsFull(pVnode->pBufPool->inuse); -} - -SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode) { return pVnode->pBufPool->pMAF; } - -/* ------------------------ STATIC METHODS ------------------------ */ -typedef struct { - SVnode * pVnode; - SVMemAllocator *pVMA; -} SVMAWrapper; - -static FORCE_INLINE void *vmaMaloocCb(SMemAllocator *pMA, uint64_t size) { - SVMAWrapper *pWrapper = (SVMAWrapper *)(pMA->impl); - - return vmaMalloc(pWrapper->pVMA, size); -} - -// TODO: Add atomic operations here -static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) { - SMemAllocator *pMA; - SVnode * pVnode = (SVnode *)(pMAF->impl); - SVMAWrapper * pWrapper; - - pMA = (SMemAllocator *)taosMemoryCalloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); - if (pMA == NULL) { - return NULL; - } - - pVnode->pBufPool->inuse->_ref.val++; - pWrapper = POINTER_SHIFT(pMA, sizeof(*pMA)); - pWrapper->pVnode = pVnode; - pWrapper->pVMA = pVnode->pBufPool->inuse; - - pMA->impl = pWrapper; - TD_MA_MALLOC_FUNC(pMA) = vmaMaloocCb; - - return pMA; -} - -static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) { - SVMAWrapper * pWrapper = (SVMAWrapper *)(pMA->impl); - SVnode * pVnode = pWrapper->pVnode; - SVMemAllocator *pVMA = pWrapper->pVMA; - - taosMemoryFree(pMA); - if (--pVMA->_ref.val == 0) { - TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA); - vmaReset(pVMA); - TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA); - } -} diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 714497e786..df8cf8d503 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -13,23 +13,24 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" const SVnodeCfg vnodeCfgDefault = { .vgId = -1, .dbname = "", .dbId = 0, - .wsize = 96 * 1024 * 1024, - .ssize = 1 * 1024 * 1024, - .lsize = 1024, - .isHeapAllocator = false, + .szPage = 4096, + .szCache = 256, + .szBuf = 96 * 1024 * 1024, + .isHeap = false, .ttl = 0, .keep = 0, .streamMode = 0, .isWeak = 0, - .tsdbCfg = {.precision = TWO_STAGE_COMP, + .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI, .update = 0, .compression = 2, + .slLevel = 5, .days = 10, .minRows = 100, .maxRows = 4096, @@ -53,10 +54,10 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1; if (tjsonAddStringToObject(pJson, "dbname", pCfg->dbname) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "wsize", pCfg->wsize) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "ssize", pCfg->ssize) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "lsize", pCfg->lsize) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "ttl", pCfg->ttl) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep", pCfg->keep) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1; @@ -64,6 +65,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1; @@ -103,10 +105,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "vgId", pCfg->vgId) < 0) return -1; if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1; if (tjsonGetNumberValue(pJson, "dbId", pCfg->dbId) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wsize", pCfg->wsize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "ssize", pCfg->ssize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "lsize", pCfg->lsize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1; + if (tjsonGetNumberValue(pJson, "szPage", pCfg->szPage) < 0) return -1; + if (tjsonGetNumberValue(pJson, "szCache", pCfg->szCache) < 0) return -1; + if (tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf) < 0) return -1; + if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap) < 0) return -1; if (tjsonGetNumberValue(pJson, "ttl", pCfg->ttl) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep", pCfg->keep) < 0) return -1; if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1; @@ -114,6 +116,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; if (tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; if (tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; + if (tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1; if (tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1; if (tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1; if (tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1; @@ -150,10 +153,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { return 0; } -int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) { +int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) { uint32_t hashValue = 0; - switch (pVnodeOptions->hashMethod) { + switch (pVnode->config.hashMethod) { default: hashValue = MurmurHash3_32(tableFName, strlen(tableFName)); break; @@ -167,5 +170,5 @@ int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) { } #endif - return TSDB_CODE_SUCCESS; + return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index a034b13fbe..1c0e65aa95 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" @@ -22,9 +22,41 @@ static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo); static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); -static int vnodeCommit(void *arg); +static int vnodeCommitImpl(void *arg); static void vnodeWaitCommit(SVnode *pVnode); +int vnodeBegin(SVnode *pVnode) { + // alloc buffer pool + /* pthread_mutex_lock(); */ + + while (pVnode->pPool == NULL) { + /* pthread_cond_wait(); */ + } + + pVnode->inUse = pVnode->pPool; + pVnode->pPool = pVnode->inUse->next; + pVnode->inUse->next = NULL; + /* ref pVnode->inUse buffer pool */ + + /* pthread_mutex_unlock(); */ + + // begin meta + if (metaBegin(pVnode->pMeta) < 0) { + vError("vgId: %d failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + // begin tsdb + if (tsdbBegin(pVnode->pTsdb) < 0) { + vError("vgId: %d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + return 0; +} + +int vnodeShouldCommit(SVnode *pVnode) { return pVnode->inUse->size > pVnode->config.szBuf / 3; } + int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { char fname[TSDB_FILENAME_LEN]; TdFilePtr pFile; @@ -142,10 +174,10 @@ _err: int vnodeAsyncCommit(SVnode *pVnode) { vnodeWaitCommit(pVnode); - vnodeBufPoolSwitch(pVnode); - tsdbPrepareCommit(pVnode->pTsdb); + // vnodeBufPoolSwitch(pVnode); + // tsdbPrepareCommit(pVnode->pTsdb); - vnodeScheduleTask(vnodeCommit, pVnode); + vnodeScheduleTask(vnodeCommitImpl, pVnode); return 0; } @@ -157,25 +189,64 @@ int vnodeSyncCommit(SVnode *pVnode) { return 0; } -static int vnodeCommit(void *arg) { - SVnode *pVnode = (SVnode *)arg; +int vnodeCommit(SVnode *pVnode) { + SVnodeInfo info; char dir[TSDB_FILENAME_LEN]; - SVnodeInfo info = {0}; - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + vInfo("vgId:%d start to commit, version: %" PRId64, TD_VID(pVnode), pVnode->state.applied); + + pVnode->onCommit = pVnode->inUse; + pVnode->inUse = NULL; + + // save info info.config = pVnode->config; info.state.committed = pVnode->state.applied; - info.state.applied = pVnode->state.applied; + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + if (vnodeSaveInfo(dir, &info) < 0) { + ASSERT(0); + return -1; + } - vnodeSaveInfo(dir, &info); + // commit each sub-system + if (metaCommit(pVnode->pMeta) < 0) { + ASSERT(0); + return -1; + } + if (tsdbCommit(pVnode->pTsdb) < 0) { + ASSERT(0); + return -1; + } + if (tqCommit(pVnode->pTq) < 0) { + ASSERT(0); + return -1; + } + // walCommit (TODO) + + // commit info + if (vnodeCommitInfo(dir, &info) < 0) { + ASSERT(0); + return -1; + } + + // apply the commit (TODO) + vnodeBufPoolReset(pVnode->onCommit); + pVnode->onCommit->next = pVnode->pPool; + pVnode->pPool = pVnode->onCommit; + pVnode->onCommit = NULL; + + vInfo("vgId:%d commit over", TD_VID(pVnode)); + + return 0; +} + +static int vnodeCommitImpl(void *arg) { + SVnode *pVnode = (SVnode *)arg; // metaCommit(pVnode->pMeta); tqCommit(pVnode->pTq); tsdbCommit(pVnode->pTsdb); - vnodeCommitInfo(dir, &info); - - vnodeBufPoolRecycle(pVnode); + // vnodeBufPoolRecycle(pVnode); tsem_post(&(pVnode->canCommit)); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeInt.c b/source/dnode/vnode/src/vnd/vnodeInt.c index 10d8154a15..14dfc75ced 100644 --- a/source/dnode/vnode/src/vnd/vnodeInt.c +++ b/source/dnode/vnode/src/vnd/vnodeInt.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "vnodeInt.h" +#include "vnd.h" // #include "vnodeInt.h" int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 2b5b46a45d..efae74b55a 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" typedef struct SVnodeTask SVnodeTask; struct SVnodeTask { diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 11136bc7ba..ee64976df9 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" -#include "vnodeSync.h" +#include "vnd.h" int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { SVnodeInfo info = {0}; @@ -85,7 +84,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { tsem_init(&(pVnode->canCommit), 0, 1); // open buffer pool - if (vnodeOpenBufPool(pVnode) < 0) { + if (vnodeOpenBufPool(pVnode, pVnode->config.isHeap ? 0 : pVnode->config.szBuf / 3) < 0) { vError("vgId: %d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } @@ -97,9 +96,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { } // open tsdb - sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TSDB_DIR); - pVnode->pTsdb = tsdbOpen(tdir, pVnode, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode)); - if (pVnode->pTsdb == NULL) { + if (tsdbOpen(pVnode, &pVnode->pTsdb) < 0) { vError("vgId: %d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } @@ -126,17 +123,17 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { goto _err; } - // sync integration - // open sync - if (vnodeSyncOpen(pVnode, dir)) { + // vnode begin + if (vnodeBegin(pVnode) < 0) { + vError("vgId: %d failed to begin since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } -#if 0 - if (vnodeBegin() < 0) { + // open sync + if (vnodeSyncOpen(pVnode, dir)) { + vError("vgId: %d failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } -#endif return pVnode; @@ -153,14 +150,9 @@ _err: void vnodeClose(SVnode *pVnode) { if (pVnode) { - // commit (TODO: use option to control) - vnodeSyncCommit(pVnode); - // close vnode - vnodeQueryClose(pVnode); - - // sync integration + vnodeCommit(pVnode); vnodeSyncClose(pVnode); - + vnodeQueryClose(pVnode); walClose(pVnode->pWal); tqClose(pVnode->pTq); tsdbClose(pVnode->pTsdb); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 6280542ffe..5d11494d57 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, TD_VID(pVnode), NULL, (void **)&pVnode->pQuery, &pVnode->msgCb); @@ -22,94 +22,76 @@ int vnodeQueryOpen(SVnode *pVnode) { void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); } int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { - STbCfg * pTbCfg = NULL; - STbCfg * pStbCfg = NULL; - tb_uid_t uid; - int32_t nCols; - int32_t nTagCols; - SSchemaWrapper *pSW = NULL; - STableMetaRsp * pTbMetaMsg = NULL; - STableMetaRsp metaRsp = {0}; - SSchema * pTagSchema; - SRpcMsg rpcMsg; - int msgLen = 0; - int32_t code = 0; - char tableFName[TSDB_TABLE_FNAME_LEN]; - int32_t rspLen = 0; - void * pRsp = NULL; + STableInfoReq infoReq = {0}; + STableMetaRsp metaRsp = {0}; + SMetaReader mer1 = {0}; + SMetaReader mer2 = {0}; + char tableFName[TSDB_TABLE_FNAME_LEN]; + SRpcMsg rpcMsg; + int32_t code = 0; + int32_t rspLen = 0; + void *pRsp = NULL; + SSchemaWrapper schema = {0}; + SSchemaWrapper schemaTag = {0}; - STableInfoReq infoReq = {0}; + // decode req if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { code = TSDB_CODE_INVALID_MSG; goto _exit; } metaRsp.dbId = pVnode->config.dbId; - memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); strcpy(metaRsp.tbName, infoReq.tbName); + memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName); - code = vnodeValidateTableHash(&pVnode->config, tableFName); + code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit; } - pTbCfg = metaGetTbInfoByName(pVnode->pMeta, infoReq.tbName, &uid); - if (pTbCfg == NULL) { - code = TSDB_CODE_VND_TB_NOT_EXIST; + // query meta + metaReaderInit(&mer1, pVnode->pMeta, 0); + + if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) { goto _exit; } - if (pTbCfg->type == META_CHILD_TABLE) { - pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid); - if (pStbCfg == NULL) { - code = TSDB_CODE_VND_TB_NOT_EXIST; - goto _exit; - } - - pSW = metaGetTableSchema(pVnode->pMeta, pTbCfg->ctbCfg.suid, 0, true); - } else { - pSW = metaGetTableSchema(pVnode->pMeta, uid, 0, true); - } - - nCols = pSW->nCols; - if (pTbCfg->type == META_SUPER_TABLE) { - nTagCols = pTbCfg->stbCfg.nTagCols; - pTagSchema = pTbCfg->stbCfg.pTagSchema; - } else if (pTbCfg->type == META_CHILD_TABLE) { - nTagCols = pStbCfg->stbCfg.nTagCols; - pTagSchema = pStbCfg->stbCfg.pTagSchema; - } else { - nTagCols = 0; - pTagSchema = NULL; - } - - metaRsp.pSchemas = taosMemoryCalloc(nCols + nTagCols, sizeof(SSchema)); - if (metaRsp.pSchemas == NULL) { - code = TSDB_CODE_VND_OUT_OF_MEMORY; - goto _exit; - } - - if (pTbCfg->type == META_CHILD_TABLE) { - strcpy(metaRsp.stbName, pStbCfg->name); - metaRsp.suid = pTbCfg->ctbCfg.suid; - } else if (pTbCfg->type == META_SUPER_TABLE) { - strcpy(metaRsp.stbName, pTbCfg->name); - metaRsp.suid = uid; - } - metaRsp.numOfTags = nTagCols; - metaRsp.numOfColumns = nCols; - metaRsp.tableType = pTbCfg->type; - metaRsp.tuid = uid; + metaRsp.tableType = mer1.me.type; metaRsp.vgId = TD_VID(pVnode); + metaRsp.tuid = mer1.me.uid; - memcpy(metaRsp.pSchemas, pSW->pSchema, sizeof(SSchema) * pSW->nCols); - if (nTagCols) { - memcpy(POINTER_SHIFT(metaRsp.pSchemas, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols); + if (mer1.me.type == TSDB_SUPER_TABLE) { + strcpy(metaRsp.stbName, mer1.me.name); + schema = mer1.me.stbEntry.schema; + schemaTag = mer1.me.stbEntry.schemaTag; + metaRsp.suid = mer1.me.uid; + } else if (mer1.me.type == TSDB_CHILD_TABLE) { + metaReaderInit(&mer2, pVnode->pMeta, 0); + if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + + strcpy(metaRsp.stbName, mer2.me.name); + metaRsp.suid = mer2.me.uid; + schema = mer2.me.stbEntry.schema; + schemaTag = mer2.me.stbEntry.schemaTag; + } else if (mer1.me.type == TSDB_NORMAL_TABLE) { + schema = mer1.me.ntbEntry.schema; + } else { + ASSERT(0); } -_exit: + metaRsp.numOfTags = schemaTag.nCols; + metaRsp.numOfColumns = schema.nCols; + metaRsp.precision = pVnode->config.tsdbCfg.precision; + metaRsp.sversion = schema.sver; + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags)); + memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); + if (schemaTag.nCols) { + memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); + } + + // encode and send response rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); if (rspLen < 0) { code = TSDB_CODE_INVALID_MSG; @@ -123,23 +105,6 @@ _exit: } tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); - tFreeSTableMetaRsp(&metaRsp); - if (pSW != NULL) { - taosMemoryFreeClear(pSW->pSchema); - taosMemoryFreeClear(pSW); - } - - if (pTbCfg) { - taosMemoryFreeClear(pTbCfg->name); - if (pTbCfg->type == META_SUPER_TABLE) { - taosMemoryFree(pTbCfg->stbCfg.pTagSchema); - } else if (pTbCfg->type == META_SUPER_TABLE) { - kvRowFree(pTbCfg->ctbCfg.pTag); - } - - taosMemoryFreeClear(pTbCfg); - } - rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; rpcMsg.refId = pMsg->refId; @@ -148,12 +113,18 @@ _exit: rpcMsg.code = code; tmsgSendRsp(&rpcMsg); - return TSDB_CODE_SUCCESS; + +_exit: + taosMemoryFree(metaRsp.pSchemas); + metaReaderClear(&mer2); + metaReaderClear(&mer1); + return code; } int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->vgId = TD_VID(pVnode); - pLoad->syncState = TAOS_SYNC_STATE_LEADER; + //pLoad->syncState = TAOS_SYNC_STATE_LEADER; + pLoad->syncState = syncGetMyRole(pVnode->sync); // sync integration pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTimeSeries = 400; pLoad->totalStorage = 300; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8365fb9c36..efcc82853c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -13,14 +13,15 @@ * along with this program. If not, see . */ -#include "sync.h" -#include "syncTools.h" -#include "vnodeInt.h" +#include "vnd.h" -static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq); -static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg *pRsp); -static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq); -static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp); +static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); +static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); +static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { #if 0 @@ -47,44 +48,55 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { void *ptr = NULL; + void *pReq; + int len; int ret; - if (pVnode->config.streamMode == 0) { - ptr = vnodeMalloc(pVnode, pMsg->contLen); - if (ptr == NULL) { - // TODO: handle error - } + vTrace("vgId: %d start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + version); - // TODO: copy here need to be extended - memcpy(ptr, pMsg->pCont, pMsg->contLen); - } + pVnode->state.applied = version; + + // skip header + pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + len = pMsg->contLen - sizeof(SMsgHead); // todo: change the interface here if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { - // TODO: handle error + vError("vgId: %d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; } switch (pMsg->msgType) { + /* META */ case TDMT_VND_CREATE_STB: - ret = vnodeProcessCreateStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))); - break; - case TDMT_VND_CREATE_TABLE: - pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; - vnodeProcessCreateTbReq(pVnode, pMsg, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pRsp); + if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_STB: - vnodeProcessAlterStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))); + if (vnodeProcessAlterStbReq(pVnode, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_STB: - vTrace("vgId:%d, process drop stb req", TD_VID(pVnode)); + if (vnodeProcessDropStbReq(pVnode, pReq, len, pRsp) < 0) goto _err; + break; + case TDMT_VND_CREATE_TABLE: + if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + break; + case TDMT_VND_ALTER_TABLE: + if (vnodeProcessAlterTbReq(pVnode, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TABLE: + if (vnodeProcessDropTbReq(pVnode, pReq, len, pRsp) < 0) goto _err; break; + case TDMT_VND_CREATE_SMA: { // timeRangeSMA + if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { + // TODO + } + } break; + /* TSDB */ case TDMT_VND_SUBMIT: - tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, ptr, STREAM_DATA_TYPE_SUBMIT_BLOCK); - pRsp->msgType = TDMT_VND_SUBMIT_RSP; - vnodeProcessSubmitReq(pVnode, ptr, pRsp); + if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; break; + /* TQ */ case TDMT_VND_MQ_VG_CHANGE: if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { @@ -101,19 +113,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg 0) < 0) { } } break; - case TDMT_VND_CREATE_SMA: { // timeRangeSMA - if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - // TODO - } - // } break; - // case TDMT_VND_CANCEL_SMA: { // timeRangeSMA - // } break; - // case TDMT_VND_DROP_SMA: { // timeRangeSMA - // if (tsdbDropTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - // // TODO - // } - - } break; case TDMT_VND_ALTER_VNODE: break; default: @@ -121,17 +120,24 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg break; } - pVnode->state.applied = version; + vDebug("vgId: %d process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); - // Check if it needs to commit + // commit if need if (vnodeShouldCommit(pVnode)) { - // tsem_wait(&(pVnode->canCommit)); - if (vnodeAsyncCommit(pVnode) < 0) { - // TODO: handle error - } + vInfo("vgId:%d commit at version %" PRId64, TD_VID(pVnode), version); + // commit current change + vnodeCommit(pVnode); + + // start a new one + vnodeBegin(pVnode); } return 0; + +_err: + vDebug("vgId: %d process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + tstrerror(terrno), version); + return -1; } int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { @@ -193,176 +199,195 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // sync integration int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); - assert(pSyncNode != NULL); + if (syncEnvIsStart()) { + SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); + assert(pSyncNode != NULL); - ESyncState state = syncGetMyRole(pVnode->sync); - SyncTerm currentTerm = syncGetMyTerm(pVnode->sync); + ESyncState state = syncGetMyRole(pVnode->sync); + SyncTerm currentTerm = syncGetMyTerm(pVnode->sync); - SMsgHead *pHead = pMsg->pCont; + SMsgHead *pHead = pMsg->pCont; - char logBuf[512]; - char *syncNodeStr = sync2SimpleStr(pVnode->sync); - snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); - syncRpcMsgLog2(logBuf, pMsg); - taosMemoryFree(syncNodeStr); + char logBuf[512]; + char *syncNodeStr = sync2SimpleStr(pVnode->sync); + snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); + syncRpcMsgLog2(logBuf, pMsg); + taosMemoryFree(syncNodeStr); - SRpcMsg *pRpcMsg = pMsg; + SRpcMsg *pRpcMsg = pMsg; - if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { - SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { + SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); - syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); - syncTimeoutDestroy(pSyncMsg); + syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); + syncTimeoutDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { - SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { + SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); - syncNodeOnPingCb(pSyncNode, pSyncMsg); - syncPingDestroy(pSyncMsg); + syncNodeOnPingCb(pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { - SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { + SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); - syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); - syncPingReplyDestroy(pSyncMsg); + syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); + syncPingReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { - SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { + SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); - syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); - syncClientRequestDestroy(pSyncMsg); + syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + syncClientRequestDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { - SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { + SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); - syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); - syncRequestVoteDestroy(pSyncMsg); + syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { - SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); - syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); - syncRequestVoteReplyDestroy(pSyncMsg); + syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { - SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { + SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); - syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); - syncAppendEntriesDestroy(pSyncMsg); + syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { - SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); - syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); - syncAppendEntriesReplyDestroy(pSyncMsg); + syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); - } else { - vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); - } - - syncNodeRelease(pSyncNode); - - return 0; -} - -static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) { - SVCreateTbReq vCreateTbReq = {0}; - tDeserializeSVCreateTbReq(pReq, &vCreateTbReq); - if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq), NULL) < 0) { - // TODO - return -1; - } - - tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &vCreateTbReq); - - taosMemoryFree(vCreateTbReq.stbCfg.pSchema); - taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); - if (vCreateTbReq.stbCfg.pRSmaParam) { - taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg1); - taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg2); - taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); - } - taosMemoryFree(vCreateTbReq.name); - - return 0; -} - -static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg *pRsp) { - SVCreateTbBatchReq vCreateTbBatchReq = {0}; - SVCreateTbBatchRsp vCreateTbBatchRsp = {0}; - tDeserializeSVCreateTbBatchReq(pReq, &vCreateTbBatchReq); - int reqNum = taosArrayGetSize(vCreateTbBatchReq.pArray); - - STbDdlH ddlHandle = { - .ahandle = pVnode->pTsdb, - .result = NULL, - .fp = tsdbFetchTbUidList, - }; - - for (int i = 0; i < reqNum; i++) { - SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); - - char tableFName[TSDB_TABLE_FNAME_LEN]; - SMsgHead *pHead = (SMsgHead *)pMsg->pCont; - sprintf(tableFName, "%s.%s", pVnode->config.dbname, pCreateTbReq->name); - - int32_t code = vnodeValidateTableHash(&pVnode->config, tableFName); - if (code) { - SVCreateTbRsp rsp; - rsp.code = code; - - taosArrayPush(vCreateTbBatchRsp.rspList, &rsp); - } - - if (metaCreateTable(pVnode->pMeta, pCreateTbReq, &ddlHandle) < 0) { - // TODO: handle error - vError("vgId:%d, failed to create table: %s", TD_VID(pVnode), pCreateTbReq->name); - } - // TODO: to encapsule a free API - taosMemoryFree(pCreateTbReq->name); - if (pCreateTbReq->type == TD_SUPER_TABLE) { - taosMemoryFree(pCreateTbReq->stbCfg.pSchema); - taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema); - if (pCreateTbReq->stbCfg.pRSmaParam) { - taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam); - } - } else if (pCreateTbReq->type == TD_CHILD_TABLE) { - taosMemoryFree(pCreateTbReq->ctbCfg.pTag); } else { - taosMemoryFree(pCreateTbReq->ntbCfg.pSchema); - if (pCreateTbReq->ntbCfg.pRSmaParam) { - taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam); - } + vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); } + + syncNodeRelease(pSyncNode); + } else { + vError("==vnodeProcessSyncReq== error syncEnv stop"); } - - tsdbUpdateTbUidList(pVnode->pTsdb, ddlHandle.result); - - vTrace("vgId:%d process create %" PRIzu " tables", TD_VID(pVnode), taosArrayGetSize(vCreateTbBatchReq.pArray)); - taosArrayDestroy(vCreateTbBatchReq.pArray); - if (vCreateTbBatchRsp.rspList) { - int32_t contLen = tSerializeSVCreateTbBatchRsp(NULL, 0, &vCreateTbBatchRsp); - void *msg = rpcMallocCont(contLen); - tSerializeSVCreateTbBatchRsp(msg, contLen, &vCreateTbBatchRsp); - taosArrayDestroy(vCreateTbBatchRsp.rspList); - - pRsp->pCont = msg; - pRsp->contLen = contLen; - } - return 0; } -static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) { +static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { + SVCreateStbReq req = {0}; + SCoder coder; + + pRsp->msgType = TDMT_VND_CREATE_STB_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + // decode and process req + tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER); + + if (tDecodeSVCreateStbReq(&coder, &req) < 0) { + pRsp->code = terrno; + goto _err; + } + + // tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &vCreateTbReq); + + if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) { + pRsp->code = terrno; + goto _err; + } + + tCoderClear(&coder); + return 0; + +_err: + tCoderClear(&coder); + return -1; +} + +static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { + SCoder coder = {0}; + int rcode = 0; + SVCreateTbBatchReq req = {0}; + SVCreateTbReq *pCreateReq; + SVCreateTbBatchRsp rsp = {0}; + SVCreateTbRsp cRsp = {0}; + char tbName[TSDB_TABLE_FNAME_LEN]; + + pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + // decode + tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER); + if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) { + rcode = -1; + terrno = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs); + if (rsp.pArray == NULL) { + rcode = -1; + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + // loop to create table + for (int iReq = 0; iReq < req.nReqs; iReq++) { + pCreateReq = req.pReqs + iReq; + + // validate hash + sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name); + if (vnodeValidateTableHash(pVnode, tbName) < 0) { + cRsp.code = TSDB_CODE_VND_HASH_MISMATCH; + taosArrayPush(rsp.pArray, &cRsp); + continue; + } + + // do create table + if (metaCreateTable(pVnode->pMeta, version, pCreateReq) < 0) { + cRsp.code = terrno; + } else { + cRsp.code = TSDB_CODE_SUCCESS; + } + + taosArrayPush(rsp.pArray, &cRsp); + } + + tCoderClear(&coder); + + // prepare rsp + tEncodeSize(tEncodeSVCreateTbBatchRsp, &rsp, pRsp->contLen); + pRsp->pCont = rpcMallocCont(pRsp->contLen); + if (pRsp->pCont == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + rcode = -1; + goto _exit; + } + tCoderInit(&coder, TD_LITTLE_ENDIAN, pRsp->pCont, pRsp->contLen, TD_ENCODER); + tEncodeSVCreateTbBatchRsp(&coder, &rsp); + +_exit: + taosArrayClear(rsp.pArray); + tCoderClear(&coder); + return rcode; +} + +static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { + // ASSERT(0); +#if 0 SVCreateTbReq vAlterTbReq = {0}; vTrace("vgId:%d, process alter stb req", TD_VID(pVnode)); tDeserializeSVCreateTbReq(pReq, &vAlterTbReq); @@ -373,20 +398,44 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) { taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); } taosMemoryFree(vAlterTbReq.name); +#endif return 0; } -static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp) { - SSubmitRsp rsp = {0}; +static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { + // TODO + // ASSERT(0); + return 0; +} + +static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { + // TODO + ASSERT(0); + return 0; +} + +static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { + // TODO + ASSERT(0); + return 0; +} + +static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitRsp rsp = {0}; pRsp->code = 0; // handle the request - if (tsdbInsertData(pVnode->pTsdb, pSubmitReq, &rsp) < 0) { + if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { pRsp->code = terrno; return -1; } + // pRsp->msgType = TDMT_VND_SUBMIT_RSP; + // vnodeProcessSubmitReq(pVnode, ptr, pRsp); + // tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, ptr, STREAM_DATA_TYPE_SUBMIT_BLOCK); + // encode the response (TODO) pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); memcpy(pRsp->pCont, &rsp, sizeof(rsp)); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5f8bf3c7b5..26393394e9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -13,10 +13,11 @@ * along with this program. If not, see . */ -#include "sync.h" -#include "syncTools.h" -#include "tmsgcb.h" -#include "vnodeInt.h" +#include "vnd.h" +// #include "sync.h" +// #include "syncTools.h" +// #include "tmsgcb.h" +// #include "vnodeInt.h" // sync integration @@ -113,7 +114,7 @@ void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - SVnode * pVnode = (SVnode *)(pFsm->data); + SVnode *pVnode = (SVnode *)(pFsm->data); SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta); SRpcMsg applyMsg; syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 363824f2e2..00ec92465f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +// clang-format off #ifndef TDENGINE_EXECUTORIMPL_H #define TDENGINE_EXECUTORIMPL_H @@ -38,6 +39,8 @@ extern "C" { #include "tmsg.h" #include "tpagedbuf.h" +#include "vnode.h" + typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -319,27 +322,32 @@ typedef struct SColMatchInfo { bool output; } SColMatchInfo; +typedef struct SScanInfo { + int32_t numOfAsc; + int32_t numOfDesc; +} SScanInfo; + typedef struct STableScanInfo { void* dataReader; + int32_t numOfBlocks; // extract basic running information. int32_t numOfSkipped; int32_t numOfBlockStatis; int64_t numOfRows; - int32_t order; // scan order - int32_t times; // repeat counts + int64_t elapsedTime; + int32_t prevGroupId; // previous table group id + SScanInfo scanInfo; int32_t current; - int32_t reverseTimes; // 0 by default - SNode* pFilterNode; // filter operator info - SqlFunctionCtx* pCtx; // next operator query context + SNode* pFilterNode; // filter operator info + SqlFunctionCtx* pCtx; // next operator query context SResultRowInfo* pResultRowInfo; int32_t* rowCellInfoOffset; SExprInfo* pExpr; SSDataBlock* pResBlock; SArray* pColMatchInfo; int32_t numOfOutput; - int64_t elapsedTime; - int32_t prevGroupId; // previous table group id + SQueryTableDataCond cond; int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; double sampleRatio; // data block sample ratio, 1 by default @@ -380,7 +388,7 @@ typedef struct SSysTableScanInfo { int32_t accountId; bool showRewrite; SNode* pCondition; // db_name filter condition, to discard data that are not in current database - void* pCur; // cursor for iterate the local table meta store. + SMTbCursor* pCur; // cursor for iterate the local table meta store. SArray* scanCols; // SArray scan column id list SName name; SSDataBlock* pRes; @@ -620,6 +628,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList); void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win); +int32_t getTableScanOrder(SOperatorInfo* pOperator); void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); @@ -627,9 +636,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(void* pReaderHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime, - int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, - SInterval* pInterval, double ratio, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput, int32_t dataLoadFlag, const uint8_t* scanInfo, + SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 434591c4c3..3541013015 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -191,7 +191,7 @@ static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutpu static int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo* pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -// static STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win); +// static SQueryTableDataCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win); static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); @@ -207,7 +207,6 @@ static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput); static void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput); -static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput); @@ -3588,8 +3587,8 @@ static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQuer #endif } -// STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win) { -// STsdbQueryCond cond = { +// SQueryTableDataCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win) { +// SQueryTableDataCond cond = { // .colList = pQueryAttr->tableCols, // .order = pQueryAttr->order.order, // .numOfCols = pQueryAttr->numOfCols, @@ -4681,7 +4680,18 @@ _error: return NULL; } -static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { return pTableScanInfo->order; } +int32_t getTableScanOrder(SOperatorInfo* pOperator) { + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { + return TSDB_ORDER_ASC; + } else { + return getTableScanOrder(pOperator->pDownstream[0]); + } + } + + STableScanInfo* pTableScanInfo = pOperator->info; + return pTableScanInfo->cond.order; +} // this is a blocking operator static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { @@ -4885,6 +4895,76 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi return true; } +enum { + PROJECT_RETRIEVE_CONTINUE = 0x1, + PROJECT_RETRIEVE_DONE = 0x2, +}; + +static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) { + SProjectOperatorInfo* pProjectInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pProjectInfo->binfo; + SSDataBlock* pRes = pInfo->pRes; + + if (pProjectInfo->curSOffset > 0) { + if (pProjectInfo->groupId == 0) { // it is the first group + pProjectInfo->groupId = pBlock->info.groupId; + blockDataCleanup(pInfo->pRes); + return PROJECT_RETRIEVE_CONTINUE; + } else if (pProjectInfo->groupId != pBlock->info.groupId) { + pProjectInfo->curSOffset -= 1; + + // ignore data block in current group + if (pProjectInfo->curSOffset > 0) { + blockDataCleanup(pInfo->pRes); + return PROJECT_RETRIEVE_CONTINUE; + } + } + + // set current group id of the project operator + pProjectInfo->groupId = pBlock->info.groupId; + } + + if (pProjectInfo->groupId != 0 && pProjectInfo->groupId != pBlock->info.groupId) { + pProjectInfo->curGroupOutput += 1; + if ((pProjectInfo->slimit.limit > 0) && (pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput)) { + pOperator->status = OP_EXEC_DONE; + blockDataCleanup(pRes); + + return PROJECT_RETRIEVE_DONE; + } + + // reset the value for a new group data + pProjectInfo->curOffset = 0; + pProjectInfo->curOutput = 0; + } + + // here we reach the start position, according to the limit/offset requirements. + + // set current group id + pProjectInfo->groupId = pBlock->info.groupId; + + if (pProjectInfo->curOffset >= pRes->info.rows) { + pProjectInfo->curOffset -= pRes->info.rows; + blockDataCleanup(pRes); + return PROJECT_RETRIEVE_CONTINUE; + } else if (pProjectInfo->curOffset < pRes->info.rows && pProjectInfo->curOffset > 0) { + blockDataTrimFirstNRows(pRes, pProjectInfo->curOffset); + pProjectInfo->curOffset = 0; + } + + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + + // check for the limitation in each group + if (pProjectInfo->limit.limit > 0 && pProjectInfo->curOutput + pRes->info.rows >= pProjectInfo->limit.limit) { + pRes->info.rows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); + } + + return PROJECT_RETRIEVE_DONE; + } else { // not full enough, continue to accumulate the output data in the buffer. + return PROJECT_RETRIEVE_CONTINUE; + } +} + static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) { SProjectOperatorInfo* pProjectInfo = pOperator->info; SOptrBasicInfo* pInfo = &pProjectInfo->binfo; @@ -4953,63 +5033,22 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) // } // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC, false); + int32_t order = getTableScanOrder(pOperator->pDownstream[0]); + + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, false); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput, pProjectInfo->pPseudoColInfo); - if (pProjectInfo->curSOffset > 0) { - if (pProjectInfo->groupId == 0) { // it is the first group - pProjectInfo->groupId = pBlock->info.groupId; - blockDataCleanup(pInfo->pRes); - continue; - } else if (pProjectInfo->groupId != pBlock->info.groupId) { - pProjectInfo->curSOffset -= 1; - - // ignore data block in current group - if (pProjectInfo->curSOffset > 0) { - blockDataCleanup(pInfo->pRes); - continue; - } - } - - pProjectInfo->groupId = pBlock->info.groupId; - } - - if (pProjectInfo->groupId != 0 && pProjectInfo->groupId != pBlock->info.groupId) { - pProjectInfo->curGroupOutput += 1; - if ((pProjectInfo->slimit.limit > 0) && (pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput)) { - pOperator->status = OP_EXEC_DONE; - return NULL; - } - - // reset the value for a new group data - pProjectInfo->curOffset = 0; - pProjectInfo->curOutput = 0; - } - - pProjectInfo->groupId = pBlock->info.groupId; - - // todo extract method - if (pProjectInfo->curOffset < pInfo->pRes->info.rows && pProjectInfo->curOffset > 0) { - blockDataTrimFirstNRows(pInfo->pRes, pProjectInfo->curOffset); - pProjectInfo->curOffset = 0; - } else if (pProjectInfo->curOffset >= pInfo->pRes->info.rows) { - pProjectInfo->curOffset -= pInfo->pRes->info.rows; - blockDataCleanup(pInfo->pRes); + int32_t status = handleLimitOffset(pOperator, pBlock); + if (status == PROJECT_RETRIEVE_CONTINUE) { continue; - } - - if (pRes->info.rows >= pOperator->resultInfo.threshold) { + } else if (status == PROJECT_RETRIEVE_DONE) { break; } } - if (pProjectInfo->limit.limit > 0 && pProjectInfo->curOutput + pInfo->pRes->info.rows >= pProjectInfo->limit.limit) { - pInfo->pRes->info.rows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); - } - pProjectInfo->curOutput += pInfo->pRes->info.rows; // copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); @@ -5661,8 +5700,7 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, - const STableGroupInfo* pTableGroupInfo) { + int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -6319,6 +6357,19 @@ static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOu static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget); static SArray* createIndexMap(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); +static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); + +static SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) { + SInterval interval = { + .interval = pTableScanNode->interval, + .sliding = pTableScanNode->sliding, + .intervalUnit = pTableScanNode->intervalUnit, + .slidingUnit = pTableScanNode->slidingUnit, + .offset = pTableScanNode->offset, + }; + + return interval; +} SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { @@ -6329,7 +6380,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - int32_t numOfCols = 0; + int32_t numOfCols = 0; tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); if (pDataReader == NULL && terrno != 0) { return NULL; @@ -6339,16 +6390,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); - SInterval interval = { - .interval = pTableScanNode->interval, - .sliding = pTableScanNode->sliding, - .intervalUnit = pTableScanNode->intervalUnit, - .slidingUnit = pTableScanNode->slidingUnit, - .offset = pTableScanNode->offset, - }; + SQueryTableDataCond cond = {0}; + int32_t code = initQueryTableDataCond(&cond, pTableScanNode); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } - return createTableScanOperatorInfo(pDataReader, pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC, - numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList, + SInterval interval = extractIntervalInfo(pTableScanNode); + return createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq, pColList, pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; @@ -6369,10 +6418,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; - SSDataBlock* pResBlock = createResDataBlock(pSysScanPhyNode->scan.node.pOutputDataBlockDesc); + SScanPhysiNode* pScanNode = &pSysScanPhyNode->scan; - struct SScanPhysiNode* pScanNode = &pSysScanPhyNode->scan; - SArray* colList = extractScanColumnId(pScanNode->pScanCols); + SSDataBlock* pResBlock = createResDataBlock(pScanNode->node.pOutputDataBlockDesc); + SArray* colList = extractScanColumnId(pScanNode->pScanCols); SOperatorInfo* pOperator = createSysTableScanOperatorInfo( pHandle->meta, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, @@ -6493,38 +6542,47 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOptr; } -static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STableGroupInfo* pGroupInfo, - void* readHandle, uint64_t queryId, uint64_t taskId) { - STsdbQueryCond cond = {.loadExternalRows = false}; - cond.order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - cond.numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); - cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo)); - if (cond.colList == NULL) { +static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) { + pCond->loadExternalRows = false; + + pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); + pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); + if (pCond->colList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; + return terrno; } - cond.twindow = pTableScanNode->scanRange; - cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER; - // cond.type = pTableScanNode->scanFlag; + pCond->twindow = pTableScanNode->scanRange; + +#if 1 + //todo work around a problem, remove it later + if ((pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey > pCond->twindow.ekey) || + (pCond->order == TSDB_ORDER_DESC && pCond->twindow.skey < pCond->twindow.ekey)) { + TSWAP(pCond->twindow.skey, pCond->twindow.ekey, int64_t); + } +#endif + + pCond->type = BLOCK_LOAD_OFFSET_SEQ_ORDER; + // pCond->type = pTableScanNode->scanFlag; int32_t j = 0; - for (int32_t i = 0; i < cond.numOfCols; ++i) { + for (int32_t i = 0; i < pCond->numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pTableScanNode->scan.pScanCols, i); SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; if (pColNode->colType == COLUMN_TYPE_TAG) { continue; } - cond.colList[j].type = pColNode->node.resType.type; - cond.colList[j].bytes = pColNode->node.resType.bytes; - cond.colList[j].colId = pColNode->colId; + pCond->colList[j].type = pColNode->node.resType.type; + pCond->colList[j].bytes = pColNode->node.resType.bytes; + pCond->colList[j].colId = pColNode->colId; j += 1; } - cond.numOfCols = j; - return tsdbQueryTables(readHandle, &cond, pGroupInfo, queryId, taskId); + pCond->numOfCols = j; + return TSDB_CODE_SUCCESS; } SArray* extractScanColumnId(SNodeList* pNodeList) { @@ -6742,7 +6800,13 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* goto _error; } - return createDataReaderImpl(pTableScanNode, pTableGroupInfo, pHandle->reader, queryId, taskId); + SQueryTableDataCond cond = {0}; + code = initQueryTableDataCond(&cond, pTableScanNode); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return tsdbQueryTables(pHandle->reader, &cond, pTableGroupInfo, queryId, taskId); _error: terrno = code; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 52bdbea8a5..27c616498e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -220,7 +220,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } // The first row of a new block does not belongs to the previous existed group - if (!equal && j == 0) { + if (j == 0) { num++; recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); continue; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a70e402871..277ab9bc6f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -245,6 +245,10 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols); + // reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream + // operator. + pBlock->info.blockId = 0; + doFilter(pTableScanInfo->pFilterNode, pBlock); if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; @@ -255,17 +259,15 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, return TSDB_CODE_SUCCESS; } -static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { - // reverse order time range +static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { SET_REVERSE_SCAN_FLAG(pTableScanInfo); switchCtxOrder(pCtx, numOfOutput); - SWITCH_ORDER(pTableScanInfo->order); - setupQueryRangeForReverseScan(pTableScanInfo); + // setupQueryRangeForReverseScan(pTableScanInfo); - pTableScanInfo->times = 1; - pTableScanInfo->current = 0; - pTableScanInfo->reverseTimes = 0; + STimeWindow* pTWindow = &pTableScanInfo->cond.twindow; + TSWAP(pTWindow->skey, pTWindow->ekey, int64_t); + pTableScanInfo->cond.order = TSDB_ORDER_DESC; } static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { @@ -294,8 +296,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { continue; } - // reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream operator. - pBlock->info.blockId = 0; return pBlock; } @@ -311,63 +311,71 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } - SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; *newgroup = false; - while (pTableScanInfo->current < pTableScanInfo->times) { + while (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator, newgroup); if (p != NULL) { return p; } - if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0 /* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) { - return NULL; - } else { - break; + pTableScanInfo->current += 1; + + if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); + pTableScanInfo->scanFlag = REPEAT_SCAN; + + STimeWindow* pWin = &pTableScanInfo->cond.twindow; + qDebug("%s start to repeat ascending order scan data blocks due to query func required, qrange:%" PRId64 + "-%" PRId64, + GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + + // do prepare for the next round table scan operation + tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); + } + } + + int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc; + if (pTableScanInfo->current < total) { + if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { + prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); + tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); + } + + STimeWindow* pWin = &pTableScanInfo->cond.twindow; + qDebug("%s start to descending order scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + + while (pTableScanInfo->current < total) { + SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + if (p != NULL) { + return p; + } + + pTableScanInfo->current += 1; + + if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); + pTableScanInfo->scanFlag = REPEAT_SCAN; + + qDebug("%s start to repeat descending order scan data blocks due to query func required, qrange:%" PRId64 + "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); + + // do prepare for the next round table scan operation + tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); } } - - // do prepare for the next round table scan operation - // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - // tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - - setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTableScanInfo->scanFlag = REPEAT_SCAN; - - // if (pResultRowInfo->size > 0) { - // pResultRowInfo->curPos = 0; - // } - - qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, - GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); } - SSDataBlock* p = NULL; - // todo refactor - if (pTableScanInfo->reverseTimes > 0) { - setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); - // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - // tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - - qDebug("%s start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, - GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); - - if (pResultRowInfo->size > 0) { - // pResultRowInfo->curPos = pResultRowInfo->size - 1; - } - - p = doTableScanImpl(pOperator, newgroup); - } - - return p; + setTaskStatus(pTaskInfo, TASK_COMPLETED); + return NULL; } -SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag, - int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, - SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) { - assert(repeatTime > 0); - +SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput, + int32_t dataLoadFlag, const uint8_t* scanInfo, SArray* pColMatchInfo, + SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, + double sampleRatio, SExecTaskInfo* pTaskInfo) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -378,26 +386,27 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int return NULL; } - pInfo->interval = *pInterval; - pInfo->sampleRatio = sampleRatio; - pInfo->dataBlockLoadFlag= dataLoadFlag; - pInfo->pResBlock = pResBlock; - pInfo->pFilterNode = pCondition; - pInfo->dataReader = pDataReader; - pInfo->times = repeatTime; - pInfo->reverseTimes = reverseTime; - pInfo->order = order; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColMatchInfo; - pOperator->name = "TableScanOperator"; + pInfo->cond = *pCond; + pInfo->scanInfo = (SScanInfo){.numOfAsc = scanInfo[0], .numOfDesc = scanInfo[1]}; + + pInfo->interval = *pInterval; + pInfo->sampleRatio = sampleRatio; + pInfo->dataBlockLoadFlag = dataLoadFlag; + pInfo->pResBlock = pResBlock; + pInfo->pFilterNode = pCondition; + pInfo->dataReader = pDataReader; + pInfo->current = 0; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; + + pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->getNextFn = doTableScan; + pOperator->pTaskInfo = pTaskInfo; static int32_t cost = 0; pOperator->cost.openCost = ++cost; @@ -411,8 +420,6 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); pInfo->dataReader = pTsdbReadHandle; - pInfo->times = 1; - pInfo->reverseTimes = 0; pInfo->current = 0; pInfo->prevGroupId = -1; @@ -816,8 +823,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { int32_t numOfRows = 0; char n[TSDB_TABLE_NAME_LEN] = {0}; - while ((tb = metaTbCursorNext(pInfo->pCur)) != NULL) { - STR_TO_VARSTR(n, tb); + while (metaTbCursorNext(pInfo->pCur) == 0) { + STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); colDataAppend(pTableNameCol, numOfRows, n, false); numOfRows += 1; if (numOfRows >= pInfo->capacity) { @@ -1117,28 +1124,29 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { - STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); +SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, + SExecTaskInfo* pTaskInfo) { + STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pReader = pReaderHandle; - pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->getNextFn = doTagScan; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pTaskInfo = pTaskInfo; - pOperator->closeFn = destroyTagScanOperatorInfo; + pInfo->pReader = pReaderHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->getNextFn = doTagScan; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; + pOperator->closeFn = destroyTagScanOperatorInfo; return pOperator; - _error: +_error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/test/CMakeLists.txt b/source/libs/executor/test/CMakeLists.txt index b1f379585b..129509d6c6 100644 --- a/source/libs/executor/test/CMakeLists.txt +++ b/source/libs/executor/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(executorTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( executorTest - PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes + PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes vnode ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index af4c8a7bdb..e19a332e66 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -24,7 +24,7 @@ extern "C" { #define FUNCTION_NAME_MAX_LENGTH 32 -#define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n) (1 << n) +#define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n) (1 << n) #define FUNC_MGT_AGG_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(0) #define FUNC_MGT_SCALAR_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(1) @@ -38,6 +38,7 @@ extern "C" { #define FUNC_MGT_SPECIAL_DATA_REQUIRED FUNC_MGT_FUNC_CLASSIFICATION_MASK(9) #define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10) #define FUNC_MGT_MULTI_RES_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(11) +#define FUNC_MGT_SCAN_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(12) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 805df08f27..45cfbae1ad 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -807,7 +807,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = NULL}, {.name = "tbname", .type = FUNCTION_TYPE_TBNAME, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 91ccdfb71e..72c7175c6d 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -15,12 +15,12 @@ #include "functionMgt.h" +#include "builtins.h" +#include "catalog.h" #include "functionMgtInt.h" #include "taos.h" #include "taoserror.h" #include "thash.h" -#include "builtins.h" -#include "catalog.h" typedef struct SFuncMgtService { SHashObj* pFuncNameHashTable; @@ -28,22 +28,24 @@ typedef struct SFuncMgtService { typedef struct SUdfInfo { SDataType outputDt; - int8_t funcType; + int8_t funcType; } SUdfInfo; static SFuncMgtService gFunMgtService; -static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT; -static int32_t initFunctionCode = 0; +static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT; +static int32_t initFunctionCode = 0; static void doInitFunctionTable() { - gFunMgtService.pFuncNameHashTable = taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + gFunMgtService.pFuncNameHashTable = + taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (NULL == gFunMgtService.pFuncNameHashTable) { initFunctionCode = TSDB_CODE_FAILED; return; } for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) { - if (TSDB_CODE_SUCCESS != taosHashPut(gFunMgtService.pFuncNameHashTable, funcMgtBuiltins[i].name, strlen(funcMgtBuiltins[i].name), &i, sizeof(int32_t))) { + if (TSDB_CODE_SUCCESS != taosHashPut(gFunMgtService.pFuncNameHashTable, funcMgtBuiltins[i].name, + strlen(funcMgtBuiltins[i].name), &i, sizeof(int32_t))) { initFunctionCode = TSDB_CODE_FAILED; return; } @@ -52,8 +54,9 @@ static void doInitFunctionTable() { static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { if (fmIsUserDefinedFunc(funcId)) { - return FUNC_MGT_AGG_FUNC == classification ? FUNC_AGGREGATE_UDF_ID == funcId : - (FUNC_MGT_SCALAR_FUNC == classification ? FUNC_SCALAR_UDF_ID == funcId : false); + return FUNC_MGT_AGG_FUNC == classification + ? FUNC_AGGREGATE_UDF_ID == funcId + : (FUNC_MGT_SCALAR_FUNC == classification ? FUNC_SCALAR_UDF_ID == funcId : false); } if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; @@ -63,7 +66,7 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { static int32_t getUdfInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) { SFuncInfo* pInfo = NULL; - int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &pInfo); + int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &pInfo); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -122,33 +125,23 @@ int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) { return TSDB_CODE_FAILED; } pFpSet->process = funcMgtBuiltins[funcId].sprocessFunc; - pFpSet->getEnv = funcMgtBuiltins[funcId].getEnvFunc; + pFpSet->getEnv = funcMgtBuiltins[funcId].getEnvFunc; return TSDB_CODE_SUCCESS; } -bool fmIsAggFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_AGG_FUNC); -} +bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_AGG_FUNC); } -bool fmIsScalarFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); -} +bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } -bool fmIsPseudoColumnFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); -} +bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } -bool fmIsWindowPseudoColumnFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC); -} +bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); } -bool fmIsWindowClauseFunc(int32_t funcId) { - return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId); -} +bool fmIsWindowPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC); } -bool fmIsNonstandardSQLFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_NONSTANDARD_SQL_FUNC); -} +bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId); } + +bool fmIsNonstandardSQLFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_NONSTANDARD_SQL_FUNC); } bool fmIsSpecialDataRequiredFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED); @@ -158,13 +151,9 @@ bool fmIsDynamicScanOptimizedFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED); } -bool fmIsMultiResFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_RES_FUNC); -} +bool fmIsMultiResFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_RES_FUNC); } -bool fmIsUserDefinedFunc(int32_t funcId) { - return funcId > FUNC_UDF_ID_START; -} +bool fmIsUserDefinedFunc(int32_t funcId) { return funcId > FUNC_UDF_ID_START; } void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 89cacf9298..f1fe68c55a 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -224,6 +224,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_ALL_SCALAR_FIELDS; COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pScanCols); + CLONE_NODE_LIST_FIELD(pScanPseudoCols); CLONE_OBJECT_FIELD(pMeta, tableMetaClone); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index f8a56109ee..0e6ec4f945 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -467,6 +467,7 @@ static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { } static const char* jkScanLogicPlanScanCols = "ScanCols"; +static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize"; static const char* jkScanLogicPlanTableMeta = "TableMeta"; @@ -477,6 +478,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkScanLogicPlanScanCols, pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkScanLogicPlanScanPseudoCols, pNode->pScanPseudoCols); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); } @@ -495,6 +499,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkScanLogicPlanScanCols, &pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanLogicPlanScanPseudoCols, &pNode->pScanPseudoCols); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize); } @@ -670,6 +677,7 @@ static int32_t jsonToName(const SJson* pJson, void* pObj) { } static const char* jkScanPhysiPlanScanCols = "ScanCols"; +static const char* jkScanPhysiPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanPhysiPlanTableId = "TableId"; static const char* jkScanPhysiPlanTableType = "TableType"; static const char* jkScanPhysiPlanTableName = "TableName"; @@ -681,6 +689,9 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkScanPhysiPlanScanCols, pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkScanPhysiPlanScanPseudoCols, pNode->pScanPseudoCols); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableId, pNode->uid); } @@ -701,6 +712,9 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkScanPhysiPlanScanCols, &pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanPhysiPlanScanPseudoCols, &pNode->pScanPseudoCols); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkScanPhysiPlanTableId, &pNode->uid); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 092d0f435d..351b50133c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1041,10 +1041,11 @@ bool nodesIsTimeorderQuery(const SNode* pQuery) { return false; } bool nodesIsTimelineQuery(const SNode* pQuery) { return false; } typedef struct SCollectColumnsCxt { - int32_t errCode; - const char* pTableAlias; - SNodeList* pCols; - SHashObj* pColHash; + int32_t errCode; + const char* pTableAlias; + ECollectColType collectType; + SNodeList* pCols; + SHashObj* pColHash; } SCollectColumnsCxt; static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pNode) { @@ -1057,25 +1058,33 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN if (NULL == taosHashGet(pCxt->pColHash, name, len)) { pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0); if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pCxt->errCode = nodesListAppend(pCxt->pCols, pNode); + pCxt->errCode = nodesListStrictAppend(pCxt->pCols, nodesCloneNode(pNode)); } return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } return DEAL_RES_CONTINUE; } +static bool isCollectType(ECollectColType collectType, EColumnType colType) { + return COLLECT_COL_TYPE_ALL == collectType + ? true + : (COLLECT_COL_TYPE_TAG == collectType ? COLUMN_TYPE_TAG == colType : COLUMN_TYPE_TAG != colType); +} + static EDealRes collectColumns(SNode* pNode, void* pContext) { SCollectColumnsCxt* pCxt = (SCollectColumnsCxt*)pContext; if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; - if (NULL == pCxt->pTableAlias || 0 == strcmp(pCxt->pTableAlias, pCol->tableAlias)) { + if (isCollectType(pCxt->collectType, pCol->colType) && + (NULL == pCxt->pTableAlias || 0 == strcmp(pCxt->pTableAlias, pCol->tableAlias))) { return doCollect(pCxt, pCol, pNode); } } return DEAL_RES_CONTINUE; } -int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols) { +int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, + SNodeList** pCols) { if (NULL == pSelect || NULL == pCols) { return TSDB_CODE_SUCCESS; } @@ -1083,6 +1092,7 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* SCollectColumnsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pTableAlias = pTableAlias, + .collectType = type, .pCols = nodesMakeList(), .pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)}; if (NULL == cxt.pCols || NULL == cxt.pColHash) { @@ -1092,11 +1102,11 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* nodesWalkSelectStmt(pSelect, clause, collectColumns, &cxt); taosHashCleanup(cxt.pColHash); if (TSDB_CODE_SUCCESS != cxt.errCode) { - nodesClearList(cxt.pCols); + nodesDestroyList(cxt.pCols); return cxt.errCode; } if (0 == LIST_LENGTH(cxt.pCols)) { - nodesClearList(cxt.pCols); + nodesDestroyList(cxt.pCols); cxt.pCols = NULL; } *pCols = cxt.pCols; @@ -1123,10 +1133,12 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod return TSDB_CODE_SUCCESS; } - SCollectFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .classifier = classifier, .pFuncs = nodesMakeList()}; + SCollectFuncsCxt cxt = { + .errCode = TSDB_CODE_SUCCESS, .classifier = classifier, .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs)}; if (NULL == cxt.pFuncs) { return TSDB_CODE_OUT_OF_MEMORY; } + *pFuncs = NULL; nodesWalkSelectStmt(pSelect, SQL_CLAUSE_GROUP_BY, collectFuncs, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pFuncs); @@ -1136,7 +1148,6 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod *pFuncs = cxt.pFuncs; } else { nodesDestroyList(cxt.pFuncs); - *pFuncs = NULL; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 5a93c5cef5..dfeb2df911 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -22,6 +22,7 @@ #include "ttime.h" #include "ttypes.h" +// clang-format off #define NEXT_TOKEN(pSql, sToken) \ do { \ int32_t index = 0; \ @@ -767,8 +768,8 @@ static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const SName* pName, SKVRo tNameGetFullDbName(pName, dbFName); pTbReq->type = TD_CHILD_TABLE; pTbReq->name = strdup(pName->tname); - pTbReq->ctbCfg.suid = suid; - pTbReq->ctbCfg.pTag = row; + pTbReq->ctb.suid = suid; + pTbReq->ctb.pTag = row; return TSDB_CODE_SUCCESS; } @@ -1007,7 +1008,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da void destroyCreateSubTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->name); - taosMemoryFreeClear(pReq->ctbCfg.pTag); + taosMemoryFreeClear(pReq->ctb.pTag); } static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { @@ -1256,10 +1257,9 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_BIND_v2* bind, char* msgBuf, - int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_QRY_APP_ERROR; @@ -1307,10 +1307,11 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* p return TSDB_CODE_SUCCESS; } -int32_t qBindStmtColsValue(void* pBlock, TAOS_BIND_v2* bind, char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + +int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; @@ -1381,11 +1382,10 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_BIND_v2* bind, char* msgBuf, int32 return TSDB_CODE_SUCCESS; } -int32_t qBindStmtSingleColValue(void* pBlock, TAOS_BIND_v2* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, - int32_t rowNum) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); +int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 42dde12bff..aee4566fa0 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +// clang-format off #include "parInsertData.h" #include "catalog.h" @@ -157,7 +157,11 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star } int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { - int32_t len = tSerializeSVCreateTbReq(NULL, pCreateTbReq); + SCoder coder = {0}; + char* pBuf; + int32_t len; + + tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len); if (pBlocks->nAllocSize - pBlocks->size < len) { pBlocks->nAllocSize += len + pBlocks->rowSize; char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize); @@ -169,8 +173,13 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) return TSDB_CODE_TSC_OUT_OF_MEMORY; } } - char* pBuf = pBlocks->pData + pBlocks->size; - tSerializeSVCreateTbReq((void**)&pBuf, pCreateTbReq); + + pBuf= pBlocks->pData + pBlocks->size; + + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, len, TD_ENCODER); + tEncodeSVCreateTbReq(&coder, pCreateTbReq); + tCoderClear(&coder); + pBlocks->size += len; pBlocks->createTbReqLen = len; return TSDB_CODE_SUCCESS; @@ -191,7 +200,7 @@ int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int3 return ret; } - if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctbCfg.pTag) { + if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctb.pTag) { ret = buildCreateTbMsg(*dataBlocks, pCreateTbReq); if (ret != TSDB_CODE_SUCCESS) { return ret; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5bb82449b3..aae932471d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1905,18 +1905,9 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; SField field = {.type = pCol->dataType.type, .bytes = calcTypeBytes(pCol->dataType)}; strcpy(field.name, pCol->colName); - taosArrayPush(*pArray, &field); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) { - *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); - SNode* pNode; - FOREACH(pNode, pList) { - SColumnNode* pCol = (SColumnNode*)pNode; - SField field = {.type = pCol->node.resType.type, .bytes = calcTypeBytes(pCol->node.resType)}; - strcpy(field.name, pCol->colName); + if (pCol->sma) { + field.flags |= SCHEMA_SMA_ON; + } taosArrayPush(*pArray, &field); } return TSDB_CODE_SUCCESS; @@ -2249,13 +2240,6 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm columnDefNodeToField(pStmt->pTags, &pReq->pTags); pReq->numOfColumns = LIST_LENGTH(pStmt->pCols); pReq->numOfTags = LIST_LENGTH(pStmt->pTags); - if (NULL == pStmt->pOptions->pSma) { - columnDefNodeToField(pStmt->pCols, &pReq->pSmas); - pReq->numOfSmas = pReq->numOfColumns; - } else { - columnNodeToField(pStmt->pOptions->pSma, &pReq->pSmas); - pReq->numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma); - } SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); @@ -3234,7 +3218,7 @@ typedef struct SVgroupTablesBatch { static void destroyCreateTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->name); - taosMemoryFreeClear(pReq->ntbCfg.pSchema); + taosMemoryFreeClear(pReq->ntb.schema.pSchema); } static int32_t buildSmaParam(STableOptions* pOptions, SVCreateTbReq* pReq) { @@ -3242,12 +3226,14 @@ static int32_t buildSmaParam(STableOptions* pOptions, SVCreateTbReq* pReq) { return TSDB_CODE_SUCCESS; } +#if 0 pReq->ntbCfg.pRSmaParam = taosMemoryCalloc(1, sizeof(SRSmaParam)); if (NULL == pReq->ntbCfg.pRSmaParam) { return TSDB_CODE_OUT_OF_MEMORY; } pReq->ntbCfg.pRSmaParam->delay = GET_OPTION_VAL(pOptions->pDelay, TSDB_DEFAULT_DB_DELAY); pReq->ntbCfg.pRSmaParam->xFilesFactor = GET_OPTION_VAL(pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR); +#endif return TSDB_CODE_SUCCESS; } @@ -3262,16 +3248,17 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* SVCreateTbReq req = {0}; req.type = TD_NORMAL_TABLE; req.name = strdup(pStmt->tableName); - req.ntbCfg.nCols = LIST_LENGTH(pStmt->pCols); - req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema)); - if (NULL == req.name || NULL == req.ntbCfg.pSchema) { + req.ntb.schema.nCols = LIST_LENGTH(pStmt->pCols); + req.ntb.schema.sver = 0; + req.ntb.schema.pSchema = taosMemoryCalloc(req.ntb.schema.nCols, sizeof(SSchema)); + if (NULL == req.name || NULL == req.ntb.schema.pSchema) { destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } SNode* pCol; col_id_t index = 0; FOREACH(pCol, pStmt->pCols) { - toSchema((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index); + toSchema((SColumnDefNode*)pCol, index + 1, req.ntb.schema.pSchema + index); ++index; } if (TSDB_CODE_SUCCESS != buildSmaParam(pStmt->pOptions, &req)) { @@ -3292,7 +3279,11 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* } static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) { - int tlen = sizeof(SMsgHead) + tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req)); + int tlen; + SCoder coder = {0}; + + tEncodeSize(tEncodeSVCreateTbBatchReq, &pTbBatch->req, tlen); + tlen += sizeof(SMsgHead); //+ tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req)); void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3300,7 +3291,10 @@ static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* ((SMsgHead*)buf)->vgId = htonl(pTbBatch->info.vgId); ((SMsgHead*)buf)->contLen = htonl(tlen); void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tSerializeSVCreateTbBatchReq(&pBuf, &(pTbBatch->req)); + + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, tlen - sizeof(SMsgHead), TD_ENCODER); + tEncodeSVCreateTbBatchReq(&coder, &pTbBatch->req); + tCoderClear(&coder); SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == pVgData) { @@ -3322,9 +3316,9 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { taosMemoryFreeClear(pTableReq->name); if (pTableReq->type == TSDB_NORMAL_TABLE) { - taosMemoryFreeClear(pTableReq->ntbCfg.pSchema); + taosMemoryFreeClear(pTableReq->ntb.schema.pSchema); } else if (pTableReq->type == TSDB_CHILD_TABLE) { - taosMemoryFreeClear(pTableReq->ctbCfg.pTag); + taosMemoryFreeClear(pTableReq->ctb.pTag); } } @@ -3405,8 +3399,8 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; req.name = strdup(pTableName); - req.ctbCfg.suid = suid; - req.ctbCfg.pTag = row; + req.ctb.suid = suid; + req.ctb.pTag = row; SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); if (pTableBatch == NULL) { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index f8ff2848af..450fc307ea 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -129,14 +129,66 @@ static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelec return code; } -static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanCols, STableMeta* pMeta) { +typedef struct SCreateColumnCxt { + int32_t errCode; + SNodeList* pList; +} SCreateColumnCxt; + +static EDealRes doCreateColumn(SNode* pNode, void* pContext) { + SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: { + SNode* pCol = nodesCloneNode(pNode); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_FUNCTION: { + SExprNode* pExpr = (SExprNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } + pCol->node.resType = pExpr->resType; + strcpy(pCol->colName, pExpr->aliasName); + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + default: + break; + } + + return DEAL_RES_CONTINUE; +} + +static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) { + SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; + if (NULL == cxt.pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + nodesWalkExprs(pExprs, doCreateColumn, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(cxt.pList); + return cxt.errCode; + } + if (NULL == *pList) { + *pList = cxt.pList; + } + return cxt.errCode; +} + +static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols, SNodeList* pScanCols, + STableMeta* pMeta) { if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { return SCAN_TYPE_STREAM; } if (NULL == pScanCols) { // select count(*) from t - return SCAN_TYPE_TABLE; + return NULL == pScanPseudoCols ? SCAN_TYPE_TABLE : SCAN_TYPE_TAG; } if (TSDB_SYSTEM_TABLE == pMeta->tableType) { @@ -186,7 +238,6 @@ static int32_t addPrimaryKeyCol(uint64_t tableId, SNodeList** pCols) { if (!found) { if (TSDB_CODE_SUCCESS != nodesListStrictAppend(*pCols, createPrimaryKeyCol(tableId))) { - nodesDestroyList(*pCols); return TSDB_CODE_OUT_OF_MEMORY; } } @@ -214,30 +265,31 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; // set columns to scan - SNodeList* pCols = NULL; - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols); + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL, + &pScan->pScanCols); + if (TSDB_CODE_SUCCESS == code) { - code = addPrimaryKeyCol(pScan->pMeta->uid, &pCols); + code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_TAG, + &pScan->pScanPseudoCols); } if (TSDB_CODE_SUCCESS == code) { - pScan->pScanCols = nodesCloneList(pCols); - if (NULL == pScan) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = nodesCollectFuncs(pSelect, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols); } - pScan->scanType = getScanType(pCxt, pCols, pScan->pMeta); + pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta); + + if (TSDB_CODE_SUCCESS == code) { + code = addPrimaryKeyCol(pScan->pMeta->uid, &pScan->pScanCols); + } // set output if (TSDB_CODE_SUCCESS == code) { - pScan->node.pTargets = nodesCloneList(pCols); - if (NULL == pScan) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = createColumnByRewriteExps(pCxt, pScan->pScanCols, &pScan->node.pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pCxt, pScan->pScanPseudoCols, &pScan->node.pTargets); } - - nodesClearList(pCols); if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pScan; @@ -362,57 +414,6 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) return pCol; } -typedef struct SCreateColumnCxt { - int32_t errCode; - SNodeList* pList; -} SCreateColumnCxt; - -static EDealRes doCreateColumn(SNode* pNode, void* pContext) { - SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; - switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: { - SNode* pCol = nodesCloneNode(pNode); - if (NULL == pCol) { - return DEAL_RES_ERROR; - } - return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); - } - case QUERY_NODE_OPERATOR: - case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_FUNCTION: { - SExprNode* pExpr = (SExprNode*)pNode; - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - return DEAL_RES_ERROR; - } - pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); - return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); - } - default: - break; - } - - return DEAL_RES_CONTINUE; -} - -static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) { - SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; - if (NULL == cxt.pList) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - nodesWalkExprs(pExprs, doCreateColumn, &cxt); - if (TSDB_CODE_SUCCESS != cxt.errCode) { - nodesDestroyList(cxt.pList); - return cxt.errCode; - } - if (NULL == *pList) { - *pList = cxt.pList; - } - return cxt.errCode; -} - static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; @@ -598,14 +599,7 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } - SNodeList* pCols = NULL; - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, &pCols); - if (TSDB_CODE_SUCCESS == code && NULL != pCols) { - pSort->node.pTargets = nodesCloneList(pCols); - if (NULL == pSort->node.pTargets) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); if (TSDB_CODE_SUCCESS == code) { pSort->pSortKeys = nodesCloneList(pSelect->pOrderByList); @@ -695,14 +689,8 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS return TSDB_CODE_OUT_OF_MEMORY; } - SNodeList* pCols = NULL; - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, NULL, &pCols); - if (TSDB_CODE_SUCCESS == code && NULL != pCols) { - pPartition->node.pTargets = nodesCloneList(pCols); - if (NULL == pPartition->node.pTargets) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } + int32_t code = + nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, NULL, COLLECT_COL_TYPE_ALL, &pPartition->node.pTargets); if (TSDB_CODE_SUCCESS == code) { pPartition->pPartitionKeys = nodesCloneList(pSelect->pPartitionByList); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4997cfd644..707e6aac14 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -380,6 +380,10 @@ static int32_t sortScanCols(SNodeList* pScanCols) { } static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) { + if (NULL == pScanCols) { + return TSDB_CODE_SUCCESS; + } + pScanPhysiNode->pScanCols = nodesCloneList(pScanCols); if (NULL == pScanPhysiNode->pScanCols) { return TSDB_CODE_OUT_OF_MEMORY; @@ -394,9 +398,22 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNo // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t code = addDataBlockSlots(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); } + + if (TSDB_CODE_SUCCESS == code && NULL != pScanLogicNode->pScanPseudoCols) { + pScanPhysiNode->pScanPseudoCols = nodesCloneList(pScanLogicNode->pScanPseudoCols); + if (NULL == pScanPhysiNode->pScanPseudoCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pScanPhysiNode->pScanPseudoCols, pScanPhysiNode->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode); } + if (TSDB_CODE_SUCCESS == code) { pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; @@ -1190,6 +1207,11 @@ static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubp ++(pQueryPlan->numOfSubplans); } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pSubplan); + return code; + } + if (TSDB_CODE_SUCCESS == code && NULL != pParent) { code = nodesListMakeAppend(&pParent->pChildren, pSubplan); if (TSDB_CODE_SUCCESS == code) { @@ -1207,10 +1229,6 @@ static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubp } } - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pSubplan); - } - return code; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 2fa6395aed..4d4670379e 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -101,8 +101,8 @@ int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstream return setSubplanExecutionNode(subplan->pNode, groupId, pSource); } -static int32_t setValueByBindParam(SValueNode* pVal, TAOS_BIND_v2* pParam) { - if (1 == *(pParam->is_null)) { +static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { + if (pParam->is_null && 1 == *(pParam->is_null)) { pVal->node.resType.type = TSDB_DATA_TYPE_NULL; pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; return TSDB_CODE_SUCCESS; @@ -168,10 +168,18 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_BIND_v2* pParam) { return TSDB_CODE_SUCCESS; } -int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams) { - int32_t index = 0; - SNode* pNode = NULL; - FOREACH(pNode, pPlan->pPlaceholderValues) { setValueByBindParam((SValueNode*)pNode, pParams + index); } +int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx) { + if (colIdx < 0) { + int32_t index = 0; + SNode* pNode = NULL; + + FOREACH(pNode, pPlan->pPlaceholderValues) { + setValueByBindParam((SValueNode*)pNode, pParams + index); + ++index; + } + } else { + setValueByBindParam((SValueNode*)nodesListGetNode(pPlan->pPlaceholderValues, colIdx), pParams); + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/planSTableTest.cpp b/source/libs/planner/test/planSTableTest.cpp index 49002fe826..0b60eaea32 100644 --- a/source/libs/planner/test/planSTableTest.cpp +++ b/source/libs/planner/test/planSTableTest.cpp @@ -19,7 +19,7 @@ using namespace std; class PlanSuperTableTest : public PlannerTestBase {}; -TEST_F(PlanSuperTableTest, unionAll) { +TEST_F(PlanSuperTableTest, tbname) { useDb("root", "test"); run("select tbname from st1"); diff --git a/source/libs/planner/test/planStmtTest.cpp b/source/libs/planner/test/planStmtTest.cpp index 7366e7663e..edf1990704 100644 --- a/source/libs/planner/test/planStmtTest.cpp +++ b/source/libs/planner/test/planStmtTest.cpp @@ -26,7 +26,7 @@ class PlanStmtTest : public PlannerTestBase { } void bindParam(int32_t val) { - TAOS_BIND_v2* pBind = pBindParams_ + paramNo_++; + TAOS_MULTI_BIND* pBind = pBindParams_ + paramNo_++; pBind->buffer_type = TSDB_DATA_TYPE_INT; pBind->num = 1; pBind->buffer_length = sizeof(int32_t); @@ -42,9 +42,9 @@ class PlanStmtTest : public PlannerTestBase { // todo } - private: - TAOS_BIND_v2* pBindParams_; - int32_t paramNo_; +private: + TAOS_MULTI_BIND* pBindParams_; + int32_t paramNo_; }; TEST_F(PlanStmtTest, stmt) { diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 9142cc41c4..021e0e0f96 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -34,6 +34,7 @@ typedef struct SScalarCtx { #define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type)) #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) +#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type)) #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index adbdd13a84..f80ffc70a2 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -86,8 +86,10 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) p = getVectorDoubleValue_JSON; } else if (srcType == TSDB_DATA_TYPE_BOOL) { p = getVectorDoubleValue_BOOL; + } else if (srcType == TSDB_DATA_TYPE_NULL) { + p = NULL; } else { - assert(0); + ASSERT(0); } return p; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index c17e854aa9..5522bfc6a3 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1031,18 +1031,29 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; out.columnData->info.type = type; + out.columnData->info.bytes = tDataTypes[type].bytes; for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { SValueNode *valueNode = (SValueNode *)cell->pNode; - code = doConvertDataType(valueNode, &out); - if (code) { -// fltError("convert from %d to %d failed", in.type, out.type); - FLT_ERR_RET(code); - } - - len = tDataTypes[type].bytes; + if (valueNode->node.resType.type != type) { + code = doConvertDataType(valueNode, &out); + if (code) { + // fltError("convert from %d to %d failed", in.type, out.type); + FLT_ERR_RET(code); + } + + len = tDataTypes[type].bytes; - filterAddField(info, NULL, (void**) &out.columnData->pData, FLD_TYPE_VALUE, &right, len, true); + filterAddField(info, NULL, (void**) &out.columnData->pData, FLD_TYPE_VALUE, &right, len, true); + out.columnData->pData = NULL; + } else { + void *data = taosMemoryCalloc(1, tDataTypes[type].bytes); + if (NULL == data) { + FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + memcpy(data, nodesGetValueFromNode(valueNode), tDataTypes[type].bytes); + filterAddField(info, NULL, (void**) &data, FLD_TYPE_VALUE, &right, len, true); + } filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx); SFilterGroup fgroup = {0}; @@ -3576,6 +3587,11 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } + if (node->opType == OP_TYPE_NOT_IN || node->opType == OP_TYPE_NOT_LIKE || node->opType > OP_TYPE_IS_NOT_NULL) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + if (NULL == node->pRight) { if (scalarGetOperatorParamNum(node->opType) > 1) { fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType); @@ -3599,7 +3615,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { + if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight)) && (QUERY_NODE_NODE_LIST != nodeType(node->pRight))) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } @@ -3774,11 +3790,12 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; output.columnData = createColumnInfoData(&type, pSrc->info.rows); - *p = (int8_t *)output.columnData->pData; SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); + *p = (int8_t *)output.columnData->pData; + taosArrayDestroy(pList); return false; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 294afc8073..34b0cf730d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -71,7 +71,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { SValueNode *valueNode = (SValueNode *)cell->pNode; - + if (valueNode->node.resType.type != type) { out.columnData->info.type = type; out.columnData->info.bytes = tDataTypes[type].bytes; @@ -176,7 +176,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCL_ERR_RET(scalarGenerateSetFromList((void**) ¶m->pHashFilter, node, nodeList->dataType.type)); + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, nodeList->dataType.type)); if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pHashFilter); sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); @@ -483,6 +483,79 @@ _return: SCL_RET(code); } +EDealRes sclRewriteBasedOnOptr(SNode** pNode, SScalarCtx *ctx, EOperatorType opType) { + if (opType <= OP_TYPE_CALC_MAX) { + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType.type = TSDB_DATA_TYPE_NULL; + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + } else { + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType.type = TSDB_DATA_TYPE_BOOL; + res->datum.b = false; + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) { + SOperatorNode *node = (SOperatorNode *)*pNode; + + if (node->pLeft && (QUERY_NODE_VALUE == nodeType(node->pLeft))) { + SValueNode *valueNode = (SValueNode *)node->pLeft; + if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { + return sclRewriteBasedOnOptr(pNode, ctx, node->opType); + } + } + + if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) { + SValueNode *valueNode = (SValueNode *)node->pRight; + if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { + return sclRewriteBasedOnOptr(pNode, ctx, node->opType); + } + } + + if (node->pRight && (QUERY_NODE_NODE_LIST == nodeType(node->pRight))) { + SNodeListNode *listNode = (SNodeListNode *)node->pRight; + SNode* tnode = NULL; + WHERE_EACH(tnode, listNode->pNodeList) { + if (SCL_IS_NULL_VALUE_NODE(tnode)) { + if (node->opType == OP_TYPE_IN) { + ERASE_NODE(listNode->pNodeList); + continue; + } else { //OP_TYPE_NOT_IN + return sclRewriteBasedOnOptr(pNode, ctx, node->opType); + } + } + + WHERE_NEXT; + } + + if (listNode->pNodeList->length <= 0) { + return sclRewriteBasedOnOptr(pNode, ctx, node->opType); + } + } + + return DEAL_RES_CONTINUE; +} + EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SNode* tnode = NULL; @@ -575,12 +648,8 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; - if (!SCL_IS_CONST_NODE(node->pLeft)) { - return DEAL_RES_CONTINUE; - } - - if (!SCL_IS_CONST_NODE(node->pRight)) { - return DEAL_RES_CONTINUE; + if ((!SCL_IS_CONST_NODE(node->pLeft)) || (!SCL_IS_CONST_NODE(node->pRight))) { + return sclRewriteOperatorForNullValue(pNode, ctx); } SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; @@ -591,21 +660,25 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { - sclError("make value node failed"); - sclFreeParam(&output); + sclError("make value node failed"); + sclFreeParam(&output); ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } - res->node.resType = node->node.resType; res->translate = true; - int32_t type = output.columnData->info.type; - if (IS_VAR_DATA_TYPE(type)) { // todo refactor - res->datum.p = output.columnData->pData; - output.columnData->pData = NULL; + if (colDataIsNull_s(output.columnData, 0)) { + res->node.resType.type = TSDB_DATA_TYPE_NULL; } else { - memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + res->node.resType = node->node.resType; + int32_t type = output.columnData->info.type; + if (IS_VAR_DATA_TYPE(type)) { // todo refactor + res->datum.p = output.columnData->pData; + output.columnData->pData = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + } } nodesDestroyNode(*pNode); @@ -628,7 +701,7 @@ EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { return sclRewriteOperator(pNode, ctx); - } + } return DEAL_RES_CONTINUE; } @@ -636,7 +709,7 @@ EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)pNode; SScalarParam output = {0}; - + ctx->code = sclExecFunction(node, ctx, &output); if (ctx->code) { return DEAL_RES_ERROR; @@ -653,7 +726,7 @@ EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { EDealRes sclWalkLogic(SNode* pNode, SScalarCtx *ctx) { SLogicConditionNode *node = (SLogicConditionNode *)pNode; SScalarParam output = {0}; - + ctx->code = sclExecLogic(node, ctx, &output); if (ctx->code) { return DEAL_RES_ERROR; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f63e539a96..4fcebedc98 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -150,34 +150,36 @@ int64_t getVectorBigintValue_JSON(void *src, int32_t index){ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { _getBigintValue_fn_t p = NULL; - if(srcType==TSDB_DATA_TYPE_TINYINT) { - p = getVectorBigintValue_TINYINT; - }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { - p = getVectorBigintValue_UTINYINT; - }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { - p = getVectorBigintValue_SMALLINT; - }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { - p = getVectorBigintValue_USMALLINT; - }else if(srcType==TSDB_DATA_TYPE_INT) { - p = getVectorBigintValue_INT; - }else if(srcType==TSDB_DATA_TYPE_UINT) { - p = getVectorBigintValue_UINT; - }else if(srcType==TSDB_DATA_TYPE_BIGINT) { - p = getVectorBigintValue_BIGINT; - }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { - p = getVectorBigintValue_UBIGINT; - }else if(srcType==TSDB_DATA_TYPE_FLOAT) { - p = getVectorBigintValue_FLOAT; - }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { - p = getVectorBigintValue_DOUBLE; - }else if(srcType==TSDB_DATA_TYPE_TIMESTAMP) { - p = getVectorBigintValue_BIGINT; - }else if(srcType==TSDB_DATA_TYPE_BOOL) { - p = getVectorBigintValue_BOOL; - }else if(srcType==TSDB_DATA_TYPE_JSON) { - p = getVectorBigintValue_JSON; - }else { - assert(0); + if (srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorBigintValue_TINYINT; + } else if (srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorBigintValue_UTINYINT; + } else if (srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorBigintValue_SMALLINT; + } else if (srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorBigintValue_USMALLINT; + } else if (srcType==TSDB_DATA_TYPE_INT) { + p = getVectorBigintValue_INT; + } else if (srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorBigintValue_UINT; + } else if (srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorBigintValue_BIGINT; + } else if (srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorBigintValue_UBIGINT; + } else if (srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorBigintValue_FLOAT; + } else if (srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorBigintValue_DOUBLE; + } else if (srcType==TSDB_DATA_TYPE_TIMESTAMP) { + p = getVectorBigintValue_BIGINT; + } else if (srcType==TSDB_DATA_TYPE_BOOL) { + p = getVectorBigintValue_BOOL; + } else if (srcType==TSDB_DATA_TYPE_JSON) { + p = getVectorBigintValue_JSON; + } else if (srcType==TSDB_DATA_TYPE_NULL){ + p = NULL; + } else { + ASSERT(0); } return p; } @@ -1367,7 +1369,8 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pRight->pHashFilter != NULL) { for (; i >= 0 && i < pLeft->numOfRows; i += step) { if (colDataIsNull_s(pLeft->columnData, i)) { - colDataAppendNULL(pOut->columnData, i); + bool res = false; + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; } @@ -1381,7 +1384,8 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step) { if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { - colDataAppendNULL(pOut->columnData, i); + bool res = false; + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; // TODO set null or ignore } @@ -1402,8 +1406,9 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * } else if (pRight->numOfRows == 1) { ASSERT(pLeft->pHashFilter == NULL); for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (colDataIsNull_s(pLeft->columnData, i)) { - colDataAppendNULL(pOut->columnData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, 0)) { + bool res = false; + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; } @@ -1422,8 +1427,9 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * } } else if (pLeft->numOfRows == 1) { for (; i >= 0 && i < pRight->numOfRows; i += step) { - if (colDataIsNull_s(pRight->columnData, i)) { - colDataAppendNULL(pOut->columnData, i); + if (colDataIsNull_s(pRight->columnData, i) || colDataIsNull_s(pLeft->columnData, 0)) { + bool res = false; + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; } @@ -1594,7 +1600,7 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { case OP_TYPE_JSON_CONTAINS: return vectorJsonContains; default: - assert(0); + ASSERT(0); return NULL; } } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 1aff0951e1..31d2da9380 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1076,17 +1076,17 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SVCreateTbBatchRsp batchRsp = {0}; if (msg) { SCH_ERR_JRET(tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp)); - if (batchRsp.rspList) { - int32_t num = taosArrayGetSize(batchRsp.rspList); + if (batchRsp.pArray) { + int32_t num = taosArrayGetSize(batchRsp.pArray); for (int32_t i = 0; i < num; ++i) { - SVCreateTbRsp *rsp = taosArrayGet(batchRsp.rspList, i); + SVCreateTbRsp *rsp = taosArrayGet(batchRsp.pArray, i); if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { - taosArrayDestroy(batchRsp.rspList); + taosArrayDestroy(batchRsp.pArray); SCH_ERR_JRET(rsp->code); } } - taosArrayDestroy(batchRsp.rspList); + taosArrayDestroy(batchRsp.pArray); } } diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index e9550ff989..4f91990ccf 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -39,6 +39,8 @@ extern "C" { #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) typedef struct SSyncEnv { + uint8_t isStart; + // tick timer tmr_h pEnvTickTimer; int32_t envTickTimerMS; diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index ff2d4d4d27..4b2bc4130a 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -26,6 +26,14 @@ static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv); static void syncEnvTick(void *param, void *tmrId); // -------------------------------- +bool syncEnvIsStart() { + if (gSyncEnv == NULL) { + return false; + } + + return atomic_load_8(&(gSyncEnv->isStart)); +} + int32_t syncEnvStart() { int32_t ret = 0; taosSeedRand(taosGetTimestampSec()); @@ -88,12 +96,15 @@ static SSyncEnv *doSyncEnvStart() { // start tmr thread pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); + + atomic_store_8(&(pSyncEnv->isStart), 1); return pSyncEnv; } static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { assert(pSyncEnv == gSyncEnv); if (pSyncEnv != NULL) { + atomic_store_8(&(pSyncEnv->isStart), 0); taosTmrCleanUp(pSyncEnv->pTimerManager); taosMemoryFree(pSyncEnv); } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0dfebe9820..0cee494ae8 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -42,8 +42,7 @@ struct SBTree { ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \ TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0)) - -#pragma pack(push,1) +#pragma pack(push, 1) typedef struct { TDB_BTREE_PAGE_COMMON_HDR } SLeafHdr; @@ -71,8 +70,7 @@ typedef struct { static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbBtreeOpenImpl(SBTree *pBt); -static int tdbBtreeZeroPage(SPage *pPage, void *arg); -static int tdbBtreeInitPage(SPage *pPage, void *arg); +static int tdbBtreeInitPage(SPage *pPage, void *arg, int init); static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); @@ -312,75 +310,57 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { } // Try to create a new database - SBtreeInitPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; - ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg, NULL); + ret = tdbPagerAllocPage(pBt->pPager, &pgno); if (ret < 0) { + ASSERT(0); return -1; } - // TODO: here still has problem - tdbPagerReturnPage(pBt->pPager, pPage, NULL); - ASSERT(pgno != 0); pBt->root = pgno; return 0; } -static int tdbBtreeInitPage(SPage *pPage, void *arg) { +static int tdbBtreeInitPage(SPage *pPage, void *arg, int init) { SBTree *pBt; u8 flags; - u8 isLeaf; - - pBt = (SBTree *)arg; - flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); - isLeaf = TDB_BTREE_PAGE_IS_LEAF(pPage); - - ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); - - tdbPageInit(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); - - TDB_BTREE_ASSERT_FLAG(flags); - - if (isLeaf) { - pPage->kLen = pBt->keyLen; - pPage->vLen = pBt->valLen; - pPage->maxLocal = pBt->maxLeaf; - pPage->minLocal = pBt->minLeaf; - } else { - pPage->kLen = pBt->keyLen; - pPage->vLen = sizeof(SPgno); - pPage->maxLocal = pBt->maxLocal; - pPage->minLocal = pBt->minLocal; - } - - return 0; -} - -static int tdbBtreeZeroPage(SPage *pPage, void *arg) { - u8 flags; - SBTree *pBt; u8 leaf; - flags = ((SBtreeInitPageArg *)arg)->flags; pBt = ((SBtreeInitPageArg *)arg)->pBt; - leaf = flags & TDB_BTREE_LEAF; - tdbPageZero(pPage, leaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); + if (init) { + // init page + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + leaf = TDB_BTREE_PAGE_IS_LEAF(pPage); + TDB_BTREE_ASSERT_FLAG(flags); + + tdbPageInit(pPage, leaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); + } else { + // zero page + flags = ((SBtreeInitPageArg *)arg)->flags; + leaf = flags & TDB_BTREE_LEAF; + TDB_BTREE_ASSERT_FLAG(flags); + + tdbPageZero(pPage, leaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); + + if (leaf) { + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pData); + pLeafHdr->flags = flags; + + } else { + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pData); + pIntHdr->flags = flags; + pIntHdr->pgno = 0; + } + } if (leaf) { - SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pData); - pLeafHdr->flags = flags; - pPage->kLen = pBt->keyLen; pPage->vLen = pBt->valLen; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { - SIntHdr *pIntHdr = (SIntHdr *)(pPage->pData); - pIntHdr->flags = flags; - pIntHdr->pgno = 0; - pPage->kLen = pBt->keyLen; pPage->vLen = sizeof(SPgno); pPage->maxLocal = pBt->maxLocal; @@ -405,10 +385,11 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot); leaf = TDB_BTREE_PAGE_IS_LEAF(pRoot); - // Allocate a new child page + // allocate a new child page + pgnoChild = 0; zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); zArg.pBt = pBt; - ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg, pTxn); + ret = tdbPagerFetchPage(pPager, &pgnoChild, &pChild, tdbBtreeInitPage, &zArg, pTxn); if (ret < 0) { return -1; } @@ -430,7 +411,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN // Reinitialize the root page zArg.flags = TDB_BTREE_ROOT; zArg.pBt = pBt; - ret = tdbBtreeZeroPage(pRoot, &zArg); + ret = tdbBtreeInitPage(pRoot, &zArg, 0); if (ret < 0) { return -1; } @@ -483,7 +464,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx pgno = *(SPgno *)pCell; } - ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt, pTxn); + ret = tdbPagerFetchPage(pBt->pPager, &pgno, pOlds + i, tdbBtreeInitPage, + &((SBtreeInitPageArg){.pBt = pBt, .flags = 0}), pTxn); if (ret < 0) { ASSERT(0); return -1; @@ -644,9 +626,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx if (iNew < nOlds) { pNews[iNew] = pOlds[iNew]; } else { + pgno = 0; iarg.pBt = pBt; iarg.flags = flags; - ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg, pTxn); + ret = tdbPagerFetchPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeInitPage, &iarg, pTxn); if (ret < 0) { ASSERT(0); } @@ -674,13 +657,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx iarg.pBt = pBt; iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); for (int i = 0; i < nOlds; i++) { - tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], NULL, NULL); - tdbBtreeZeroPage(pOldsCopy[i], &iarg); + tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], tdbDefaultMalloc, NULL); + tdbBtreeInitPage(pOldsCopy[i], &iarg, 0); tdbPageCopy(pOlds[i], pOldsCopy[i]); } iNew = 0; nNewCells = 0; - tdbBtreeZeroPage(pNews[iNew], &iarg); + tdbBtreeInitPage(pNews[iNew], &iarg, 0); for (int iOld = 0; iOld < nOlds; iOld++) { SPage *pPage; @@ -721,7 +704,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx iNew++; nNewCells = 0; if (iNew < nNews) { - tdbBtreeZeroPage(pNews[iNew], &iarg); + tdbBtreeInitPage(pNews[iNew], &iarg, 0); } } } else { @@ -740,7 +723,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx iNew++; nNewCells = 0; if (iNew < nNews) { - tdbBtreeZeroPage(pNews[iNew], &iarg); + tdbBtreeInitPage(pNews[iNew], &iarg, 0); } } } @@ -760,7 +743,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } for (int i = 0; i < nOlds; i++) { - tdbPageDestroy(pOldsCopy[i], NULL, NULL); + tdbPageDestroy(pOldsCopy[i], tdbDefaultFree, NULL); } } @@ -1035,7 +1018,13 @@ int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) { pBtc->iPage = -1; pBtc->pPage = NULL; pBtc->idx = -1; - pBtc->pTxn = pTxn; + + if (pTxn == NULL) { + pBtc->pTxn = &pBtc->txn; + tdbTxnOpen(pBtc->pTxn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); + } else { + pBtc->pTxn = pTxn; + } return 0; } @@ -1052,7 +1041,8 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { if (pBtc->iPage < 0) { // move a clean cursor - ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn); + ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, + &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { ASSERT(0); return -1; @@ -1117,7 +1107,8 @@ int tdbBtcMoveToLast(SBTC *pBtc) { if (pBtc->iPage < 0) { // move a clean cursor - ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn); + ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, + &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { ASSERT(0); return -1; @@ -1286,13 +1277,16 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; } + ASSERT(pgno); + pBtc->pgStack[pBtc->iPage] = pBtc->pPage; pBtc->idxStack[pBtc->iPage] = pBtc->idx; pBtc->iPage++; pBtc->pPage = NULL; pBtc->idx = -1; - ret = tdbPagerFetchPage(pBtc->pBt->pPager, pgno, &pBtc->pPage, tdbBtreeInitPage, pBtc->pBt, pBtc->pTxn); + ret = tdbPagerFetchPage(pBtc->pBt->pPager, &pgno, &pBtc->pPage, tdbBtreeInitPage, + &((SBtreeInitPageArg){.pBt = pBtc->pBt, .flags = 0}), pBtc->pTxn); if (ret < 0) { ASSERT(0); return -1; @@ -1327,7 +1321,8 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { if (pBtc->iPage < 0) { // move from a clear cursor - ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn); + ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, + &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { // TODO ASSERT(0); diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index ce753ad930..0ccf7f5f92 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -95,6 +95,8 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { i32 nRef; + ASSERT(pTxn); + nRef = TDB_UNREF_PAGE(pPage); ASSERT(nRef >= 0); @@ -108,13 +110,12 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { if (pPage->isLocal) { tdbPCacheUnpinPage(pCache, pPage); } else { - // remove from hash - tdbPCacheRemovePageFromHash(pCache, pPage); - - // free the page - if (pTxn && pTxn->xFree) { - tdbPageDestroy(pPage, pTxn->xFree, pTxn->xArg); + if (TDB_TXN_IS_WRITE(pTxn)) { + // remove from hash + tdbPCacheRemovePageFromHash(pCache, pPage); } + + tdbPageDestroy(pPage, pTxn->xFree, pTxn->xArg); } } @@ -125,8 +126,11 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; } static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { - int ret; - SPage *pPage; + int ret = 0; + SPage *pPage = NULL; + SPage *pPageH = NULL; + + ASSERT(pTxn); // 1. Search the hash table pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; @@ -136,12 +140,17 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) } if (pPage) { - // TODO: the page need to be copied and - // replaced the page in hash table - tdbPCachePinPage(pCache, pPage); - return pPage; + if (pPage->isLocal || TDB_TXN_IS_WRITE(pTxn)) { + tdbPCachePinPage(pCache, pPage); + return pPage; + } } + // 1. pPage == NULL + // 2. pPage && pPage->isLocal == 0 && !TDB_TXN_IS_WRITE(pTxn) + pPageH = pPage; + pPage = NULL; + // 2. Try to allocate a new page from the free list if (pCache->pFree) { pPage = pCache->pFree; @@ -158,7 +167,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) } // 4. Try a create new page - if (!pPage && pTxn && pTxn->xMalloc) { + if (!pPage) { ret = tdbPageCreate(pCache->pageSize, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0) { // TODO @@ -176,12 +185,27 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) // or by recycling or allocated streesly, // need to initialize it if (pPage) { - memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); - pPage->pLruNext = NULL; - pPage->pPager = NULL; + if (pPageH) { + // copy the page content + memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); + pPage->pLruNext = NULL; + pPage->pPager = pPageH->pPager; - // TODO: allocated page may not add to hash - tdbPCacheAddPageToHash(pCache, pPage); + memcpy(pPage->pData, pPageH->pData, pPage->pageSize); + tdbPageInit(pPage, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize); + pPage->kLen = pPageH->kLen; + pPage->vLen = pPageH->vLen; + pPage->maxLocal = pPageH->maxLocal; + pPage->minLocal = pPageH->minLocal; + } else { + memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); + pPage->pLruNext = NULL; + pPage->pPager = NULL; + + if (pPage->isLocal || TDB_TXN_IS_WRITE(pTxn)) { + tdbPCacheAddPageToHash(pCache, pPage); + } + } } return pPage; @@ -249,7 +273,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pCache->nFree = 0; pCache->pFree = NULL; for (int i = 0; i < pCache->cacheSize; i++) { - ret = tdbPageCreate(pCache->pageSize, &pPage, NULL, NULL); + ret = tdbPageCreate(pCache->pageSize, &pPage, tdbDefaultMalloc, NULL); if (ret < 0) { // TODO: handle error return -1; @@ -272,7 +296,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { // Open the hash table pCache->nPage = 0; - pCache->nHash = pCache->cacheSize; + pCache->nHash = pCache->cacheSize < 8 ? 8 : pCache->cacheSize; pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *)); if (pCache->pgHash == NULL) { // TODO diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 05939579d8..f0fdd82944 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -43,15 +43,14 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) u8 *ptr; int size; + ASSERT(xMalloc); + ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); *ppPage = NULL; size = pageSize + sizeof(*pPage); - if (xMalloc == NULL) { - xMalloc = tdbDefaultMalloc; - } - ptr = (u8 *)((*xMalloc)(arg, size)); + ptr = (u8 *)(xMalloc(arg, size)); if (ptr == NULL) { return -1; } @@ -75,12 +74,10 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; - if (!xFree) { - xFree = tdbDefaultFree; - } + ASSERT(xFree); ptr = pPage->pData; - (*xFree)(arg, ptr); + xFree(arg, ptr); return 0; } @@ -436,7 +433,7 @@ static int tdbPageDefragment(SPage *pPage) { /* ---------------------------------------------------------------------------------------------------------- */ -#pragma pack(push,1) +#pragma pack(push, 1) typedef struct { u16 cellNum; u16 cellBody; @@ -520,7 +517,7 @@ SPageMethods pageMethods = { setPageFreeCellInfo // setFreeCellInfo }; -#pragma pack(push,1) +#pragma pack(push, 1) typedef struct { u8 cellNum[3]; u8 cellBody[3]; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 8835b7f759..1941592602 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -15,7 +15,7 @@ #include "tdbInt.h" -#pragma pack(push,1) +#pragma pack(push, 1) typedef struct { u8 hdrString[16]; u16 pageSize; @@ -29,7 +29,8 @@ TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct") #define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL) -static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage); +static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *, int), void *arg, + u8 loadPage); static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage); static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage); @@ -80,6 +81,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { pPager->pageSize = tdbPCacheGetPageSize(pCache); // pPager->dbOrigSize ret = tdbGetFileSize(pPager->fd, pPager->pageSize, &(pPager->dbOrigSize)); + pPager->dbFileSize = pPager->dbOrigSize; *ppPager = pPager; return 0; @@ -228,13 +230,30 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { return 0; } -int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg, +int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn) { SPage *pPage; SPgid pgid; int ret; + SPgno pgno; + u8 loadPage; - // Fetch a page container from the page cache + pgno = *ppgno; + loadPage = 1; + + // alloc new page + if (pgno == 0) { + loadPage = 0; + ret = tdbPagerAllocPage(pPager, &pgno); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + + ASSERT(pgno > 0); + + // fetch a page container memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); pgid.pgno = pgno; pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn); @@ -242,9 +261,9 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage return -1; } - // Initialize the page if need + // init page if need if (!TDB_PAGE_INITIALIZED(pPage)) { - ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 1); + ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); if (ret < 0) { return -1; } @@ -253,46 +272,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage ASSERT(TDB_PAGE_INITIALIZED(pPage)); ASSERT(pPage->pPager == pPager); - *ppPage = pPage; - return 0; -} - -int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg, - TXN *pTxn) { - int ret; - SPage *pPage; - SPgid pgid; - - // Allocate a page number - ret = tdbPagerAllocPage(pPager, ppgno); - if (ret < 0) { - ASSERT(0); - return -1; - } - - ASSERT(*ppgno != 0); - - // Fetch a page container from the page cache - memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); - pgid.pgno = *ppgno; - pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn); - if (pPage == NULL) { - ASSERT(0); - return -1; - } - - ASSERT(!TDB_PAGE_INITIALIZED(pPage)); - - // Initialize the page if need - ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 0); - if (ret < 0) { - ASSERT(0); - return -1; - } - - ASSERT(TDB_PAGE_INITIALIZED(pPage)); - ASSERT(pPage->pPager == pPager); - + *ppgno = pgno; *ppPage = pPage; return 0; } @@ -333,11 +313,14 @@ int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { return 0; } -static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage) { - int ret; - int lcode; - int nLoops; - i64 nRead; +static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *, int), void *arg, + u8 loadPage) { + int ret; + int lcode; + int nLoops; + i64 nRead; + SPgno pgno; + int init = 0; lcode = TDB_TRY_LOCK_PAGE(pPage); if (lcode == P_LOCK_SUCC) { @@ -346,20 +329,21 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage return 0; } - if (loadPage) { - nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * TDB_PAGE_PGNO(pPage)); - if (nRead < 0) { - // TODO - ASSERT(0); - return -1; - } else if (nRead < pPage->pageSize) { - // TODO + pgno = TDB_PAGE_PGNO(pPage); + + if (loadPage && pgno <= pPager->dbOrigSize) { + init = 1; + + nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * (pgno - 1)); + if (nRead < pPage->pageSize) { ASSERT(0); return -1; } + } else { + init = 0; } - ret = (*initPage)(pPage, arg); + ret = (*initPage)(pPage, arg, init); if (ret < 0) { TDB_UNLOCK_PAGE(pPage); return -1; @@ -409,7 +393,7 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { i64 offset; int ret; - offset = pPage->pageSize * TDB_PAGE_PGNO(pPage); + offset = pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { ASSERT(0); return -1; diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index 3cdd30c7b5..2a9e8533d2 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -36,6 +36,7 @@ struct SBTC { int idxStack[BTREE_MAX_DEPTH + 1]; SPage *pgStack[BTREE_MAX_DEPTH + 1]; TXN *pTxn; + TXN txn; }; // SBTree diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h index ca196785d0..c39f833c73 100644 --- a/source/libs/tdb/src/inc/tdbPager.h +++ b/source/libs/tdb/src/inc/tdbPager.h @@ -42,10 +42,8 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); int tdbPagerWrite(SPager *pPager, SPage *pPage); int tdbPagerBegin(SPager *pPager, TXN *pTxn); int tdbPagerCommit(SPager *pPager, TXN *pTxn); -int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg, +int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn); -int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg, - TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 904ad64fe7..49763ae937 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,5 +1,6 @@ #include +#include "os.h" #include "tdbInt.h" #include @@ -122,6 +123,8 @@ TEST(tdb_test, simple_test) { int nData = 10000000; TXN txn; + taosRemoveDir("tdb"); + // Open Env ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); GTEST_ASSERT_EQ(ret, 0); @@ -225,6 +228,91 @@ TEST(tdb_test, simple_test) { // Close a database tdbDbClose(pDb); + // Close Env + ret = tdbEnvClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); +} + +TEST(tdb_test, simple_test2) { + int ret; + TENV *pEnv; + TDB *pDb; + FKeyComparator compFunc; + int nData = 1000000; + TXN txn; + + taosRemoveDir("tdb"); + + // Open Env + ret = tdbEnvOpen("tdb", 1024, 10, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tDefaultKeyCmpr; + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + { + char key[64]; + char val[64]; + int64_t txnid = 0; + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + txnid++; + tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn); + + for (int iData = 1; iData <= nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(val, "value%d", iData); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // Iterate to query the DB data + TDBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + + ret = tdbDbcOpen(pDb, &pDBC); + GTEST_ASSERT_EQ(ret, 0); + + for (;;) { + ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + + std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + std::cout << std::endl; + + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbDbcClose(pDBC); + + TDB_FREE(pKey); + TDB_FREE(pVal); + } + } + + // commit the transaction + tdbCommit(pEnv, &txn); + tdbTxnClose(&txn); + + ret = tdbDbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbDbClose(pDb); + // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index d025ca3781..dc31086e9f 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -251,11 +251,14 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body, int32_t bodyLen) { - if (pWal == NULL) return -1; int code = 0; // no wal if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0; + if (bodyLen > TSDB_MAX_WAL_SIZE) { + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + return -1; + } if (index == pWal->vers.lastVer + 1) { if (taosArrayGetSize(pWal->fileInfoSet) == 0) { diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 0ddc76e415..7f1fdf3306 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -433,7 +433,8 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - int32_t ret = WCSPatternMatch((TdUcs4*)pattern, (TdUcs4*)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); + int32_t ret = + WCSPatternMatch((TdUcs4 *)pattern, (TdUcs4 *)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); taosMemoryFree(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 5bfef810c3..b864f48522 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -145,12 +145,6 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } - if (taosRealPath(fullDir, NULL, PATH_MAX) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); - return -1; - } - taosMemoryFreeClear(pItem->str); pItem->str = strdup(fullDir); if (pItem->str == NULL) { diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index c40d5b02e6..037eba5ed2 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -33,12 +33,19 @@ void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, pCoder->data = data; pCoder->size = size; pCoder->pos = 0; - tFreeListInit(&(pCoder->fl)); + pCoder->mList = NULL; TD_SLIST_INIT(&(pCoder->stack)); } void tCoderClear(SCoder* pCoder) { - tFreeListClear(&(pCoder->fl)); + SCoderMem* pMem; + + // clear memory + for (pMem = pCoder->mList; pMem; pMem = pCoder->mList) { + pCoder->mList = pMem->next; + taosMemoryFree(pMem); + } + struct SCoderNode* pNode; for (;;) { pNode = TD_SLIST_HEAD(&(pCoder->stack)); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 399a2255ac..61f2015fe5 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -144,17 +144,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt cl // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Mnode not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, "Message need to be reprocessed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_OPTIONS, "Invalid mnode options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_VERSION, "Incompatible protocol version") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_LEN, "Invalid message length") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_TYPE, "Invalid message type") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_SHELL_CONNS, "Too many connections") // mnode-show TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired") @@ -277,8 +270,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exist TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") -// mnode-topic +// mnode-mq TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with aggregation is unsupported") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer waiting for rebalance") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_EXIST, "Consumer not exist") // mnode-sma TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 3564700d53..8531f2c8ea 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -92,6 +92,7 @@ int32_t sDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t tqDebugFlag = 135; int32_t fsDebugFlag = 135; +int32_t metaDebugFlag = 135; int32_t fnDebugFlag = 135; int64_t dbgEmptyW = 0; diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index b4da43dfb7..9ddbd24353 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -230,7 +230,7 @@ static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { const char *tstr; uint64_t len; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; - TCODER_MALLOC(pSAV1->A_c, char*, len + 1, pCoder); + pSAV1->A_c = (char *)TCODER_MALLOC(pCoder, len + 1); memcpy(pSAV1->A_c, tstr, len + 1); tEndDecode(pCoder); @@ -269,7 +269,7 @@ static int32_t tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { const char *tstr; uint64_t len; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; - TCODER_MALLOC(pSAV2->A_c, char*, len + 1, pCoder); + pSAV2->A_c = (char *)TCODER_MALLOC(pCoder, len + 1); memcpy(pSAV2->A_c, tstr, len + 1); // ------------------------NEW FIELDS DECODE------------------------------- @@ -305,7 +305,7 @@ static int32_t tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { static int32_t tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { if (tStartDecode(pCoder) < 0) return -1; - TCODER_MALLOC(ps1->pA, SStructA_v1*, sizeof(*(ps1->pA)), pCoder); + ps1->pA = (SStructA_v1 *)TCODER_MALLOC(pCoder, sizeof(*(ps1->pA))); if (tSStructA_v1_decode(pCoder, ps1->pA) < 0) return -1; if (tDecodeI32(pCoder, &ps1->v_a) < 0) return -1; if (tDecodeI8(pCoder, &ps1->v_b) < 0) return -1; @@ -339,7 +339,7 @@ static int32_t tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { if (tStartDecode(pCoder) < 0) return -1; - TCODER_MALLOC(ps2->pA, SStructA_v2*, sizeof(*(ps2->pA)), pCoder); + ps2->pA = (SStructA_v2 *)TCODER_MALLOC(pCoder, sizeof(*(ps2->pA))); if (tSStructA_v2_decode(pCoder, ps2->pA) < 0) return -1; if (tDecodeI32(pCoder, &ps2->v_a) < 0) return -1; if (tDecodeI8(pCoder, &ps2->v_b) < 0) return -1; diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 2e046bd3ff..8fd3272055 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -31,7 +31,7 @@ typedef struct { char* binaryData; char* isNull; int32_t* binaryLen; - TAOS_BIND_v2* pBind; + TAOS_MULTI_BIND* pBind; char* sql; int32_t* colTypes; int32_t colNum; @@ -97,15 +97,14 @@ CaseCfg gCase[] = { {"insert:MBME4-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, {"insert:MBME4-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, - {"insert:MPME1-FULL", tListLen(fullColList), fullColList, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 1}, {"insert:MPME1-C012", tListLen(fullColList), fullColList, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1}, - }; CaseCfg *gCurCase = NULL; typedef struct { + char caseCatalog[255]; int32_t bindNullNum; bool autoCreate; bool checkParamNum; @@ -118,6 +117,8 @@ typedef struct { int32_t bindColTypeNum; int32_t* bindColTypeList; int32_t runTimes; + int32_t caseIdx; + int32_t caseRunNum; } CaseCtrl; CaseCtrl gCaseCtrl = { @@ -133,6 +134,8 @@ CaseCtrl gCaseCtrl = { .checkParamNum = false, .printRes = true, .runTimes = 0, + .caseIdx = -1, + .caseRunNum = -1, }; int32_t taosGetTimeOfDay(struct timeval *tv) { @@ -163,7 +166,7 @@ static int64_t taosGetTimestampUs() { return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; } -bool colExists(TAOS_BIND_v2* pBind, int32_t dataType) { +bool colExists(TAOS_MULTI_BIND* pBind, int32_t dataType) { int32_t i = 0; while (true) { if (0 == pBind[i].buffer_type) { @@ -393,7 +396,7 @@ int32_t prepareData(BindData *data) { data->colNum = 0; data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); data->sql = taosMemoryCalloc(1, 1024); - data->pBind = taosMemoryCalloc((allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_BIND_v2)); + data->pBind = taosMemoryCalloc((allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); data->tsData = taosMemoryMalloc(allRowNum * sizeof(int64_t)); data->boolData = taosMemoryMalloc(allRowNum * sizeof(bool)); data->tinyData = taosMemoryMalloc(allRowNum * sizeof(int8_t)); @@ -463,7 +466,7 @@ void destroyData(BindData *data) { taosMemoryFree(data->colTypes); } -int32_t bpBindParam(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { +int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { static int32_t n = 0; if (gCurCase->bindRowNum > 1) { @@ -951,7 +954,7 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1164,7 +1167,7 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1378,7 +1381,7 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1569,7 +1572,7 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*5); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); // int one_null = 1; int one_not_null = 0; @@ -1722,7 +1725,7 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1911,7 +1914,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2128,7 +2131,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2343,7 +2346,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2570,7 +2573,7 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2791,7 +2794,7 @@ int stmt_funcb4(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -2950,7 +2953,7 @@ int stmt_funcb5(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 3000*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 18000); char* no_null = taosMemoryMalloc(sizeof(char) * 18000); @@ -3094,7 +3097,7 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(30000 * sizeof(int)); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 3000*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); char* no_null = taosMemoryMalloc(sizeof(int) * 200000); for (int i = 0; i < 30000; ++i) { @@ -3185,7 +3188,7 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3347,7 +3350,7 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3505,7 +3508,7 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3665,7 +3668,7 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 60*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -4183,9 +4186,15 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int autoCreate) { void* runcase(TAOS *taos) { TAOS_STMT *stmt = NULL; - int32_t caseIdx = 0; + static int32_t caseIdx = 0; + static int32_t caseRunNum = 0; + int64_t beginUs, endUs, totalUs; for (int32_t i = 0; i < sizeof(gCase)/sizeof(gCase[0]); ++i) { + if (gCaseCtrl.caseRunNum > 0 && caseRunNum >= gCaseCtrl.caseRunNum) { + break; + } + CaseCfg cfg = gCase[i]; gCurCase = &cfg; @@ -4193,7 +4202,10 @@ void* runcase(TAOS *taos) { continue; } - printf("* Case %d - %s Begin *\n", caseIdx, gCurCase->caseDesc); + if (gCaseCtrl.caseIdx >= 0 && caseIdx < gCaseCtrl.caseIdx) { + caseIdx++; + continue; + } if (gCaseCtrl.runTimes) { gCurCase->runTimes = gCaseCtrl.runTimes; @@ -4220,10 +4232,15 @@ void* runcase(TAOS *taos) { gCurCase->bindColNum = gCaseCtrl.bindColTypeNum; gCurCase->fullCol = false; } - + + printf("* Case %d - [%s]%s Begin *\n", caseIdx, gCaseCtrl.caseCatalog, gCurCase->caseDesc); + + totalUs = 0; for (int32_t n = 0; n < gCurCase->runTimes; ++n) { prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->autoCreate); - + + beginUs = taosGetTimestampUs(); + stmt = taos_stmt_init(taos); if (NULL == stmt) { printf("taos_stmt_init failed, error:%s\n", taos_stmt_errstr(stmt)); @@ -4232,62 +4249,73 @@ void* runcase(TAOS *taos) { (*gCurCase->runFn)(stmt); - prepareCheckResult(taos); - taos_stmt_close(stmt); + + endUs = taosGetTimestampUs(); + totalUs += (endUs - beginUs); + + prepareCheckResult(taos); } - printf("* Case %d - %s End *\n", caseIdx, gCurCase->caseDesc); + printf("* Case %d - [%s]%s [AvgTime:%.3fms] End *\n", caseIdx, gCaseCtrl.caseCatalog, gCurCase->caseDesc, ((double)totalUs)/1000/gCurCase->runTimes); caseIdx++; + caseRunNum++; } - printf("test end\n"); - return NULL; - } void runAll(TAOS *taos) { - printf("Normal Test\n"); + strcpy(gCaseCtrl.caseCatalog, "Normal Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); runcase(taos); - printf("Null Test\n"); + strcpy(gCaseCtrl.caseCatalog, "Null Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindNullNum = 1; runcase(taos); gCaseCtrl.bindNullNum = 0; - printf("Bind Row Test\n"); + strcpy(gCaseCtrl.caseCatalog, "Bind Row Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindRowNum = 1; runcase(taos); gCaseCtrl.bindRowNum = 0; - printf("Row Num Test\n"); + strcpy(gCaseCtrl.caseCatalog, "Row Num Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.rowNum = 1000; gCaseCtrl.printRes = false; runcase(taos); gCaseCtrl.rowNum = 0; gCaseCtrl.printRes = true; - printf("Runtimes Test\n"); + strcpy(gCaseCtrl.caseCatalog, "Runtimes Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.runTimes = 2; runcase(taos); gCaseCtrl.runTimes = 0; - printf("Check Param Test\n"); + strcpy(gCaseCtrl.caseCatalog, "Check Param Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.checkParamNum = true; runcase(taos); gCaseCtrl.checkParamNum = false; - printf("Bind Col Num Test\n"); + strcpy(gCaseCtrl.caseCatalog, "Bind Col Num Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindColNum = 6; runcase(taos); gCaseCtrl.bindColNum = 0; - printf("Bind Col Type Test\n"); + strcpy(gCaseCtrl.caseCatalog, "Bind Col Type Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindColTypeNum = tListLen(bindColTypeList); gCaseCtrl.bindColTypeList = bindColTypeList; runcase(taos); + + printf("All Test End\n"); } int main(int argc, char *argv[]) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 83bd016d2f..6ec6d15028 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -34,6 +34,7 @@ ./test.sh -f tsim/query/charScalarFunction.sim ./test.sh -f tsim/query/explain.sim ./test.sh -f tsim/query/session.sim +./test.sh -f tsim/query/scalarNull.sim # ---- qnode ./test.sh -f tsim/qnode/basic1.sim @@ -82,7 +83,7 @@ ./test.sh -f tsim/mnode/basic1.sim -m # --- sma -./test.sh -f tsim/sma/tsmaCreateInsertData.sim +# ./test.sh -f tsim/sma/tsmaCreateInsertData.sim # --- valgrind #./test.sh -f tsim/valgrind/checkError.sim -v diff --git a/tests/script/sh/massiveTable/compileVersion.sh b/tests/script/sh/massiveTable/compileVersion.sh index 6b45e98ea3..e8c10b908e 100755 --- a/tests/script/sh/massiveTable/compileVersion.sh +++ b/tests/script/sh/massiveTable/compileVersion.sh @@ -44,7 +44,7 @@ function gitPullBranchInfo () { ## git submodule update --init --recursive git pull origin $branch_name ||: echo "==== git pull $branch_name end ====" - git pull --recurse-submodules +# git pull --recurse-submodules } function compileTDengineVersion() { diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 57324367b5..3b61b907de 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -126,6 +126,49 @@ if $data16_db != ns then # precision return -1 endi +sleep 3000 +sql show db.vgroups +if $data[0][4] == LEADER then + if $data[0][6] != FOLLOWER then + return -1 + endi + if $data[0][8] != FOLLOWER then + return -1 + endi +endi +if $data[0][6] == LEADER then + if $data[0][4] != FOLLOWER then + return -1 + endi + if $data[0][8] != FOLLOWER then + return -1 + endi +endi +if $data[0][8] == LEADER then + if $data[0][4] != FOLLOWER then + return -1 + endi + if $data[0][6] != FOLLOWER then + return -1 + endi +endi + +if $data[0][4] != LEADER then + if $data[0][4] != FOLLOWER then + return -1 + endi +endi +if $data[0][6] != LEADER then + if $data[0][6] != FOLLOWER then + return -1 + endi +endi +if $data[0][8] != LEADER then + if $data[0][8] != FOLLOWER then + return -1 + endi +endi + print ============== not support modify options: name, create_time, vgroups, ntables sql_error alter database db name dba sql_error alter database db create_time "2022-03-03 15:08:13.329" diff --git a/tests/script/tsim/query/scalarNull.sim b/tests/script/tsim/query/scalarNull.sim new file mode 100644 index 0000000000..c7e7aa9a34 --- /dev/null +++ b/tests/script/tsim/query/scalarNull.sim @@ -0,0 +1,92 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 + +print ========= start dnode1 as LEADER +system sh/exec.sh -n dnode1 -s start +sleep 2000 +sql connect + +print ======== step1 +sql create database db1 vgroups 3; +sql use db1; +sql show databases; +sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int); +sql create table tb1 using st1 tags(1); +sql insert into tb1 values ('2022-04-26 15:15:00', 1, "a"); +sql insert into tb1 values ('2022-04-26 15:15:01', 2, "b"); +sql insert into tb1 values ('2022-04-26 15:15:02', 3, "c"); +sql insert into tb1 values ('2022-04-26 15:15:03', 4, "d"); +sql insert into tb1 values ('2022-04-26 15:15:04', 5, "e"); +sql insert into tb1 values ('2022-04-26 15:15:05', 6, "f"); +sql insert into tb1 values ('2022-04-26 15:15:06', null, null); +sql insert into tb1 values ('2022-04-26 15:15:07', null, "g"); +sql insert into tb1 values ('2022-04-26 15:15:08', 7, null); + +sql select * from tb1 where f1 in (1,2,3); +if $rows != 3 then + return -1 +endi +sql select * from tb1 where f1 <>3; +if $rows != 6 then + return -1 +endi +sql select * from tb1 where f1 in (1,2,3,null); +if $rows != 3 then + return -1 +endi +sql select * from tb1 where f1 not in (1,2,3,null); +if $rows != 0 then + return -1 +endi +sql select * from tb1 where f1 in (null); +if $rows != 0 then + return -1 +endi +sql select * from tb1 where f1 = null; +if $rows != 0 then + return -1 +endi +sql select * from tb1 where f1 <> null; +if $rows != 0 then + return -1 +endi +sql select * from tb1 where f1 + 3 <> null; +if $rows != 0 then + return -1 +endi +sql select * from tb1 where f1+1 <>3+null; +if $rows != 0 then + return -1 +endi +sql select * from tb1 where f1+1*null <>3+null; +if $rows != 0 then + return -1 +endi +sql select * from tb1 where null; +if $rows != 0 then + return -1 +endi +sql select * from tb1 where null = null; +if $rows != 0 then + return -1 +endi +sql select * from tb1 where null <> null; +if $rows != 0 then + return -1 +endi + +#TODO: ENABLE IT +#sql select * from tb1 where not (null <> null); +#if $rows != 9 then +# return -1 +#endi + + +#TODO: MOVE IT TO NORMAL CASE +sql_error select * from tb1 where not (null); + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py new file mode 100644 index 0000000000..f9f80bb910 --- /dev/null +++ b/tests/system-test/0-others/taosdlog.py @@ -0,0 +1,65 @@ +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + tdSql.query("create user testpy pass 'testpy'") + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + logPath = buildPath + "/../sim/dnode1/log" + tdLog.info("log path: %s" % logPath) + + tdDnodes.stop(1) + time.sleep(2) + tdSql.error("show databases") + os.system("rm -rf %s" % logPath) + if os.path.exists(logPath) == True: + tdLog.exit("log pat still exist!") + + tdDnodes.start(1) + time.sleep(2) + if os.path.exists(logPath) != True: + tdLog.exit("log pat is not generated!") + + tdSql.query("show databases") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 79e1411cd1..0847162308 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +python3 ./test.py -f 0-others/taosdlog.py + #python3 ./test.py -f 2-query/between.py #python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index af28373510..d218bbf373 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -85,7 +85,6 @@ typedef struct { TAOS* conn; TdThread pid; tsem_t cancelSem; - int64_t result; } SShellObj; // shellArguments.c diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1f036ab25b..befcfc0731 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -213,13 +213,10 @@ void shellRunSingleCommandImp(char *command) { return; } - int64_t oresult = atomic_load_64(&shell.result); - if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - atomic_store_64(&shell.result, 0); taos_free_result(pSql); return; @@ -230,10 +227,7 @@ void shellRunSingleCommandImp(char *command) { int32_t error_no = 0; int32_t numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); - if (numOfRows < 0) { - atomic_store_64(&shell.result, 0); - return; - } + if (numOfRows < 0) return; et = taosGetTimestampUs(); if (error_no == 0) { @@ -250,8 +244,6 @@ void shellRunSingleCommandImp(char *command) { } printf("\n"); - - atomic_store_64(&shell.result, 0); } char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { @@ -398,7 +390,6 @@ int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres) { row = taos_fetch_row(tres); } while (row != NULL); - atomic_store_64(&shell.result, 0); taosCloseFile(&pFile); return numOfRows; @@ -766,7 +757,6 @@ void shellWriteHistory() { void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); - atomic_store_ptr(&shell.result, 0); fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6); taos_free_result(tres); } @@ -872,7 +862,6 @@ void shellGetGrantInfo() { fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\n", serverVersion, sinfo, expiretime); } - atomic_store_64(&shell.result, 0); taos_free_result(tres); }