diff --git a/example/src/tmq.c b/example/src/tmq.c index c36675fa7e..99e0c443dd 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -44,35 +44,35 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + /*pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");*/ /*if (taos_errno(pRes) != 0) {*/ /*printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));*/ /*return -1;*/ /*}*/ - taos_free_result(pRes); + /*taos_free_result(pRes);*/ - pRes = taos_query(pConn, "create table tu using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); + /*pRes = taos_query(pConn, "create table if not exists tu using st1 tags(1)");*/ + /*if (taos_errno(pRes) != 0) {*/ + /*printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes));*/ + /*return -1;*/ + /*}*/ + /*taos_free_result(pRes);*/ - pRes = taos_query(pConn, "create table tu2 using st1 tags(2)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); + /*pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");*/ + /*if (taos_errno(pRes) != 0) {*/ + /*printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));*/ + /*return -1;*/ + /*}*/ + /*taos_free_result(pRes);*/ const char* sql = "select * from st1"; pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); - /*if (taos_errno(pRes) != 0) {*/ - /*printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));*/ - /*return -1;*/ - /*}*/ - /*taos_free_result(pRes);*/ + if (taos_errno(pRes) != 0) { + printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); taos_close(pConn); return 0; } diff --git a/include/common/common.h b/include/common/common.h index e85f635aaf..fd5b6717ab 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -22,21 +22,34 @@ extern "C" { #endif #include "taosdef.h" -#include "tmsg.h" #include "tarray.h" +#include "tmsg.h" #include "tvariant.h" +// typedef struct STimeWindow { +// TSKEY skey; +// TSKEY ekey; +// } STimeWindow; -#define TMQ_REQ_TYPE_COMMIT_ONLY 0 -#define TMQ_REQ_TYPE_CONSUME_ONLY 1 -#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 +// typedef struct { +// int32_t dataLen; +// char name[TSDB_TABLE_FNAME_LEN]; +// char *data; +// } STagData; -#define TMQ_REQ_TYPE_COMMIT_ONLY 0 -#define TMQ_REQ_TYPE_CONSUME_ONLY 1 +// typedef struct SSchema { +// uint8_t type; +// char name[TSDB_COL_NAME_LEN]; +// int16_t colId; +// int16_t bytes; +// } SSchema; + +#define TMQ_REQ_TYPE_COMMIT_ONLY 0 +#define TMQ_REQ_TYPE_CONSUME_ONLY 1 #define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 typedef struct { uint32_t numOfTables; - SArray * pGroupList; + SArray *pGroupList; SHashObj *map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; @@ -67,9 +80,9 @@ typedef struct SConstantItem { // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; - SArray * pDataBlock; // SArray - SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. - SDataBlockInfo info; + SArray *pDataBlock; // SArray + SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. + SDataBlockInfo info; } SSDataBlock; typedef struct SVarColAttr { @@ -112,7 +125,7 @@ static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlo return tlen; } -static FORCE_INLINE void* tDecodeDataBlock(void* buf, SSDataBlock* pBlock) { +static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { int32_t sz; buf = taosDecodeFixedI64(buf, &pBlock->info.uid); @@ -129,7 +142,7 @@ static FORCE_INLINE void* tDecodeDataBlock(void* buf, SSDataBlock* pBlock) { buf = taosDecodeBinary(buf, (void**)&data.pData, colSz); taosArrayPush(pBlock->pDataBlock, &data); } - return buf; + return (void*)buf; } static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) { @@ -148,7 +161,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp } tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pBlock = (SSDataBlock*) taosArrayGet(pRsp->pBlockData, i); + SSDataBlock* pBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); tlen += tEncodeDataBlock(buf, pBlock); } return tlen; @@ -181,19 +194,18 @@ static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { return; } - //int32_t numOfOutput = pBlock->info.numOfCols; + // int32_t numOfOutput = pBlock->info.numOfCols; int32_t sz = taosArrayGetSize(pBlock->pDataBlock); - for(int32_t i = 0; i < sz; ++i) { + for (int32_t i = 0; i < sz; ++i) { SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); tfree(pColInfoData->pData); } taosArrayDestroy(pBlock->pDataBlock); tfree(pBlock->pBlockAgg); - //tfree(pBlock); + // tfree(pBlock); } - static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { if (pRsp->schemas) { if (pRsp->schemas->nCols) { @@ -201,11 +213,11 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { } free(pRsp->schemas); } - taosArrayDestroyEx(pRsp->pBlockData, (void(*)(void*))tDeleteSSDataBlock); - pRsp->pBlockData = NULL; - //for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { - //SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); - //tDeleteSSDataBlock(pDataBlock); + taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock); + pRsp->pBlockData = NULL; + // for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { + // SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); + // tDeleteSSDataBlock(pDataBlock); //} } @@ -229,7 +241,7 @@ typedef struct SOrder { } SOrder; typedef struct SGroupbyExpr { - SArray *columnInfo; // SArray, group by columns information + SArray* columnInfo; // SArray, group by columns information bool groupbyTag; // group by tag or column } SGroupbyExpr; @@ -238,16 +250,16 @@ typedef struct SSqlExpr { char token[TSDB_COL_NAME_LEN]; // original token SSchema resSchema; - int32_t numOfCols; - SColumn *pColumns; // data columns that are required by query - int32_t interBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - SVariant param[3]; // parameters are not more than 3 + int32_t numOfCols; + SColumn* pColumns; // data columns that are required by query + int32_t interBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + SVariant param[3]; // parameters are not more than 3 } SSqlExpr; typedef struct SExprInfo { struct SSqlExpr base; - struct tExprNode *pExpr; + struct tExprNode* pExpr; } SExprInfo; typedef struct SStateWindow { diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 46c0c98ff0..e5a7a3563e 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -23,7 +23,7 @@ extern "C" { #include "taos.h" #include "tdef.h" -typedef uint64_t tb_uid_t; +typedef int64_t tb_uid_t; #define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) #define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) @@ -38,12 +38,12 @@ typedef enum { } EQType; typedef enum { - TSDB_SUPER_TABLE = 1, // super table - TSDB_CHILD_TABLE = 2, // table created from super table - TSDB_NORMAL_TABLE = 3, // ordinary table - TSDB_STREAM_TABLE = 4, // table created from stream computing - TSDB_TEMP_TABLE = 5, // temp table created by nest query - TSDB_TABLE_MAX = 6 + TSDB_SUPER_TABLE = 1, // super table + TSDB_CHILD_TABLE = 2, // table created from super table + TSDB_NORMAL_TABLE = 3, // ordinary table + TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_TEMP_TABLE = 5, // temp table created by nest query + TSDB_TABLE_MAX = 6 } ETableType; typedef enum { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f001d6b2e4..b25f799868 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -275,6 +275,7 @@ typedef struct { int32_t tSerializeSMCreateStbReq(void** buf, SMCreateStbReq* pReq); void* tDeserializeSMCreateStbReq(void* buf, SMCreateStbReq* pReq); +void tFreeSMCreateStbReq(SMCreateStbReq* pReq); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -328,6 +329,26 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { return buf; } +static FORCE_INLINE int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp) { + if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1; + if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; + for (int i = 0; i < TSDB_MAX_REPLICA; i++) { + if (tEncodeU16(pEncoder, pEp->eps[i].port) < 0) return -1; + if (tEncodeCStr(pEncoder, pEp->eps[i].fqdn) < 0) return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp) { + if (tDecodeI8(pDecoder, &pEp->inUse) < 0) return -1; + if (tDecodeI8(pDecoder, &pEp->numOfEps) < 0) return -1; + for (int i = 0; i < TSDB_MAX_REPLICA; i++) { + if (tDecodeU16(pDecoder, &pEp->eps[i].port) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pEp->eps[i].fqdn) < 0) return -1; + } + return 0; +} + typedef struct { int32_t acctId; int64_t clusterId; @@ -349,15 +370,15 @@ typedef struct { int64_t maxStorage; // In unit of GB } SCreateAcctReq, SAlterAcctReq; -int32_t tSerializeSCreateAcctReq(void** buf, SCreateAcctReq* pReq); -void* tDeserializeSCreateAcctReq(void* buf, SCreateAcctReq* pReq); +int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); +int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); typedef struct { char user[TSDB_USER_LEN]; } SDropUserReq, SDropAcctReq; -int32_t tSerializeSDropUserReq(void** buf, SDropUserReq* pReq); -void* tDeserializeSDropUserReq(void* buf, SDropUserReq* pReq); +int32_t tSerializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq); +int32_t tDeserializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq); typedef struct { int8_t createType; @@ -366,8 +387,8 @@ typedef struct { char pass[TSDB_PASSWORD_LEN]; } SCreateUserReq; -int32_t tSerializeSCreateUserReq(void** buf, SCreateUserReq* pReq); -void* tDeserializeSCreateUserReq(void* buf, SCreateUserReq* pReq); +int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); +int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); typedef struct { int8_t alterType; @@ -377,15 +398,15 @@ typedef struct { char dbname[TSDB_DB_FNAME_LEN]; } SAlterUserReq; -int32_t tSerializeSAlterUserReq(void** buf, SAlterUserReq* pReq); -void* tDeserializeSAlterUserReq(void* buf, SAlterUserReq* pReq); +int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); +int32_t tDeserializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); typedef struct { char user[TSDB_USER_LEN]; } SGetUserAuthReq; -int32_t tSerializeSGetUserAuthReq(void** buf, SGetUserAuthReq* pReq); -void* tDeserializeSGetUserAuthReq(void* buf, SGetUserAuthReq* pReq); +int32_t tSerializeSGetUserAuthReq(void* buf, int32_t bufLen, SGetUserAuthReq* pReq); +int32_t tDeserializeSGetUserAuthReq(void* buf, int32_t bufLen, SGetUserAuthReq* pReq); typedef struct { char user[TSDB_USER_LEN]; @@ -394,8 +415,8 @@ typedef struct { SHashObj* writeDbs; } SGetUserAuthRsp; -int32_t tSerializeSGetUserAuthRsp(void** buf, SGetUserAuthRsp* pReq); -void* tDeserializeSGetUserAuthRsp(void* buf, SGetUserAuthRsp* pReq); +int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); +int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); typedef struct { int16_t colId; // column id @@ -525,28 +546,6 @@ typedef struct { int32_t code; } SQueryTableRsp; -// todo: the show handle should be replaced with id -typedef struct { - SMsgHead header; - union { - int64_t showId; - int64_t qhandle; - int64_t qId; - }; // query handle - int8_t free; -} SRetrieveTableReq; - -typedef struct { - int64_t useconds; - int8_t completed; // all results are returned to client - int8_t precision; - int8_t compressed; - int32_t compLen; - - int32_t numOfRows; - char data[]; -} SRetrieveTableRsp; - typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t numOfVgroups; @@ -570,6 +569,9 @@ typedef struct { int8_t ignoreExist; } SCreateDbReq; +int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); +int32_t tDeserializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t totalBlocks; @@ -582,28 +584,60 @@ typedef struct { int8_t cacheLastRow; } SAlterDbReq; +int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); +int32_t tDeserializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int8_t ignoreNotExists; } SDropDbReq; +int32_t tSerializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); +int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); + typedef struct { char db[TSDB_DB_FNAME_LEN]; uint64_t uid; } SDropDbRsp; +int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); +int32_t tDeserializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t vgVersion; } SUseDbReq; +int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); +int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); + typedef struct { - char db[TSDB_DB_FNAME_LEN]; -} SSyncDbReq; + char db[TSDB_DB_FNAME_LEN]; + uint64_t uid; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; + SArray* pVgroupInfos; // Array of SVgroupInfo +} SUseDbRsp; + +int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); +int32_t tDeserializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); +void tFreeSUsedbRsp(SUseDbRsp* pRsp); + +typedef struct { + SArray* pArray; // Array of SUseDbRsp +} SUseDbBatchRsp; + +int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); +int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); +void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; -} SCompactDbReq; +} SSyncDbReq, SCompactDbReq; + +int32_t tSerializeSSyncDbReq(void* buf, int32_t bufLen, SSyncDbReq* pReq); +int32_t tDeserializeSSyncDbReq(void* buf, int32_t bufLen, SSyncDbReq* pReq); typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -616,22 +650,31 @@ typedef struct { int64_t signature; int32_t commentSize; int32_t codeSize; - char pCont[]; + char pComment[TSDB_FUNC_COMMENT_LEN]; + char pCode[TSDB_FUNC_CODE_LEN]; } SCreateFuncReq; +int32_t tSerializeSCreateFuncReq(void* buf, int32_t bufLen, SCreateFuncReq* pReq); +int32_t tDeserializeSCreateFuncReq(void* buf, int32_t bufLen, SCreateFuncReq* pReq); + typedef struct { char name[TSDB_FUNC_NAME_LEN]; int8_t igNotExists; } SDropFuncReq; +int32_t tSerializeSDropFuncReq(void* buf, int32_t bufLen, SDropFuncReq* pReq); +int32_t tDeserializeSDropFuncReq(void* buf, int32_t bufLen, SDropFuncReq* pReq); + typedef struct { int32_t numOfFuncs; - char pFuncNames[]; + SArray* pFuncNames; } SRetrieveFuncReq; +int32_t tSerializeSRetrieveFuncReq(void* buf, int32_t bufLen, SRetrieveFuncReq* pReq); +int32_t tDeserializeSRetrieveFuncReq(void* buf, int32_t bufLen, SRetrieveFuncReq* pReq); + typedef struct { char name[TSDB_FUNC_NAME_LEN]; - int8_t align; int8_t funcType; int8_t scriptType; int8_t outputType; @@ -640,14 +683,18 @@ typedef struct { int64_t signature; int32_t commentSize; int32_t codeSize; - char pCont[]; + char pComment[TSDB_FUNC_COMMENT_LEN]; + char pCode[TSDB_FUNC_CODE_LEN]; } SFuncInfo; typedef struct { int32_t numOfFuncs; - char pFuncInfos[]; + SArray* pFuncInfos; } SRetrieveFuncRsp; +int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); +int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); + typedef struct { int32_t statusInterval; int64_t checkTime; // 1970-01-01 00:00:00.000 @@ -814,27 +861,22 @@ typedef struct { char* data; } STagData; -typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; - int32_t vgVersion; - int32_t vgNum; - int8_t hashMethod; - SVgroupInfo vgroupInfo[]; -} SUseDbRsp; - /* * sql: show tables like '%a_%' * payload is the query condition, e.g., '%a_%' * payloadLen is the length of payload */ typedef struct { - int8_t type; + int32_t type; char db[TSDB_DB_FNAME_LEN]; - int16_t payloadLen; - char payload[]; + int32_t payloadLen; + char* payload; } SShowReq; +int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); +int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); +void tFreeSShowReq(SShowReq* pReq); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t numOfVgroup; @@ -846,24 +888,57 @@ typedef struct { STableMetaRsp tableMeta; } SShowRsp; +// todo: the show handle should be replaced with id +typedef struct { + int64_t showId; + int8_t free; +} SRetrieveTableReq; + +int32_t tSerializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq); +int32_t tDeserializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq); + +typedef struct { + int64_t useconds; + int8_t completed; // all results are returned to client + int8_t precision; + int8_t compressed; + int32_t compLen; + + int32_t numOfRows; + char data[]; +} SRetrieveTableRsp; + typedef struct { char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port int32_t port; } SCreateDnodeReq; +int32_t tSerializeSCreateDnodeReq(void* buf, int32_t bufLen, SCreateDnodeReq* pReq); +int32_t tDeserializeSCreateDnodeReq(void* buf, int32_t bufLen, SCreateDnodeReq* pReq); + typedef struct { int32_t dnodeId; } SDropDnodeReq; +int32_t tSerializeSDropDnodeReq(void* buf, int32_t bufLen, SDropDnodeReq* pReq); +int32_t tDeserializeSDropDnodeReq(void* buf, int32_t bufLen, SDropDnodeReq* pReq); + typedef struct { int32_t dnodeId; char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; } SMCfgDnodeReq, SDCfgDnodeReq; +int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); +int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); + typedef struct { int32_t dnodeId; } SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq; +int32_t tSerializeSMCreateDropMnodeReq(void* buf, int32_t bufLen, SMCreateMnodeReq* pReq); +int32_t tDeserializeSMCreateDropMnodeReq(void* buf, int32_t bufLen, SMCreateMnodeReq* pReq); + typedef struct { int32_t dnodeId; int8_t replica; @@ -872,15 +947,11 @@ typedef struct { typedef struct { int32_t dnodeId; -} SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq; +} SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, SMCreateSnodeReq, SMDropSnodeReq, + SDCreateSnodeReq, SDDropSnodeReq, SMCreateBnodeReq, SMDropBnodeReq, SDCreateBnodeReq, SDDropBnodeReq; -typedef struct { - int32_t dnodeId; -} SMCreateSnodeReq, SMDropSnodeReq, SDCreateSnodeReq, SDDropSnodeReq; - -typedef struct { - int32_t dnodeId; -} SMCreateBnodeReq, SMDropBnodeReq, SDCreateBnodeReq, SDDropBnodeReq; +int32_t tSerializeSMCreateDropQSBNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); +int32_t tDeserializeSMCreateDropQSBNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); typedef struct { char sql[TSDB_SHOW_SQL_LEN]; @@ -1499,43 +1570,30 @@ typedef struct { SArray* rsps; // SArray } SClientHbBatchRsp; -static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { - return taosIntHash_64(key, keyLen); -} +static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { return taosIntHash_64(key, keyLen); } -int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); -void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq); - -int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp); -void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp); - - -static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { - void *pIter = taosHashIterate(info, NULL); +static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { + void* pIter = taosHashIterate(info, NULL); while (pIter != NULL) { SKv* kv = (SKv*)pIter; - tfree(kv->value); - pIter = taosHashIterate(info, pIter); } } - -static FORCE_INLINE void tFreeClientHbReq(void *pReq) { +static FORCE_INLINE void tFreeClientHbReq(void* pReq) { SClientHbReq* req = (SClientHbReq*)pReq; if (req->info) { tFreeReqKvHash(req->info); - taosHashCleanup(req->info); } } -int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); -void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); +int32_t tSerializeSClientHbBatchReq(void* buf, int32_t bufLen, const SClientHbBatchReq* pReq); +int32_t tDeserializeSClientHbBatchReq(void* buf, int32_t bufLen, SClientHbBatchReq* pReq); static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { - SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; + SClientHbBatchReq* req = (SClientHbBatchReq*)pReq; if (deep) { taosArrayDestroyEx(req->reqs, tFreeClientHbReq); } else { @@ -1544,54 +1602,52 @@ static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { free(pReq); } -static FORCE_INLINE void tFreeClientKv(void *pKv) { - SKv *kv = (SKv *)pKv; +static FORCE_INLINE void tFreeClientKv(void* pKv) { + SKv* kv = (SKv*)pKv; if (kv) { tfree(kv->value); } } -static FORCE_INLINE void tFreeClientHbRsp(void *pRsp) { +static FORCE_INLINE void tFreeClientHbRsp(void* pRsp) { SClientHbRsp* rsp = (SClientHbRsp*)pRsp; if (rsp->info) taosArrayDestroyEx(rsp->info, tFreeClientKv); } - static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) { - SClientHbBatchRsp *rsp = (SClientHbBatchRsp*)pRsp; + SClientHbBatchRsp* rsp = (SClientHbBatchRsp*)pRsp; taosArrayDestroyEx(rsp->rsps, tFreeClientHbRsp); } +int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp); +int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp); -int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp); -void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp); - -static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKv->key); - tlen += taosEncodeFixedI32(buf, pKv->valueLen); - tlen += taosEncodeBinary(buf, pKv->value, pKv->valueLen); - return tlen; +static FORCE_INLINE int32_t tEncodeSKv(SCoder* pEncoder, const SKv* pKv) { + if (tEncodeI32(pEncoder, pKv->key) < 0) return -1; + if (tEncodeI32(pEncoder, pKv->valueLen) < 0) return -1; + if (tEncodeCStrWithLen(pEncoder, (const char*)pKv->value, pKv->valueLen) < 0) return -1; + return 0; } -static FORCE_INLINE void* taosDecodeSKv(void* buf, SKv* pKv) { - buf = taosDecodeFixedI32(buf, &pKv->key); - buf = taosDecodeFixedI32(buf, &pKv->valueLen); - buf = taosDecodeBinary(buf, &pKv->value, pKv->valueLen); - return buf; +static FORCE_INLINE int32_t tDecodeSKv(SCoder* pDecoder, SKv* pKv) { + if (tDecodeI32(pDecoder, &pKv->key) < 0) return -1; + if (tDecodeI32(pDecoder, &pKv->valueLen) < 0) return -1; + pKv->value = malloc(pKv->valueLen + 1); + if (pKv->value == NULL) return -1; + if (tDecodeCStrTo(pDecoder, (char*)pKv->value) < 0) return -1; + return 0; } -static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKey->connId); - tlen += taosEncodeFixedI32(buf, pKey->hbType); - return tlen; +static FORCE_INLINE int32_t tEncodeSClientHbKey(SCoder* pEncoder, const SClientHbKey* pKey) { + if (tEncodeI32(pEncoder, pKey->connId) < 0) return -1; + if (tEncodeI32(pEncoder, pKey->hbType) < 0) return -1; + return 0; } -static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { - buf = taosDecodeFixedI32(buf, &pKey->connId); - buf = taosDecodeFixedI32(buf, &pKey->hbType); - return buf; +static FORCE_INLINE int32_t tDecodeSClientHbKey(SCoder* pDecoder, SClientHbKey* pKey) { + if (tDecodeI32(pDecoder, &pKey->connId) < 0) return -1; + if (tDecodeI32(pDecoder, &pKey->hbType) < 0) return -1; + return 0; } typedef struct SMqHbVgInfo { @@ -1684,8 +1740,7 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { typedef struct { int64_t leftForVer; int32_t vgId; - int64_t oldConsumerId; - int64_t newConsumerId; + int64_t consumerId; char topicName[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN]; char* sql; @@ -1694,55 +1749,30 @@ typedef struct { char* qmsg; } SMqSetCVgReq; -static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) { - int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pMsg->sId); - tlen += taosEncodeFixedU64(buf, pMsg->queryId); - tlen += taosEncodeFixedU64(buf, pMsg->taskId); - tlen += taosEncodeFixedU32(buf, pMsg->sqlLen); - tlen += taosEncodeFixedU32(buf, pMsg->phyLen); - //tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); - return tlen; -} - -static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { - buf = taosDecodeFixedU64(buf, &pMsg->sId); - buf = taosDecodeFixedU64(buf, &pMsg->queryId); - buf = taosDecodeFixedU64(buf, &pMsg->taskId); - buf = taosDecodeFixedU32(buf, &pMsg->sqlLen); - buf = taosDecodeFixedU32(buf, &pMsg->phyLen); - //buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); - return buf; -} - static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->leftForVer); tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); - tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); + tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->topicName); tlen += taosEncodeString(buf, pReq->cgroup); tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); tlen += taosEncodeString(buf, pReq->qmsg); - //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; } static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeFixedI64(buf, &pReq->leftForVer); buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); - buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); + buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->topicName); buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); buf = taosDecodeString(buf, &pReq->qmsg); - //buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; } @@ -1996,4 +2026,4 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } #endif -#endif /*_TD_COMMON_TAOS_MSG_H_*/ +#endif /*_TD_COMMON_TAOS_MSG_H_*/ \ No newline at end of file diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ab1298785a..b6054c02b4 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -30,7 +30,7 @@ extern "C" { #include "tmsg.h" #include "transport.h" -struct SCatalog; +typedef struct SCatalog SCatalog; enum { CTG_DBG_DB_NUM = 1, @@ -64,6 +64,7 @@ typedef struct SCatalogCfg { typedef struct SSTableMetaVersion { char dbFName[TSDB_DB_FNAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; + uint64_t dbId; uint64_t suid; int16_t sversion; int16_t tversion; @@ -84,7 +85,7 @@ int32_t catalogInit(SCatalogCfg *cfg); * @param catalogHandle (output, NO need to free it) * @return error code */ -int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle); +int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); /** * Free a cluster's all catalog info, usually it's not necessary, until the application is closing. @@ -92,9 +93,9 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle); * @param pCatalog (input, NO more usage) * @return error code */ -void catalogFreeHandle(struct SCatalog* pCatalog); +void catalogFreeHandle(SCatalog* pCatalog); -int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); +int32_t catalogGetDBVgVersion(SCatalog* pCatalog, const char* dbName, int32_t* version); /** * Get a DB's all vgroup info. @@ -106,13 +107,13 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList); +int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList); -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgroupInfo* dbInfo); +int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo); -int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId); +int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); -int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid); +int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); /** * Get a table's meta data. @@ -123,7 +124,7 @@ int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, c * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); /** * Get a super table's meta data. @@ -134,13 +135,13 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, cons * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ -int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetSTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); -int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg); +int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); /** - * Force renew a table's local cached meta data. + * Force refresh a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) @@ -148,10 +149,10 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ - int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); + int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); /** - * Force renew a table's local cached meta data and get the new one. + * Force refresh a table's local cached meta data and get the new one. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) @@ -160,7 +161,7 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ - int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); + int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); @@ -173,7 +174,7 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList); +int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList); /** * Get a table's vgroup from its name's hash value. @@ -184,7 +185,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pTransporter, * @param vgInfo (output, vgroup info) * @return error code */ -int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo); +int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo); /** @@ -196,14 +197,14 @@ int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter * @param pRsp (output, response data) * @return error code */ -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); +int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); +int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); -int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); +int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); -int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); +int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); /** diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index bf3a45fcca..c3c8b5c4ce 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "nodes.h" +#include "querynodes.h" typedef enum EFunctionType { // aggregate function diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h new file mode 100644 index 0000000000..321190d254 --- /dev/null +++ b/include/libs/nodes/nodes.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_NODES_H_ +#define _TD_NODES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tdef.h" + +#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) +#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) + +#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) + +#define FOREACH(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) + +// only be use in FOREACH +#define ERASE_NODE(list) cell = nodesListErase(list, cell); + +#define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode) + +#define FORBOTH(node1, list1, node2, list2) \ + for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ + (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ + cell1 = cell1->pNext, cell2 = cell2->pNext) + +#define FOREACH_FOR_REWRITE(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) + +typedef enum ENodeType { + // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN, VALUE, OPERATOR, FUNCTION and so on. + QUERY_NODE_COLUMN = 1, + QUERY_NODE_VALUE, + QUERY_NODE_OPERATOR, + QUERY_NODE_LOGIC_CONDITION, + QUERY_NODE_IS_NULL_CONDITION, + QUERY_NODE_FUNCTION, + QUERY_NODE_REAL_TABLE, + QUERY_NODE_TEMP_TABLE, + QUERY_NODE_JOIN_TABLE, + QUERY_NODE_GROUPING_SET, + QUERY_NODE_ORDER_BY_EXPR, + QUERY_NODE_LIMIT, + QUERY_NODE_STATE_WINDOW, + QUERY_NODE_SESSION_WINDOW, + QUERY_NODE_INTERVAL_WINDOW, + QUERY_NODE_NODE_LIST, + QUERY_NODE_FILL, + + // Only be used in parser module. + QUERY_NODE_RAW_EXPR, + + // Statement nodes are used in parser and planner module. + QUERY_NODE_SET_OPERATOR, + QUERY_NODE_SELECT_STMT, + QUERY_NODE_SHOW_STMT +} ENodeType; + +/** + * The first field of a node of any type is guaranteed to be the ENodeType. + * Hence the type of any node can be gotten by casting it to SNode. + */ +typedef struct SNode { + ENodeType type; +} SNode; + +typedef struct SListCell { + struct SListCell* pPrev; + struct SListCell* pNext; + SNode* pNode; +} SListCell; + +typedef struct SNodeList { + int16_t length; + SListCell* pHead; + SListCell* pTail; +} SNodeList; + +SNode* nodesMakeNode(ENodeType type); +void nodesDestroyNode(SNode* pNode); + +SNodeList* nodesMakeList(); +SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode); +SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); +SNode* nodesListGetNode(SNodeList* pList, int32_t index); +void nodesDestroyList(SNodeList* pList); + +typedef enum EDealRes { + DEAL_RES_CONTINUE = 1, + DEAL_RES_IGNORE_CHILD, + DEAL_RES_ERROR, +} EDealRes; + +typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); +void nodesWalkNode(SNode* pNode, FNodeWalker walker, void* pContext); +void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkNodePostOrder(SNode* pNode, FNodeWalker walker, void* pContext); +void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); + +typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext); +void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); + +bool nodesEqualNode(const SNode* a, const SNode* b); + +void nodesCloneNode(const SNode* pNode); + +int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); +int32_t nodesStringToNode(const char* pStr, SNode** pNode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_NODES_H_*/ diff --git a/include/nodes/nodesShowStmts.h b/include/libs/nodes/nodesShowStmts.h similarity index 100% rename from include/nodes/nodesShowStmts.h rename to include/libs/nodes/nodesShowStmts.h diff --git a/include/nodes/nodes.h b/include/libs/nodes/querynodes.h similarity index 65% rename from include/nodes/nodes.h rename to include/libs/nodes/querynodes.h index b3ef5ba45c..646ea63c83 100644 --- a/include/nodes/nodes.h +++ b/include/libs/nodes/querynodes.h @@ -13,88 +13,14 @@ * along with this program. If not, see . */ -#ifndef _TD_NODES_H_ -#define _TD_NODES_H_ +#ifndef _TD_QUERY_NODES_H_ +#define _TD_QUERY_NODES_H_ #ifdef __cplusplus extern "C" { #endif -#include "tdef.h" - -#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) -#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) - -#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) - -#define FOREACH(node, list) \ - for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) - -// only be use in FOREACH -#define ERASE_NODE(list) \ - if (NULL == cell->pPrev) { \ - (list)->pHead = cell->pNext; \ - } else { \ - cell->pPrev->pNext = cell->pNext; \ - cell->pNext->pPrev = cell->pPrev; \ - } \ - SListCell* tmp = cell; \ - cell = cell->pNext; \ - tfree(tmp); - -#define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode) - -#define FORBOTH(node1, list1, node2, list2) \ - for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ - (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ - cell1 = cell1->pNext, cell2 = cell2->pNext) - -typedef enum ENodeType { - QUERY_NODE_COLUMN = 1, - QUERY_NODE_VALUE, - QUERY_NODE_OPERATOR, - QUERY_NODE_LOGIC_CONDITION, - QUERY_NODE_IS_NULL_CONDITION, - QUERY_NODE_FUNCTION, - QUERY_NODE_REAL_TABLE, - QUERY_NODE_TEMP_TABLE, - QUERY_NODE_JOIN_TABLE, - QUERY_NODE_GROUPING_SET, - QUERY_NODE_ORDER_BY_EXPR, - QUERY_NODE_LIMIT, - QUERY_NODE_STATE_WINDOW, - QUERY_NODE_SESSION_WINDOW, - QUERY_NODE_INTERVAL_WINDOW, - QUERY_NODE_NODE_LIST, - QUERY_NODE_FILL, - - // only for parser - QUERY_NODE_RAW_EXPR, - - QUERY_NODE_SET_OPERATOR, - QUERY_NODE_SELECT_STMT, - QUERY_NODE_SHOW_STMT -} ENodeType; - -/** - * The first field of a node of any type is guaranteed to be the ENodeType. - * Hence the type of any node can be gotten by casting it to SNode. - */ -typedef struct SNode { - ENodeType type; -} SNode; - -typedef struct SListCell { - struct SListCell* pPrev; - struct SListCell* pNext; - SNode* pNode; -} SListCell; - -typedef struct SNodeList { - int16_t length; - SListCell* pHead; - SListCell* pTail; -} SNodeList; +#include "nodes.h" typedef struct SRawExprNode { ENodeType nodeType; @@ -338,29 +264,7 @@ typedef struct SSetOperator { SNode* pLimit; } SSetOperator; -SNode* nodesMakeNode(ENodeType type); -void nodesDestroyNode(SNode* pNode); - -SNodeList* nodesMakeList(); -SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode); -SNode* nodesListGetNode(SNodeList* pList, int32_t index); -void nodesDestroyList(SNodeList* pList); - -typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); - -void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); -void nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext); -void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext); -void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext); - -bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext); - -bool nodesEqualNode(const SNode* a, const SNode* b); - -void nodesCloneNode(const SNode* pNode); - -int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); -int32_t nodesStringToNode(const char* pStr, SNode** pNode); +bool nodesIsExprNode(const SNode* pNode); bool nodesIsArithmeticOp(const SOperatorNode* pOp); bool nodesIsComparisonOp(const SOperatorNode* pOp); @@ -373,4 +277,4 @@ bool nodesIsTimelineQuery(const SNode* pQuery); } #endif -#endif /*_TD_NODES_H_*/ +#endif /*_TD_QUERY_NODES_H_*/ \ No newline at end of file diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 549f36a898..1b70c78af9 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -80,16 +80,16 @@ typedef struct STableMeta { SSchema schema[]; } STableMeta; -typedef struct SDBVgroupInfo { +typedef struct SDBVgInfo { int32_t vgVersion; int8_t hashMethod; SHashObj *vgHash; //key:vgId, value:SVgroupInfo -} SDBVgroupInfo; +} SDBVgInfo; typedef struct SUseDbOutput { char db[TSDB_DB_FNAME_LEN]; uint64_t dbId; - SDBVgroupInfo *dbVgroup; + SDBVgInfo *dbVgroup; } SUseDbOutput; enum { diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 283604508f..00ba1120e7 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 TAOS Data, Inc. + * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 @@ -23,7 +23,7 @@ extern "C" { #include #include "taosdef.h" -typedef int32_t SyncNodeId; +typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; typedef int64_t SyncIndex; typedef uint64_t SyncTerm; @@ -46,109 +46,113 @@ typedef struct { } SNodeInfo; typedef struct { - int32_t selfIndex; - int32_t replica; + int32_t replicaNum; SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; -} SSyncCluster; +} SSyncCfg; typedef struct { - int32_t selfIndex; - int32_t replica; - SNodeInfo node[TSDB_MAX_REPLICA]; + int32_t replicaNum; + SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; ESyncState role[TSDB_MAX_REPLICA]; } SNodesRole; +// abstract definition of snapshot +typedef struct SSnapshot { + void* data; + SyncIndex lastApplyIndex; +} SSnapshot; + typedef struct SSyncFSM { - void* pData; + void* data; - // apply committed log, bufs will be free by sync module - int32_t (*applyLog)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf, void* pData); + // when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result + // user can do something according to the code and isWeak. for example, write data into tsdb + void (*FpCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); - // cluster commit callback - int32_t (*onClusterChanged)(struct SSyncFSM* fsm, const SSyncCluster* cluster, void* pData); + // when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result + // user can do something according to the code and isWeak. for example, write data into tsdb + void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); - // fsm return snapshot in ppBuf, bufs will be free by sync module - // TODO: getSnapshot SHOULD be async? - int32_t (*getSnapshot)(struct SSyncFSM* fsm, SSyncBuffer** ppBuf, int32_t* objId, bool* isLast); + // when log entry is updated by a new one, FpRollBackCb is called + // user can do something to roll back. for example, delete data from tsdb, or just ignore it + void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); - // fsm apply snapshot with pBuf data - int32_t (*applySnapshot)(struct SSyncFSM* fsm, SSyncBuffer* pBuf, int32_t objId, bool isLast); + // user should implement this function, use "data" to take snapshot into "snapshot" + int32_t (*FpTakeSnapshot)(SSnapshot* snapshot); - // call when restore snapshot and log done - int32_t (*onRestoreDone)(struct SSyncFSM* fsm); - - void (*onRollback)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf); - - void (*onRoleChanged)(struct SSyncFSM* fsm, const SNodesRole* pRole); + // user should implement this function, restore "data" from "snapshot" + int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot); } SSyncFSM; +// abstract definition of log store in raft +// SWal implements it typedef struct SSyncLogStore { - void* pData; + void* data; - // write log with given index - int32_t (*logWrite)(struct SSyncLogStore* logStore, SyncIndex index, SSyncBuffer* pBuf); + // append one log entry + int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); - /** - * read log from given index(included) with limit, return the actual num in nBuf, - * pBuf will be free in sync module - **/ - int32_t (*logRead)(struct SSyncLogStore* logStore, SyncIndex index, int limit, - SSyncBuffer* pBuf, int* nBuf); + // get one log entry, user need to free pBuf->data + int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf); - // mark log with given index has been commtted - int32_t (*logCommit)(struct SSyncLogStore* logStore, SyncIndex index); + // update log store commit index with "index" + int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index); - // prune log before given index(not included) - int32_t (*logPrune)(struct SSyncLogStore* logStore, SyncIndex index); + // truncate log with index, entries after the given index (>index) will be deleted + int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex index); - // rollback log after given index(included) - int32_t (*logRollback)(struct SSyncLogStore* logStore, SyncIndex index); + // return commit index of log + SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); + + // return index of last entry + SyncIndex (*getLastIndex)(struct SSyncLogStore* pLogStore); + + // return term of last entry + SyncTerm (*getLastTerm)(struct SSyncLogStore* pLogStore); - // return last index of log - SyncIndex (*logLastIndex)(struct SSyncLogStore* logStore); } SSyncLogStore; -typedef struct SStateManager { - void* pData; +// raft need to persist two variables in storage: currentTerm, voteFor +typedef struct SStateMgr { + void* data; - // save serialized server state data, buffer will be free by Sync - int32_t (*saveServerState)(struct SStateManager* stateMng, const char* buffer, int n); + int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm); + int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm); - // read serialized server state data, buffer will be free by Sync - int32_t (*readServerState)(struct SStateManager* stateMng, char** ppBuffer, int* n); + int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor); + int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor); - // save serialized cluster state data, buffer will be free by Sync - void (*saveClusterState)(struct SStateManager* stateMng, const char* buffer, int n); + int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); + int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); - // read serialized cluster state data, buffer will be free by Sync - int32_t (*readClusterState)(struct SStateManager* stateMng, char** ppBuffer, int* n); -} SStateManager; +} SStateMgr; typedef struct { SyncGroupId vgId; - SyncIndex appliedIndex; - SSyncCluster syncCfg; - SSyncFSM fsm; + SSyncCfg syncCfg; SSyncLogStore logStore; - SStateManager stateManager; + SStateMgr stateManager; + SSyncFSM syncFsm; + } SSyncInfo; -struct SSyncNode; -typedef struct SSyncNode SSyncNode; +// will be defined in syncInt.h, here just for complie +typedef struct SSyncNode { +} SSyncNode; int32_t syncInit(); void syncCleanUp(); -SSyncNode* syncStart(const SSyncInfo*); -void syncReconfig(const SSyncNode*, const SSyncCluster*); -void syncStop(const SSyncNode*); +int64_t syncStart(const SSyncInfo*); +void syncStop(int64_t rid); +int32_t syncReconfig(int64_t rid, const SSyncCfg*); -int32_t syncPropose(SSyncNode* syncNode, const SSyncBuffer* pBuf, void* pData, bool isWeak); +// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); +int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); -int32_t syncAddNode(SSyncNode syncNode, const SNodeInfo *pNode); - -int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode); +ESyncState syncGetMyRole(int64_t rid); +void syncGetNodesRole(int64_t rid, SNodesRole*); extern int32_t sDebugFlag; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2451b2b904..0a822927ad 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -247,7 +247,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) #define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) -// mnode-topic +// mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) #define TSDB_CODE_MND_TOPIC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E1) #define TSDB_CODE_MND_TOO_MANY_TOPICS TAOS_DEF_ERROR_CODE(0, 0x03E2) @@ -256,7 +256,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5) #define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6) #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) -#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E7) +#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) +#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) +#define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) @@ -454,6 +456,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2607) //Inconsistent datatypes #define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) //There mustn't be aggregation #define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) //ORDER BY item must be the number of a SELECT-list expression +#define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) //Not a GROUP BY expression +#define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260B) //Not SELECTed expression +#define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) //Not a single-group group function #ifdef __cplusplus } diff --git a/include/util/tcoding.h b/include/util/tcoding.h index c105ce1ab9..fed9d12cee 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -37,7 +37,7 @@ static FORCE_INLINE int taosEncodeFixedU8(void **buf, uint8_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedU8(void *buf, uint8_t *value) { +static FORCE_INLINE void *taosDecodeFixedU8(const void *buf, uint8_t *value) { *value = ((uint8_t *)buf)[0]; return POINTER_SHIFT(buf, sizeof(*value)); } @@ -51,7 +51,7 @@ static FORCE_INLINE int taosEncodeFixedI8(void **buf, int8_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedI8(void *buf, int8_t *value) { +static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) { *value = ((int8_t *)buf)[0]; return POINTER_SHIFT(buf, sizeof(*value)); } @@ -71,7 +71,7 @@ static FORCE_INLINE int taosEncodeFixedU16(void **buf, uint16_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedU16(void *buf, uint16_t *value) { +static FORCE_INLINE void *taosDecodeFixedU16(const void *buf, uint16_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { @@ -87,9 +87,9 @@ static FORCE_INLINE int taosEncodeFixedI16(void **buf, int16_t value) { return taosEncodeFixedU16(buf, ZIGZAGE(int16_t, value)); } -static FORCE_INLINE void *taosDecodeFixedI16(void *buf, int16_t *value) { +static FORCE_INLINE void *taosDecodeFixedI16(const void *buf, int16_t *value) { uint16_t tvalue = 0; - void * ret = taosDecodeFixedU16(buf, &tvalue); + void *ret = taosDecodeFixedU16(buf, &tvalue); *value = ZIGZAGD(int16_t, tvalue); return ret; } @@ -111,7 +111,7 @@ static FORCE_INLINE int taosEncodeFixedU32(void **buf, uint32_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedU32(void *buf, uint32_t *value) { +static FORCE_INLINE void *taosDecodeFixedU32(const void *buf, uint32_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { @@ -129,9 +129,9 @@ static FORCE_INLINE int taosEncodeFixedI32(void **buf, int32_t value) { return taosEncodeFixedU32(buf, ZIGZAGE(int32_t, value)); } -static FORCE_INLINE void *taosDecodeFixedI32(void *buf, int32_t *value) { +static FORCE_INLINE void *taosDecodeFixedI32(const void *buf, int32_t *value) { uint32_t tvalue = 0; - void * ret = taosDecodeFixedU32(buf, &tvalue); + void *ret = taosDecodeFixedU32(buf, &tvalue); *value = ZIGZAGD(int32_t, tvalue); return ret; } @@ -158,7 +158,7 @@ static FORCE_INLINE int taosEncodeFixedU64(void **buf, uint64_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedU64(void *buf, uint64_t *value) { +static FORCE_INLINE void *taosDecodeFixedU64(const void *buf, uint64_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { @@ -180,9 +180,9 @@ static FORCE_INLINE int taosEncodeFixedI64(void **buf, int64_t value) { return taosEncodeFixedU64(buf, ZIGZAGE(int64_t, value)); } -static FORCE_INLINE void *taosDecodeFixedI64(void *buf, int64_t *value) { +static FORCE_INLINE void *taosDecodeFixedI64(const void *buf, int64_t *value) { uint64_t tvalue = 0; - void * ret = taosDecodeFixedU64(buf, &tvalue); + void *ret = taosDecodeFixedU64(buf, &tvalue); *value = ZIGZAGD(int64_t, tvalue); return ret; } @@ -205,7 +205,7 @@ static FORCE_INLINE int taosEncodeVariantU16(void **buf, uint16_t value) { return i + 1; } -static FORCE_INLINE void *taosDecodeVariantU16(void *buf, uint16_t *value) { +static FORCE_INLINE void *taosDecodeVariantU16(const void *buf, uint16_t *value) { int i = 0; uint16_t tval = 0; *value = 0; @@ -228,9 +228,9 @@ static FORCE_INLINE int taosEncodeVariantI16(void **buf, int16_t value) { return taosEncodeVariantU16(buf, ZIGZAGE(int16_t, value)); } -static FORCE_INLINE void *taosDecodeVariantI16(void *buf, int16_t *value) { +static FORCE_INLINE void *taosDecodeVariantI16(const void *buf, int16_t *value) { uint16_t tvalue = 0; - void * ret = taosDecodeVariantU16(buf, &tvalue); + void *ret = taosDecodeVariantU16(buf, &tvalue); *value = ZIGZAGD(int16_t, tvalue); return ret; } @@ -253,7 +253,7 @@ static FORCE_INLINE int taosEncodeVariantU32(void **buf, uint32_t value) { return i + 1; } -static FORCE_INLINE void *taosDecodeVariantU32(void *buf, uint32_t *value) { +static FORCE_INLINE void *taosDecodeVariantU32(const void *buf, uint32_t *value) { int i = 0; uint32_t tval = 0; *value = 0; @@ -276,9 +276,9 @@ static FORCE_INLINE int taosEncodeVariantI32(void **buf, int32_t value) { return taosEncodeVariantU32(buf, ZIGZAGE(int32_t, value)); } -static FORCE_INLINE void *taosDecodeVariantI32(void *buf, int32_t *value) { +static FORCE_INLINE void *taosDecodeVariantI32(const void *buf, int32_t *value) { uint32_t tvalue = 0; - void * ret = taosDecodeVariantU32(buf, &tvalue); + void *ret = taosDecodeVariantU32(buf, &tvalue); *value = ZIGZAGD(int32_t, tvalue); return ret; } @@ -301,7 +301,7 @@ static FORCE_INLINE int taosEncodeVariantU64(void **buf, uint64_t value) { return i + 1; } -static FORCE_INLINE void *taosDecodeVariantU64(void *buf, uint64_t *value) { +static FORCE_INLINE void *taosDecodeVariantU64(const void *buf, uint64_t *value) { int i = 0; uint64_t tval = 0; *value = 0; @@ -324,9 +324,9 @@ static FORCE_INLINE int taosEncodeVariantI64(void **buf, int64_t value) { return taosEncodeVariantU64(buf, ZIGZAGE(int64_t, value)); } -static FORCE_INLINE void *taosDecodeVariantI64(void *buf, int64_t *value) { +static FORCE_INLINE void *taosDecodeVariantI64(const void *buf, int64_t *value) { uint64_t tvalue = 0; - void * ret = taosDecodeVariantU64(buf, &tvalue); + void *ret = taosDecodeVariantU64(buf, &tvalue); *value = ZIGZAGD(int64_t, tvalue); return ret; } @@ -346,7 +346,7 @@ static FORCE_INLINE int taosEncodeString(void **buf, const char *value) { return tlen; } -static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { +static FORCE_INLINE void *taosDecodeString(const void *buf, char **value) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); @@ -360,7 +360,7 @@ static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { return POINTER_SHIFT(buf, size); } -static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) { +static FORCE_INLINE void *taosDecodeStringTo(const void *buf, char *value) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); @@ -373,7 +373,7 @@ static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) { // ---- binary static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t valueLen) { - int tlen = 0; + int tlen = 0; if (buf != NULL) { memcpy(*buf, value, valueLen); @@ -384,8 +384,7 @@ static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t return tlen; } -static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int32_t valueLen) { - +static FORCE_INLINE void *taosDecodeBinary(const void *buf, void **value, int32_t valueLen) { *value = malloc((size_t)valueLen); if (*value == NULL) return NULL; memcpy(*value, buf, (size_t)valueLen); @@ -393,8 +392,7 @@ static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int32_t valu return POINTER_SHIFT(buf, valueLen); } -static FORCE_INLINE void *taosDecodeBinaryTo(void *buf, void *value, int32_t valueLen) { - +static FORCE_INLINE void *taosDecodeBinaryTo(const void *buf, void *value, int32_t valueLen) { memcpy(value, buf, (size_t)valueLen); return POINTER_SHIFT(buf, valueLen); } diff --git a/include/util/tdef.h b/include/util/tdef.h index 88b526f102..747020a4f9 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -219,6 +219,7 @@ do { \ #define TSDB_ERROR_MSG_LEN 1024 #define TSDB_DNODE_CONFIG_LEN 128 +#define TSDB_DNODE_VALUE_LEN 256 #define TSDB_MQTT_HOSTNAME_LEN 64 #define TSDB_MQTT_PORT_LEN 8 diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index fbf045b99c..2833b329a7 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -3,5 +3,4 @@ add_subdirectory(util) add_subdirectory(common) add_subdirectory(libs) add_subdirectory(client) -add_subdirectory(dnode) -add_subdirectory(nodes) \ No newline at end of file +add_subdirectory(dnode) \ No newline at end of file diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 3e1af765b0..6737467d17 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -28,22 +28,23 @@ static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) } static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { - int32_t msgLen = 0; int32_t code = 0; - - while (msgLen < valueLen) { - SUseDbRsp *rsp = (SUseDbRsp *)((char *)value + msgLen); - rsp->vgVersion = ntohl(rsp->vgVersion); - rsp->vgNum = ntohl(rsp->vgNum); - rsp->uid = be64toh(rsp->uid); + SUseDbBatchRsp batchUseRsp = {0}; + if (tDeserializeSUseDbBatchRsp(value, valueLen, &batchUseRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%"PRIx64, rsp->db, rsp->vgVersion, rsp->uid); if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); } else { - SDBVgroupInfo vgInfo = {0}; + SDBVgInfo vgInfo = {0}; vgInfo.vgVersion = rsp->vgVersion; vgInfo.hashMethod = rsp->hashMethod; vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -52,33 +53,21 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog return TSDB_CODE_TSC_OUT_OF_MEMORY; } - for (int32_t i = 0; i < rsp->vgNum; ++i) { - rsp->vgroupInfo[i].vgId = ntohl(rsp->vgroupInfo[i].vgId); - rsp->vgroupInfo[i].hashBegin = ntohl(rsp->vgroupInfo[i].hashBegin); - rsp->vgroupInfo[i].hashEnd = ntohl(rsp->vgroupInfo[i].hashEnd); - - for (int32_t n = 0; n < rsp->vgroupInfo[i].epset.numOfEps; ++n) { - rsp->vgroupInfo[i].epset.eps[n].port = ntohs(rsp->vgroupInfo[i].epset.eps[n].port); - } - - if (0 != taosHashPut(vgInfo.vgHash, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { + for (int32_t j = 0; j < rsp->vgNum; ++j) { + SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); + if (taosHashPut(vgInfo.vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { tscError("hash push failed, errno:%d", errno); taosHashCleanup(vgInfo.vgHash); return TSDB_CODE_TSC_OUT_OF_MEMORY; } } - code = catalogUpdateDBVgroup(pCatalog, rsp->db, rsp->uid, &vgInfo); - if (code) { - taosHashCleanup(vgInfo.vgHash); - } + catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, &vgInfo); } if (code) { return code; } - - msgLen += sizeof(SUseDbRsp) + rsp->vgNum * sizeof(SVgroupInfo); } return TSDB_CODE_SUCCESS; @@ -94,13 +83,14 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo rsp->numOfColumns = ntohl(rsp->numOfColumns); rsp->suid = be64toh(rsp->suid); + rsp->dbId = be64toh(rsp->dbId); if (rsp->numOfColumns < 0) { schemaNum = 0; tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); - catalogRemoveSTableMeta(pCatalog, rsp->dbFName, rsp->stbName, rsp->suid); + catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid); } else { tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); @@ -201,9 +191,10 @@ static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code tfree(param); return -1; } + char *key = (char *)param; SClientHbBatchRsp pRsp = {0}; - tDeserializeSClientHbBatchRsp(pMsg->pData, &pRsp); + tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); int32_t rspNum = taosArrayGetSize(pRsp.rsps); @@ -416,7 +407,7 @@ static void* hbThreadFunc(void* param) { if (pReq == NULL) { continue; } - int tlen = tSerializeSClientHbBatchReq(NULL, pReq); + int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); void *buf = malloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -424,8 +415,7 @@ static void* hbThreadFunc(void* param) { hbClearReqInfo(pAppHbMgr); break; } - void *abuf = buf; - tSerializeSClientHbBatchReq(&abuf, pReq); + tSerializeSClientHbBatchReq(buf, tlen, pReq); SMsgSendInfo *pInfo = malloc(sizeof(SMsgSendInfo)); if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 8ab1880069..16b017b0c5 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -97,14 +97,15 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { if (pRequest->type == TDMT_MND_SHOW_RETRIEVE || pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { - SRetrieveTableReq* pRetrieveMsg = calloc(1, sizeof(SRetrieveTableReq)); - if (pRetrieveMsg == NULL) { - return NULL; - } + SRetrieveTableReq retrieveReq = {0}; + retrieveReq.showId = pRequest->body.showInfo.execId; - pRetrieveMsg->showId = htobe64(pRequest->body.showInfo.execId); - pMsgSendInfo->msgInfo.pData = pRetrieveMsg; - pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableReq); + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq); + void* pReq = malloc(contLen); + tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq); + + pMsgSendInfo->msgInfo.pData = pReq; + pMsgSendInfo->msgInfo.len = contLen; } else { SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); if (pFetchMsg == NULL) { @@ -264,9 +265,13 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - SUseDbRsp* pUseDbRsp = (SUseDbRsp*) pMsg->pData; + SUseDbRsp usedbRsp = {0}; + tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); + SName name = {0}; - tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT|T_NAME_DB); + tNameFromString(&name, usedbRsp.db, T_NAME_ACCT|T_NAME_DB); + + tFreeSUsedbRsp(&usedbRsp); char db[TSDB_DB_NAME_LEN] = {0}; tNameGetDbName(&name, db); @@ -300,14 +305,12 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - SDropDbRsp *rsp = (SDropDbRsp *)pMsg->pData; - - struct SCatalog *pCatalog = NULL; - rsp->uid = be64toh(rsp->uid); + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp); + struct SCatalog* pCatalog = NULL; catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - - catalogRemoveDB(pCatalog, rsp->db, rsp->uid); + catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); tsem_post(&pRequest->body.rspSem); return code; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index f465f09ee1..ebf9cfc1a7 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -113,10 +113,10 @@ typedef struct SMqConsumeCbParam { } SMqConsumeCbParam; typedef struct SMqCommitCbParam { - tmq_t* tmq; - SMqClientVg* pVg; - int32_t async; - tsem_t rspSem; + tmq_t* tmq; + SMqClientVg* pVg; + int32_t async; + tsem_t rspSem; } SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { @@ -163,8 +163,8 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { } int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqCommitCbParam* pParam = (SMqCommitCbParam*) param; - tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; + SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; + tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; if (pParam->tmq->commit_cb) { pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); } @@ -221,13 +221,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tNameExtractFullName(&name, topicFname); tscDebug("subscribe topic: %s", topicFname); SMqClientTopic topic = { - .nextVgIdx = 0, - .sql = NULL, - .sqlLen = 0, - .topicId = 0, - .topicName = topicFname, - .vgs = NULL - }; + .nextVgIdx = 0, .sql = NULL, .sqlLen = 0, .topicId = 0, .topicName = topicFname, .vgs = NULL}; topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); taosArrayPush(tmq->clientTopics, &topic); /*SMqClientTopic topic = {*/ @@ -461,10 +455,10 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { void tmqShowMsg(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; - static bool noPrintSchema; - char pBuf[128]; + static bool noPrintSchema; + char pBuf[128]; SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; - int32_t colNum = pRsp->schemas->nCols; + int32_t colNum = pRsp->schemas->nCols; if (!noPrintSchema) { printf("|"); for (int32_t i = 0; i < colNum; i++) { @@ -506,7 +500,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; SMqClientVg* pVg = pParam->pVg; if (code != 0) { - /*printf("msg discard\n");*/ + printf("msg discard\n"); tsem_post(&pParam->rspSem); return 0; } @@ -517,7 +511,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { return -1; } tDecodeSMqConsumeRsp(pMsg->pData, pRsp); - /*printf("rsp %ld %ld %d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ + /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->numOfTopics == 0) { /*printf("no data\n");*/ free(pRsp); @@ -584,7 +578,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { +int32_t tmqAskEp(tmq_t* tmq, bool wait) { int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* buf = malloc(tlen); if (buf == NULL) { @@ -603,7 +597,7 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; - SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); + SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam)); if (pParam == NULL) { tscError("failed to malloc subscribe param"); goto END; @@ -653,7 +647,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* tmq_message = NULL; int64_t status = atomic_load_64(&tmq->status); - tmqAsyncAskEp(tmq, status == 0); + tmqAskEp(tmq, status == 0); if (blocking_time < 0) blocking_time = 1; if (blocking_time > 1000) blocking_time = 1000; @@ -671,48 +665,58 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { } tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg); - if (pReq == NULL) { - usleep(blocking_time * 1000); - return NULL; + int32_t beginVgIdx = pTopic->nextVgIdx; + while (1) { + pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); + /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg); + if (pReq == NULL) { + ASSERT(false); + usleep(blocking_time * 1000); + return NULL; + } + + SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); + if (param == NULL) { + ASSERT(false); + usleep(blocking_time * 1000); + return NULL; + } + param->tmq = tmq; + param->retMsg = &tmq_message; + param->pVg = pVg; + tsem_init(¶m->rspSem, 0, 0); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = param; + sendInfo->fp = tmqPollCb; + + /*printf("req offset: %ld\n", pReq->offset);*/ + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + tmq->pollCnt++; + + tsem_wait(¶m->rspSem); + tsem_destroy(¶m->rspSem); + free(param); + + if (tmq_message == NULL) { + if (beginVgIdx == pTopic->nextVgIdx) { + usleep(blocking_time * 1000); + } else { + continue; + } + } + + return tmq_message; } - SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); - if (param == NULL) { - usleep(blocking_time * 1000); - return NULL; - } - param->tmq = tmq; - param->retMsg = &tmq_message; - param->pVg = pVg; - tsem_init(¶m->rspSem, 0, 0); - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = param; - sendInfo->fp = tmqPollCb; - - /*printf("req offset: %ld\n", pReq->offset);*/ - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - tmq->pollCnt++; - - tsem_wait(¶m->rspSem); - tsem_destroy(¶m->rspSem); - free(param); - - if (tmq_message == NULL) { - usleep(blocking_time * 1000); - } - - return tmq_message; - /*tsem_wait(&pRequest->body.rspSem);*/ /*if (body != NULL) {*/ @@ -727,21 +731,20 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { } tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { - if (tmq_topic_vgroup_list != NULL) { - //TODO + // TODO } - //TODO: change semaphore to gate + // TODO: change semaphore to gate for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); - + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; - SMqCommitCbParam *pParam = malloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); if (pParam == NULL) { continue; } @@ -772,9 +775,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) { free(tmq_message); } -tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { - return TMQ_RESP_ERR__SUCCESS; -} +tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; } const char* tmq_err2str(tmq_resp_err_t err) { if (err == TMQ_RESP_ERR__SUCCESS) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 33add3eb6b..e151ec9ae2 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -85,118 +85,156 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { } } -int32_t tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { - int32_t tlen = 0; - tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); +static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) { + if (tEncodeSClientHbKey(pEncoder, &pReq->connKey) < 0) return -1; int32_t kvNum = taosHashGetSize(pReq->info); - tlen += taosEncodeFixedI32(buf, kvNum); - SKv *kv; + if (tEncodeI32(pEncoder, kvNum) < 0) return -1; void *pIter = taosHashIterate(pReq->info, NULL); while (pIter != NULL) { - kv = pIter; - tlen += taosEncodeSKv(buf, kv); - + SKv *kv = pIter; + if (tEncodeSKv(pEncoder, kv) < 0) return -1; pIter = taosHashIterate(pReq->info, pIter); } - return tlen; + + return 0; } -void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) { - buf = taosDecodeSClientHbKey(buf, &pReq->connKey); +static int32_t tDeserializeSClientHbReq(SCoder *pDecoder, SClientHbReq *pReq) { + if (tDecodeSClientHbKey(pDecoder, &pReq->connKey) < 0) return -1; - // TODO: error handling - int32_t kvNum; - buf = taosDecodeFixedI32(buf, &kvNum); + int32_t kvNum = 0; + if (tDecodeI32(pDecoder, &kvNum) < 0) return -1; if (pReq->info == NULL) { pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } + if (pReq->info == NULL) return -1; for (int32_t i = 0; i < kvNum; i++) { - SKv kv; - buf = taosDecodeSKv(buf, &kv); + SKv kv = {0}; + if (tDecodeSKv(pDecoder, &kv) < 0) return -1; taosHashPut(pReq->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); } - return buf; + return 0; } -int32_t tSerializeSClientHbRsp(void **buf, const SClientHbRsp *pRsp) { - int32_t tlen = 0; +static int32_t tSerializeSClientHbRsp(SCoder *pEncoder, const SClientHbRsp *pRsp) { + if (tEncodeSClientHbKey(pEncoder, &pRsp->connKey) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->status) < 0) return -1; + int32_t kvNum = taosArrayGetSize(pRsp->info); - tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey); - tlen += taosEncodeFixedI32(buf, pRsp->status); - tlen += taosEncodeFixedI32(buf, kvNum); + if (tEncodeI32(pEncoder, kvNum) < 0) return -1; for (int32_t i = 0; i < kvNum; i++) { - SKv *kv = (SKv *)taosArrayGet(pRsp->info, i); - tlen += taosEncodeSKv(buf, kv); + SKv *kv = taosArrayGet(pRsp->info, i); + if (tEncodeSKv(pEncoder, kv) < 0) return -1; } - return tlen; + + return 0; } -void *tDeserializeSClientHbRsp(void *buf, SClientHbRsp *pRsp) { +static int32_t tDeserializeSClientHbRsp(SCoder *pDecoder, SClientHbRsp *pRsp) { + if (tDecodeSClientHbKey(pDecoder, &pRsp->connKey) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->status) < 0) return -1; + int32_t kvNum = 0; - buf = taosDecodeSClientHbKey(buf, &pRsp->connKey); - buf = taosDecodeFixedI32(buf, &pRsp->status); - buf = taosDecodeFixedI32(buf, &kvNum); + if (tDecodeI32(pDecoder, &kvNum) < 0) return -1; pRsp->info = taosArrayInit(kvNum, sizeof(SKv)); + if (pRsp->info == NULL) return -1; for (int32_t i = 0; i < kvNum; i++) { SKv kv = {0}; - buf = taosDecodeSKv(buf, &kv); + tDecodeSKv(pDecoder, &kv); taosArrayPush(pRsp->info, &kv); } - return buf; + return 0; } -int32_t tSerializeSClientHbBatchReq(void **buf, const SClientHbBatchReq *pBatchReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pBatchReq->reqId); +int32_t tSerializeSClientHbBatchReq(void *buf, int32_t bufLen, const SClientHbBatchReq *pBatchReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pBatchReq->reqId) < 0) return -1; + int32_t reqNum = taosArrayGetSize(pBatchReq->reqs); - tlen += taosEncodeFixedI32(buf, reqNum); + if (tEncodeI32(&encoder, reqNum) < 0) return -1; for (int32_t i = 0; i < reqNum; i++) { SClientHbReq *pReq = taosArrayGet(pBatchReq->reqs, i); - tlen += tSerializeSClientHbReq(buf, pReq); + if (tSerializeSClientHbReq(&encoder, pReq) < 0) return -1; } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSClientHbBatchReq(void *buf, SClientHbBatchReq *pBatchReq) { - buf = taosDecodeFixedI64(buf, &pBatchReq->reqId); - if (pBatchReq->reqs == NULL) { - pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq)); - } +int32_t tDeserializeSClientHbBatchReq(void *buf, int32_t bufLen, SClientHbBatchReq *pBatchReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - int32_t reqNum; - buf = taosDecodeFixedI32(buf, &reqNum); + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pBatchReq->reqId) < 0) return -1; + + int32_t reqNum = 0; + if (tDecodeI32(&decoder, &reqNum) < 0) return -1; + if (pBatchReq->reqs == NULL) { + pBatchReq->reqs = taosArrayInit(reqNum, sizeof(SClientHbReq)); + } for (int32_t i = 0; i < reqNum; i++) { SClientHbReq req = {0}; - buf = tDeserializeSClientHbReq(buf, &req); + tDeserializeSClientHbReq(&decoder, &req); taosArrayPush(pBatchReq->reqs, &req); } - return buf; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSClientHbBatchRsp(void **buf, const SClientHbBatchRsp *pBatchRsp) { - int32_t tlen = 0; - int32_t sz = taosArrayGetSize(pBatchRsp->rsps); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { +int32_t tSerializeSClientHbBatchRsp(void *buf, int32_t bufLen, const SClientHbBatchRsp *pBatchRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pBatchRsp->reqId) < 0) return -1; + if (tEncodeI64(&encoder, pBatchRsp->rspId) < 0) return -1; + + int32_t rspNum = taosArrayGetSize(pBatchRsp->rsps); + if (tEncodeI32(&encoder, rspNum) < 0) return -1; + for (int32_t i = 0; i < rspNum; i++) { SClientHbRsp *pRsp = taosArrayGet(pBatchRsp->rsps, i); - tlen += tSerializeSClientHbRsp(buf, pRsp); + if (tSerializeSClientHbRsp(&encoder, pRsp) < 0) return -1; } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSClientHbBatchRsp(void *buf, SClientHbBatchRsp *pBatchRsp) { - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pBatchRsp->rsps = taosArrayInit(sz, sizeof(SClientHbRsp)); - for (int32_t i = 0; i < sz; i++) { +int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchRsp *pBatchRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pBatchRsp->reqId) < 0) return -1; + if (tDecodeI64(&decoder, &pBatchRsp->rspId) < 0) return -1; + + int32_t rspNum = 0; + if (tDecodeI32(&decoder, &rspNum) < 0) return -1; + if (pBatchRsp->rsps == NULL) { + pBatchRsp->rsps = taosArrayInit(rspNum, sizeof(SClientHbReq)); + } + for (int32_t i = 0; i < rspNum; i++) { SClientHbRsp rsp = {0}; - buf = tDeserializeSClientHbRsp(buf, &rsp); + tDeserializeSClientHbRsp(&decoder, &rsp); taosArrayPush(pBatchRsp->rsps, &rsp); } - return buf; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { @@ -402,6 +440,11 @@ void *tDeserializeSMCreateStbReq(void *buf, SMCreateStbReq *pReq) { return buf; } +void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { + taosArrayDestroy(pReq->pColumns); + taosArrayDestroy(pReq->pTags); +} + int32_t tSerializeSMDropStbReq(void **buf, SMDropStbReq *pReq) { int32_t tlen = 0; @@ -600,142 +643,901 @@ void *tDeserializeSStatusRsp(void *buf, SStatusRsp *pRsp) { return buf; } -int32_t tSerializeSCreateAcctReq(void **buf, SCreateAcctReq *pReq) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pReq->user); - tlen += taosEncodeString(buf, pReq->pass); - tlen += taosEncodeFixedI32(buf, pReq->maxUsers); - tlen += taosEncodeFixedI32(buf, pReq->maxDbs); - tlen += taosEncodeFixedI32(buf, pReq->maxTimeSeries); - tlen += taosEncodeFixedI32(buf, pReq->maxStreams); - tlen += taosEncodeFixedI32(buf, pReq->accessState); - tlen += taosEncodeFixedI64(buf, pReq->maxStorage); +int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; + if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1; + if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1; + if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1; + if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1; + if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1; + if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSCreateAcctReq(void *buf, SCreateAcctReq *pReq) { - buf = taosDecodeStringTo(buf, pReq->user); - buf = taosDecodeStringTo(buf, pReq->pass); - buf = taosDecodeFixedI32(buf, &pReq->maxUsers); - buf = taosDecodeFixedI32(buf, &pReq->maxDbs); - buf = taosDecodeFixedI32(buf, &pReq->maxTimeSeries); - buf = taosDecodeFixedI32(buf, &pReq->maxStreams); - buf = taosDecodeFixedI32(buf, &pReq->accessState); - buf = taosDecodeFixedI64(buf, &pReq->maxStorage); - return buf; +int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSDropUserReq(void **buf, SDropUserReq *pReq) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pReq->user); +int32_t tSerializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSDropUserReq(void *buf, SDropUserReq *pReq) { - buf = taosDecodeStringTo(buf, pReq->user); - return buf; +int32_t tDeserializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSCreateUserReq(void **buf, SCreateUserReq *pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI8(buf, pReq->createType); - tlen += taosEncodeFixedI8(buf, pReq->superUser); - tlen += taosEncodeString(buf, pReq->user); - tlen += taosEncodeString(buf, pReq->pass); +int32_t tSerializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI8(&encoder, pReq->createType) < 0) return -1; + if (tEncodeI8(&encoder, pReq->superUser) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSCreateUserReq(void *buf, SCreateUserReq *pReq) { - buf = taosDecodeFixedI8(buf, &pReq->createType); - buf = taosDecodeFixedI8(buf, &pReq->superUser); - buf = taosDecodeStringTo(buf, pReq->user); - buf = taosDecodeStringTo(buf, pReq->pass); - return buf; +int32_t tDeserializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->createType) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->superUser) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSAlterUserReq(void **buf, SAlterUserReq *pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI8(buf, pReq->alterType); - tlen += taosEncodeString(buf, pReq->user); - tlen += taosEncodeString(buf, pReq->pass); - tlen += taosEncodeString(buf, pReq->dbname); - tlen += taosEncodeFixedI8(buf, pReq->superUser); +int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1; + if (tEncodeI8(&encoder, pReq->superUser) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->dbname) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSAlterUserReq(void *buf, SAlterUserReq *pReq) { - buf = taosDecodeFixedI8(buf, &pReq->alterType); - buf = taosDecodeStringTo(buf, pReq->user); - buf = taosDecodeStringTo(buf, pReq->pass); - buf = taosDecodeStringTo(buf, pReq->dbname); - buf = taosDecodeFixedI8(buf, &pReq->superUser); - return buf; +int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->alterType) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->superUser) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->dbname) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSGetUserAuthReq(void **buf, SGetUserAuthReq *pReq) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pReq->user); +int32_t tSerializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSGetUserAuthReq(void *buf, SGetUserAuthReq *pReq) { - buf = taosDecodeStringTo(buf, pReq->user); - return buf; +int32_t tDeserializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSGetUserAuthRsp(void **buf, SGetUserAuthRsp *pReq) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pReq->user); - tlen += taosEncodeFixedI8(buf, pReq->superAuth); +int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - int32_t numOfReadDbs = taosHashGetSize(pReq->readDbs); - int32_t numOfWriteDbs = taosHashGetSize(pReq->writeDbs); - tlen += taosEncodeFixedI32(buf, numOfReadDbs); - tlen += taosEncodeFixedI32(buf, numOfWriteDbs); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->user) < 0) return -1; + if (tEncodeI8(&encoder, pRsp->superAuth) < 0) return -1; - char *db = taosHashIterate(pReq->readDbs, NULL); + int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs); + int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs); + if (tEncodeI32(&encoder, numOfReadDbs) < 0) return -1; + if (tEncodeI32(&encoder, numOfWriteDbs) < 0) return -1; + + char *db = taosHashIterate(pRsp->readDbs, NULL); while (db != NULL) { - tlen += taosEncodeString(buf, db); - db = taosHashIterate(pReq->readDbs, db); + if (tEncodeCStr(&encoder, db) < 0) return -1; + db = taosHashIterate(pRsp->readDbs, db); } - db = taosHashIterate(pReq->writeDbs, NULL); + db = taosHashIterate(pRsp->writeDbs, NULL); while (db != NULL) { - tlen += taosEncodeString(buf, db); - db = taosHashIterate(pReq->writeDbs, db); + if (tEncodeCStr(&encoder, db) < 0) return -1; + db = taosHashIterate(pRsp->writeDbs, db); } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSGetUserAuthRsp(void *buf, SGetUserAuthRsp *pReq) { - buf = taosDecodeStringTo(buf, pReq->user); - buf = taosDecodeFixedI8(buf, &pReq->superAuth); - - pReq->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - pReq->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - if (pReq->readDbs == NULL || pReq->writeDbs == NULL) { - return NULL; +int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) { + return -1; } + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->user) < 0) return -1; + if (tDecodeI8(&decoder, &pRsp->superAuth) < 0) return -1; + int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - buf = taosDecodeFixedI32(buf, &numOfReadDbs); - buf = taosDecodeFixedI32(buf, &numOfWriteDbs); + if (tDecodeI32(&decoder, &numOfReadDbs) < 0) return -1; + if (tDecodeI32(&decoder, &numOfWriteDbs) < 0) return -1; for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - buf = taosDecodeStringTo(buf, db); + if (tDecodeCStrTo(&decoder, db) < 0) return -1; int32_t len = strlen(db) + 1; - taosHashPut(pReq->readDbs, db, len, db, len); + taosHashPut(pRsp->readDbs, db, len, db, len); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - buf = taosDecodeStringTo(buf, db); + if (tDecodeCStrTo(&decoder, db) < 0) return -1; int32_t len = strlen(db) + 1; - taosHashPut(pReq->writeDbs, db, len, db, len); + taosHashPut(pRsp->writeDbs, db, len, db, len); } - return buf; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSMCreateDropQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMCreateDropQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) { + return tSerializeSMCreateDropQSBNodeReq(buf, bufLen, (SMCreateQnodeReq *)pReq); +} + +int32_t tDeserializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) { + return tDeserializeSMCreateDropQSBNodeReq(buf, bufLen, (SMCreateQnodeReq *)pReq); +} + +int32_t tSerializeSMCreateDropMnodeReq(void *buf, int32_t bufLen, SMCreateMnodeReq *pReq) { + return tSerializeSMCreateDropQSBNodeReq(buf, bufLen, (SMCreateQnodeReq *)pReq); +} + +int32_t tDeserializeSMCreateDropMnodeReq(void *buf, int32_t bufLen, SMCreateMnodeReq *pReq) { + return tDeserializeSMCreateDropQSBNodeReq(buf, bufLen, (SMCreateQnodeReq *)pReq); +} + +int32_t tSerializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->config) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->value) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->config) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->value) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->fqdn) < 0) return -1; + if (tEncodeI32(&encoder, pReq->port) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->fqdn) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->port) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeI8(&encoder, pReq->funcType) < 0) return -1; + if (tEncodeI8(&encoder, pReq->scriptType) < 0) return -1; + if (tEncodeI8(&encoder, pReq->outputType) < 0) return -1; + if (tEncodeI32(&encoder, pReq->outputLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->bufSize) < 0) return -1; + if (tEncodeI64(&encoder, pReq->signature) < 0) return -1; + if (tEncodeI32(&encoder, pReq->commentSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->codeSize) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->funcType) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->scriptType) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->outputType) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->outputLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->bufSize) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->commentSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->codeSize) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->pCode) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSDropFuncReq(void *buf, int32_t bufLen, SDropFuncReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDropFuncReq(void *buf, int32_t bufLen, SDropFuncReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1; + + if (pReq->numOfFuncs != (int32_t)taosArrayGetSize(pReq->pFuncNames)) return -1; + for (int32_t i = 0; i < pReq->numOfFuncs; ++i) { + char *fname = taosArrayGet(pReq->pFuncNames, i); + if (tEncodeCStr(&encoder, fname) < 0) return -1; + } + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1; + + pReq->pFuncNames = taosArrayInit(pReq->numOfFuncs, TSDB_FUNC_NAME_LEN); + if (pReq->pFuncNames == NULL) return -1; + + for (int32_t i = 0; i < pReq->numOfFuncs; ++i) { + char fname[TSDB_FUNC_NAME_LEN] = {0}; + if (tDecodeCStrTo(&decoder, fname) < 0) return -1; + taosArrayPush(pReq->pFuncNames, fname); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->numOfFuncs) < 0) return -1; + + if (pRsp->numOfFuncs != (int32_t)taosArrayGetSize(pRsp->pFuncInfos)) return -1; + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncInfo *pInfo = taosArrayGet(pRsp->pFuncInfos, i); + + if (tEncodeCStr(&encoder, pInfo->name) < 0) return -1; + if (tEncodeI8(&encoder, pInfo->funcType) < 0) return -1; + if (tEncodeI8(&encoder, pInfo->scriptType) < 0) return -1; + if (tEncodeI8(&encoder, pInfo->outputType) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->outputLen) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->bufSize) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->signature) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1; + if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; + if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; + } + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->numOfFuncs) < 0) return -1; + + pRsp->pFuncInfos = taosArrayInit(pRsp->numOfFuncs, sizeof(SFuncInfo)); + if (pRsp->pFuncInfos == NULL) return -1; + + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncInfo fInfo = {0}; + if (tDecodeCStrTo(&decoder, fInfo.name) < 0) return -1; + if (tDecodeI8(&decoder, &fInfo.funcType) < 0) return -1; + if (tDecodeI8(&decoder, &fInfo.scriptType) < 0) return -1; + if (tDecodeI8(&decoder, &fInfo.outputType) < 0) return -1; + if (tDecodeI32(&decoder, &fInfo.outputLen) < 0) return -1; + if (tDecodeI32(&decoder, &fInfo.bufSize) < 0) return -1; + if (tDecodeI64(&decoder, &fInfo.signature) < 0) return -1; + if (tDecodeI32(&decoder, &fInfo.commentSize) < 0) return -1; + if (tDecodeI32(&decoder, &fInfo.codeSize) < 0) return -1; + if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1; + if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1; + taosArrayPush(pRsp->pFuncInfos, &fInfo); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfVgroups) < 0) return -1; + if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; + if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1; + if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1; + if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1; + if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; + if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; + if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; + if (tEncodeI8(&encoder, pReq->replications) < 0) return -1; + if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1; + if (tEncodeI8(&encoder, pReq->update) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfVgroups) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->update) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; + if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; + if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI8(&encoder, pReq->ignoreNotExists) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->ignoreNotExists) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1; + if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1; + if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { + if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; + if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1; + + for (int32_t i = 0; i < pRsp->vgNum; ++i) { + SVgroupInfo *pVgInfo = taosArrayGet(pRsp->pVgroupInfos, i); + if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1; + if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1; + if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1; + } + + return 0; +} + +int32_t tSerializeSUseDbRsp(void *buf, int32_t bufLen, SUseDbRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tSerializeSUseDbRspImp(&encoder, pRsp) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSUseDbRspImp(&encoder, pUsedbRsp) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { + if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1; + if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; + + pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo)); + if (pRsp->pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < pRsp->vgNum; ++i) { + SVgroupInfo vgInfo = {0}; + if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1; + if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1; + if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1; + taosArrayPush(pRsp->pVgroupInfos, &vgInfo); + } + + return 0; +} + +int32_t tDeserializeSUseDbRsp(void *buf, int32_t bufLen, SUseDbRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDeserializeSUseDbRspImp(&decoder, pRsp) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SUseDbBatchRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SUseDbRsp usedbRsp = {0}; + if (tDeserializeSUseDbRspImp(&decoder, &usedbRsp) < 0) return -1; + taosArrayPush(pRsp->pArray, &usedbRsp); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSUsedbRsp(SUseDbRsp *pRsp) { taosArrayDestroy(pRsp->pVgroupInfos); } + +void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + for (int32_t i = 0; i < numOfBatch; ++i) { + SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); + tFreeSUsedbRsp(pUsedbRsp); + } + + taosArrayDestroy(pRsp->pArray); +} + +int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->type) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI32(&encoder, pReq->payloadLen) < 0) return -1; + if (pReq->payloadLen > 0) { + if (tEncodeCStr(&encoder, pReq->payload) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; + if (pReq->payloadLen > 0) { + pReq->payload = malloc(pReq->payloadLen); + if (pReq->payload == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +void tFreeSShowReq(SShowReq *pReq) { free(pReq->payload); } + +int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; + if (tEncodeI8(&encoder, pReq->free) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->free) < 0) return -1; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 78eab3151f..3eeb28e532 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -382,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) { pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId}; + SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; SVnode * pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); @@ -594,6 +594,7 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { vnodeCfg.pDnode = pDnode; vnodeCfg.pTfs = pDnode->pTfs; + vnodeCfg.dbId = wrapperCfg.dbUid; SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index dc8390fba3..61119c4308 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -80,12 +80,16 @@ SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) { } void Testbase::SendShowMetaReq(int8_t showType, const char* db) { - int32_t contLen = sizeof(SShowReq); - SShowReq* pShow = (SShowReq*)rpcMallocCont(contLen); - pShow->type = showType; - strcpy(pShow->db, db); + SShowReq showReq = {0}; + showReq.type = showType; + strcpy(showReq.db, db); - SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW, pShow, contLen); + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + char* pReq = (char*)rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); + + SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW, pReq, contLen); SShowRsp* pShowRsp = (SShowRsp*)pRsp->pCont; ASSERT(pShowRsp != nullptr); @@ -128,13 +132,15 @@ int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; } const char* Testbase::GetMetaTbName() { return pMeta->tbName; } void Testbase::SendShowRetrieveReq() { - int32_t contLen = sizeof(SRetrieveTableReq); + SRetrieveTableReq retrieveReq = {0}; + retrieveReq.showId = showId; + retrieveReq.free = 0; - SRetrieveTableReq* pRetrieve = (SRetrieveTableReq*)rpcMallocCont(contLen); - pRetrieve->showId = htobe64(showId); - pRetrieve->free = 0; + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq); - SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW_RETRIEVE, pRetrieve, contLen); + SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW_RETRIEVE, pReq, contLen); pRetrieveRsp = (SRetrieveTableRsp*)pRsp->pCont; pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index c37ae7add5..c2c69f000b 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -29,6 +29,13 @@ int32_t mndCheckCreateUserAuth(SUserObj *pOperUser); int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb, SAlterUserReq *pAlter); int32_t mndCheckDropUserAuth(SUserObj *pOperUser); +int32_t mndCheckNodeAuth(SUserObj *pOperUser); +int32_t mndCheckFuncAuth(SUserObj *pOperUser); + +int32_t mndCheckCreateDbAuth(SUserObj *pOperUser); +int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb); +int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 5a3e6ed26e..ff0095f9f4 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -26,7 +26,7 @@ int32_t mndInitDb(SMnode *pMnode); void mndCleanupDb(SMnode *pMnode); SDbObj *mndAcquireDb(SMnode *pMnode, char *db); void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); -int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void **rsp, int32_t *rspLen); +int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 99b48adb85..84c8b7476c 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -109,4 +109,35 @@ int32_t mndCheckDropUserAuth(SUserObj *pOperUser) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; -} \ No newline at end of file +} + +int32_t mndCheckNodeAuth(SUserObj *pOperUser) { + if (pOperUser->superUser) { + return 0; + } + + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} + +int32_t mndCheckFuncAuth(SUserObj *pOperUser) { + if (pOperUser->superUser) { + return 0; + } + + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} + +int32_t mndCheckCreateDbAuth(SUserObj *pOperUser) { return 0; } + +int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb) { + if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) { + return 0; + } + + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} + +int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb) { return 0; } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index c754494c24..eb9af7b358 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -15,9 +15,11 @@ #define _DEFAULT_SOURCE #include "mndBnode.h" +#include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" +#include "mndUser.h" #define TSDB_BNODE_VER_NUMBER 1 #define TSDB_BNODE_RESERVE_SIZE 64 @@ -257,40 +259,57 @@ CREATE_BNODE_OVER: } static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMCreateBnodeReq *pCreate = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SBnodeObj *pObj = NULL; + SDnodeObj *pDnode = NULL; + SUserObj *pUser = NULL; + SMCreateBnodeReq createReq = {0}; - pCreate->dnodeId = htonl(pCreate->dnodeId); + if (tDeserializeSMCreateDropQSBNodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_BNODE_OVER; + } - mDebug("bnode:%d, start to create", pCreate->dnodeId); + mDebug("bnode:%d, start to create", createReq.dnodeId); - SBnodeObj *pObj = mndAcquireBnode(pMnode, pCreate->dnodeId); + pObj = mndAcquireBnode(pMnode, createReq.dnodeId); if (pObj != NULL) { - mError("bnode:%d, bnode already exist", pObj->id); terrno = TSDB_CODE_MND_BNODE_ALREADY_EXIST; - mndReleaseBnode(pMnode, pObj); - return -1; + goto CREATE_BNODE_OVER; } else if (terrno != TSDB_CODE_MND_BNODE_NOT_EXIST) { - mError("bnode:%d, failed to create bnode since %s", pCreate->dnodeId, terrstr()); - return -1; + goto CREATE_BNODE_OVER; } - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); if (pDnode == NULL) { - mError("bnode:%d, dnode not exist", pCreate->dnodeId); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - return -1; + goto CREATE_BNODE_OVER; } - int32_t code = mndCreateBnode(pMnode, pReq, pDnode, pCreate); + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto CREATE_BNODE_OVER; + } + + if (mndCheckNodeAuth(pUser)) { + goto CREATE_BNODE_OVER; + } + + code = mndCreateBnode(pMnode, pReq, pDnode, &createReq); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_BNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("bnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); + } + + mndReleaseBnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); + mndReleaseUser(pMnode, pUser); - if (code != 0) { - mError("bnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); - return -1; - } - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return code; } static int32_t mndSetDropBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { @@ -352,33 +371,51 @@ DROP_BNODE_OVER: } static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMDropBnodeReq *pDrop = pReq->rpcMsg.pCont; - pDrop->dnodeId = htonl(pDrop->dnodeId); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SBnodeObj *pObj = NULL; + SMDropBnodeReq dropReq = {0}; - mDebug("bnode:%d, start to drop", pDrop->dnodeId); + if (tDeserializeSMCreateDropQSBNodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_BNODE_OVER; + } - if (pDrop->dnodeId <= 0) { + mDebug("bnode:%d, start to drop", dropReq.dnodeId); + + if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_SDB_APP_ERROR; - mError("bnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + goto DROP_BNODE_OVER; } - SBnodeObj *pObj = mndAcquireBnode(pMnode, pDrop->dnodeId); + pObj = mndAcquireBnode(pMnode, dropReq.dnodeId); if (pObj == NULL) { - mError("bnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + goto DROP_BNODE_OVER; } - int32_t code = mndDropBnode(pMnode, pReq, pObj); - if (code != 0) { - sdbRelease(pMnode->pSdb, pObj); - mError("bnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto DROP_BNODE_OVER; } - sdbRelease(pMnode->pSdb, pObj); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckNodeAuth(pUser)) { + goto DROP_BNODE_OVER; + } + + code = mndDropBnode(pMnode, pReq, pObj); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_BNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("bnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); + } + + mndReleaseBnode(pMnode, pObj); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index e87021b44d..21ebea258d 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -59,6 +59,7 @@ SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pConsumer->recentRemovedTopics = taosArrayInit(0, sizeof(char*)); pConsumer->epoch = 1; pConsumer->consumerId = consumerId; atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT); @@ -169,7 +170,7 @@ SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId) { SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); if (pConsumer == NULL) { - /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ + terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; } return pConsumer; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5ed3e9e8df..7c42ed5df6 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -369,7 +370,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.pCont = pReq; action.contLen = sizeof(SDropVnodeReq); action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); return -1; @@ -451,54 +452,54 @@ CREATE_DB_OVER: } static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SCreateDbReq *pCreate = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SCreateDbReq createReq = {0}; - pCreate->numOfVgroups = htonl(pCreate->numOfVgroups); - pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); - pCreate->totalBlocks = htonl(pCreate->totalBlocks); - pCreate->daysPerFile = htonl(pCreate->daysPerFile); - pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); - pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); - pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); - pCreate->minRows = htonl(pCreate->minRows); - pCreate->maxRows = htonl(pCreate->maxRows); - pCreate->commitTime = htonl(pCreate->commitTime); - pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); + if (tDeserializeSCreateDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_DB_OVER; + } - mDebug("db:%s, start to create, vgroups:%d", pCreate->db, pCreate->numOfVgroups); + mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); - SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); + pDb = mndAcquireDb(pMnode, createReq.db); if (pDb != NULL) { - mndReleaseDb(pMnode, pDb); - if (pCreate->ignoreExist) { - mDebug("db:%s, already exist, ignore exist is set", pCreate->db); - return 0; + if (createReq.ignoreExist) { + mDebug("db:%s, already exist, ignore exist is set", createReq.db); + code = 0; + goto CREATE_DB_OVER; } else { terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + goto CREATE_DB_OVER; } } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + goto CREATE_DB_OVER; } - SUserObj *pOperUser = mndAcquireUser(pMnode, pReq->user); - if (pOperUser == NULL) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto CREATE_DB_OVER; } - int32_t code = mndCreateDb(pMnode, pReq, pCreate, pOperUser); - mndReleaseUser(pMnode, pOperUser); - - if (code != 0) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + if (mndCheckCreateDbAuth(pUser) != 0) { + goto CREATE_DB_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + code = mndCreateDb(pMnode, pReq, &createReq, pUser); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_DB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("db:%s, failed to create since %s", createReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { @@ -633,43 +634,56 @@ UPDATE_DB_OVER: } static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SAlterDbReq *pAlter = pReq->rpcMsg.pCont; - pAlter->totalBlocks = htonl(pAlter->totalBlocks); - pAlter->daysToKeep0 = htonl(pAlter->daysToKeep0); - pAlter->daysToKeep1 = htonl(pAlter->daysToKeep1); - pAlter->daysToKeep2 = htonl(pAlter->daysToKeep2); - pAlter->fsyncPeriod = htonl(pAlter->fsyncPeriod); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SAlterDbReq alterReq = {0}; - mDebug("db:%s, start to alter", pAlter->db); + if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto ALTER_DB_OVER; + } - SDbObj *pDb = mndAcquireDb(pMnode, pAlter->db); + mDebug("db:%s, start to alter", alterReq.db); + + pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { - mError("db:%s, failed to alter since %s", pAlter->db, terrstr()); - return TSDB_CODE_MND_DB_NOT_EXIST; + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + goto ALTER_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto ALTER_DB_OVER; + } + + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto ALTER_DB_OVER; } SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - int32_t code = mndSetDbCfgFromAlterDbReq(&dbObj, pAlter); + code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) { - mndReleaseDb(pMnode, pDb); - mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); - return code; + goto ALTER_DB_OVER; } dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndUpdateDb(pMnode, pReq, pDb, &dbObj); - mndReleaseDb(pMnode, pDb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; - if (code != 0) { - mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); - return code; +ALTER_DB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { @@ -771,11 +785,18 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; - int32_t rspLen = sizeof(SDropDbRsp); - SDropDbRsp *pRsp = rpcMallocCont(rspLen); - if (pRsp == NULL) goto DROP_DB_OVER; - memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); - pRsp->uid = htobe64(pDb->uid); + SDropDbRsp dropRsp = {0}; + memcpy(dropRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + dropRsp.uid = pDb->uid; + + int32_t rspLen = tSerializeSDropDbRsp(NULL, 0, &dropRsp); + void *pRsp = malloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto DROP_DB_OVER; + } + tSerializeSDropDbRsp(pRsp, rspLen, &dropRsp); + mndTransSetRpcRsp(pTrans, pRsp, rspLen); if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_DB_OVER; @@ -788,37 +809,56 @@ DROP_DB_OVER: } static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SDropDbReq *pDrop = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SDropDbReq dropReq = {0}; - mDebug("db:%s, start to drop", pDrop->db); + if (tDeserializeSDropDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_DB_OVER; + } - SDbObj *pDb = mndAcquireDb(pMnode, pDrop->db); + mDebug("db:%s, start to drop", dropReq.db); + + pDb = mndAcquireDb(pMnode, dropReq.db); if (pDb == NULL) { - if (pDrop->ignoreNotExists) { - mDebug("db:%s, not exist, ignore not exist is set", pDrop->db); - return TSDB_CODE_SUCCESS; + if (dropReq.ignoreNotExists) { + code = 0; + goto DROP_DB_OVER; } else { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); - return -1; + goto DROP_DB_OVER; } } - int32_t code = mndDropDb(pMnode, pReq, pDb); - mndReleaseDb(pMnode, pDb); - - if (code != 0) { - mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); - return code; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto DROP_DB_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto DROP_DB_OVER; + } + + code = mndDropDb(pMnode, pReq, pDb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_DB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("db:%s, failed to drop since %s", dropReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + return code; } -static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgList, int32_t *vgNum) { +static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { int32_t vindex = 0; - SSdb *pSdb = pMnode->pSdb; + SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (vindex < pDb->cfg.numOfVgroups) { @@ -827,168 +867,243 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgLis if (pIter == NULL) break; if (pVgroup->dbUid == pDb->uid) { - SVgroupInfo *pInfo = &vgList[vindex]; - pInfo->vgId = htonl(pVgroup->vgId); - pInfo->hashBegin = htonl(pVgroup->hashBegin); - pInfo->hashEnd = htonl(pVgroup->hashEnd); - pInfo->epset.numOfEps = pVgroup->replica; + SVgroupInfo vgInfo = {0}; + vgInfo.vgId = pVgroup->vgId; + vgInfo.hashBegin = pVgroup->hashBegin; + vgInfo.hashEnd = pVgroup->hashEnd; + vgInfo.epset.numOfEps = pVgroup->replica; for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; - SEp * pEp = &pInfo->epset.eps[gid]; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; + SEp *pEp = &vgInfo.epset.eps[gid]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode != NULL) { memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - pEp->port = htons(pDnode->port); + pEp->port = pDnode->port; } mndReleaseDnode(pMnode, pDnode); if (pVgid->role == TAOS_SYNC_STATE_LEADER) { - pInfo->epset.inUse = gid; + vgInfo.epset.inUse = gid; } } vindex++; + taosArrayPush(pVgList, &vgInfo); } sdbRelease(pSdb, pVgroup); } - - *vgNum = vindex; } static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SSdb *pSdb = pMnode->pSdb; - SUseDbReq *pUse = pReq->rpcMsg.pCont; - pUse->vgVersion = htonl(pUse->vgVersion); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SUseDbReq usedbReq = {0}; + SUseDbRsp usedbRsp = {0}; - SDbObj *pDb = mndAcquireDb(pMnode, pUse->db); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - mError("db:%s, failed to process use db req since %s", pUse->db, terrstr()); - return -1; + if (tDeserializeSUseDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &usedbReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto USE_DB_OVER; } - int32_t contLen = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); - SUseDbRsp *pRsp = rpcMallocCont(contLen); + pDb = mndAcquireDb(pMnode, usedbReq.db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + goto USE_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto USE_DB_OVER; + } + + if (mndCheckUseDbAuth(pUser, pDb) != 0) { + goto USE_DB_OVER; + } + + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } + + if (usedbReq.vgVersion < pDb->vgVersion) { + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + } + + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + + int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); + void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } + + tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); + + pReq->pCont = pRsp; + pReq->contLen = contLen; + code = 0; + +USE_DB_OVER: + if (code != 0) { + mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + tFreeSUsedbRsp(&usedbRsp); + + return code; +} + +int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen) { + SUseDbBatchRsp batchUseRsp = {0}; + batchUseRsp.pArray = taosArrayInit(numOfDbs, sizeof(SUseDbRsp)); + if (batchUseRsp.pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - int32_t vgNum = 0; + for (int32_t i = 0; i < numOfDbs; ++i) { + SDbVgVersion *pDbVgVersion = &pDbs[i]; + pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId); + pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); - if (pUse->vgVersion < pDb->vgVersion) { - mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); - } + SUseDbRsp usedbRsp = {0}; - memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); - pRsp->uid = htobe64(pDb->uid); - pRsp->vgVersion = htonl(pDb->vgVersion); - pRsp->vgNum = htonl(vgNum); - pRsp->hashMethod = pDb->hashMethod; - - pReq->pCont = pRsp; - pReq->contLen = contLen; - mndReleaseDb(pMnode, pDb); - - return 0; -} - -int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void **rsp, int32_t *rspLen) { - SSdb *pSdb = pMnode->pSdb; - int32_t bufSize = num * (sizeof(SUseDbRsp) + TSDB_DEFAULT_VN_PER_DB * sizeof(SVgroupInfo)); - void *buf = malloc(bufSize); - int32_t len = 0; - int32_t contLen = 0; - int32_t bufOffset = 0; - SUseDbRsp *pRsp = NULL; - - for (int32_t i = 0; i < num; ++i) { - SDbVgVersion *db = &dbs[i]; - db->dbId = be64toh(db->dbId); - db->vgVersion = ntohl(db->vgVersion); - - len = 0; - - SDbObj *pDb = mndAcquireDb(pMnode, db->dbFName); + SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); if (pDb == NULL) { - mInfo("db %s not exist", db->dbFName); - - len = sizeof(SUseDbRsp); - } else if (pDb->uid != db->dbId || db->vgVersion < pDb->vgVersion) { - len = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); - } - - if (0 == len) { + mDebug("db:%s, no exist", pDbVgVersion->dbFName); + memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDbVgVersion->dbId; + usedbRsp.vgVersion = -1; + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + } else if (pDbVgVersion->vgVersion >= pDb->vgVersion) { + mDebug("db:%s, version not changed", pDbVgVersion->dbFName); mndReleaseDb(pMnode, pDb); - continue; - } - - contLen += len; - - if (contLen > bufSize) { - buf = realloc(buf, contLen); - } - - pRsp = (SUseDbRsp *)((char *)buf + bufOffset); - memcpy(pRsp->db, db->dbFName, TSDB_DB_FNAME_LEN); - if (pDb) { - int32_t vgNum = 0; - mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); - - pRsp->uid = htobe64(pDb->uid); - pRsp->vgVersion = htonl(pDb->vgVersion); - pRsp->vgNum = htonl(vgNum); - pRsp->hashMethod = pDb->hashMethod; } else { - pRsp->uid = htobe64(db->dbId); - pRsp->vgNum = htonl(0); - pRsp->hashMethod = 0; - pRsp->vgVersion = htonl(-1); + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + mndReleaseDb(pMnode, pDb); + mError("db:%s, failed to malloc usedb response", pDb->name); + continue; + } + + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + mndReleaseDb(pMnode, pDb); } - - bufOffset += len; - - mndReleaseDb(pMnode, pDb); } - if (contLen > 0) { - *rsp = buf; - *rspLen = contLen; - } else { - *rsp = NULL; - tfree(buf); - *rspLen = 0; + int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); + void *pRsp = malloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeSUseDbBatchRsp(&batchUseRsp); + return -1; } + tSerializeSUseDbBatchRsp(pRsp, rspLen, &batchUseRsp); + *ppRsp = pRsp; + *pRspLen = rspLen; + + tFreeSUseDbBatchRsp(&batchUseRsp); return 0; } static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SSyncDbReq *pSync = pReq->rpcMsg.pCont; - SDbObj *pDb = mndAcquireDb(pMnode, pSync->db); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SSyncDbReq syncReq = {0}; + + if (tDeserializeSSyncDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &syncReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto SYNC_DB_OVER; + } + + mDebug("db:%s, start to sync", syncReq.db); + + pDb = mndAcquireDb(pMnode, syncReq.db); if (pDb == NULL) { - mError("db:%s, failed to process sync db req since %s", pSync->db, terrstr()); - return -1; + goto SYNC_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto SYNC_DB_OVER; + } + + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto SYNC_DB_OVER; + } + + // code = mndSyncDb(); + +SYNC_DB_OVER: + if (code != 0) { + mError("db:%s, failed to process sync db req since %s", syncReq.db, terrstr()); } mndReleaseDb(pMnode, pDb); - return 0; + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SCompactDbReq *pCompact = pReq->rpcMsg.pCont; - SDbObj *pDb = mndAcquireDb(pMnode, pCompact->db); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SCompactDbReq compactReq = {0}; + + if (tDeserializeSSyncDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &compactReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto SYNC_DB_OVER; + } + + mDebug("db:%s, start to sync", compactReq.db); + + pDb = mndAcquireDb(pMnode, compactReq.db); if (pDb == NULL) { - mError("db:%s, failed to process compact db req since %s", pCompact->db, terrstr()); - return -1; + goto SYNC_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto SYNC_DB_OVER; + } + + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto SYNC_DB_OVER; + } + + // code = mndSyncDb(); + +SYNC_DB_OVER: + if (code != 0) { + mError("db:%s, failed to process compact db req since %s", compactReq.db, terrstr()); } mndReleaseDb(pMnode, pDb); - return 0; + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 93c6b492bd..5126ca9db3 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -15,9 +15,11 @@ #define _DEFAULT_SOURCE #include "mndDnode.h" +#include "mndAuth.h" #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" +#include "mndUser.h" #include "mndVgroup.h" #define TSDB_DNODE_VER_NUMBER 1 @@ -354,7 +356,8 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { if (pDnode != NULL) { pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; } - mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, pMnode->cfg.sver); + mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, + pMnode->cfg.sver); terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; goto PROCESS_STATUS_MSG_OVER; } @@ -461,35 +464,53 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pReq, SCreateDnodeReq * } static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SCreateDnodeReq *pCreate = pReq->rpcMsg.pCont; - pCreate->port = htonl(pCreate->port); - mDebug("dnode:%s:%d, start to create", pCreate->fqdn, pCreate->port); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SDnodeObj *pDnode = NULL; + SCreateDnodeReq createReq = {0}; - if (pCreate->fqdn[0] == 0 || pCreate->port <= 0 || pCreate->port > UINT16_MAX) { + if (tDeserializeSCreateDnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_DNODE_OVER; + } + + mDebug("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); + + if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) { terrno = TSDB_CODE_MND_INVALID_DNODE_EP; - mError("dnode:%s:%d, failed to create since %s", pCreate->fqdn, pCreate->port, terrstr()); - return -1; + goto CREATE_DNODE_OVER; } char ep[TSDB_EP_LEN]; - snprintf(ep, TSDB_EP_LEN, "%s:%d", pCreate->fqdn, pCreate->port); - SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, ep); + snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port); + pDnode = mndAcquireDnodeByEp(pMnode, ep); if (pDnode != NULL) { - mError("dnode:%d, already exist, %s:%u", pDnode->id, pCreate->fqdn, pCreate->port); - mndReleaseDnode(pMnode, pDnode); terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; - return -1; + goto CREATE_DNODE_OVER; } - int32_t code = mndCreateDnode(pMnode, pReq, pCreate); - - if (code != 0) { - mError("dnode:%s:%d, failed to create since %s", pCreate->fqdn, pCreate->port, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto CREATE_DNODE_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckNodeAuth(pUser)) { + goto CREATE_DNODE_OVER; + } + + code = mndCreateDnode(pMnode, pReq, &createReq); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_DNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); + } + + mndReleaseDnode(pMnode, pDnode); + mndReleaseUser(pMnode, pUser); + return code; } static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) { @@ -520,44 +541,66 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; - SDropDnodeReq *pDrop = pReq->rpcMsg.pCont; - pDrop->dnodeId = htonl(pDrop->dnodeId); + int32_t code = -1; + SUserObj *pUser = NULL; + SDnodeObj *pDnode = NULL; + SMDropMnodeReq dropReq = {0}; - mDebug("dnode:%d, start to drop", pDrop->dnodeId); - - if (pDrop->dnodeId <= 0) { - terrno = TSDB_CODE_MND_INVALID_DNODE_ID; - mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + if (tDeserializeSMCreateDropMnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_DNODE_OVER; } - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDrop->dnodeId); + mDebug("dnode:%d, start to drop", dropReq.dnodeId); + + if (dropReq.dnodeId <= 0) { + terrno = TSDB_CODE_MND_INVALID_DNODE_ID; + goto DROP_DNODE_OVER; + } + + pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId); if (pDnode == NULL) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + goto DROP_DNODE_OVER; } - int32_t code = mndDropDnode(pMnode, pReq, pDnode); - if (code != 0) { - mndReleaseDnode(pMnode, pDnode); - mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto DROP_DNODE_OVER; + } + + if (mndCheckNodeAuth(pUser)) { + goto DROP_DNODE_OVER; + } + + code = mndDropDnode(pMnode, pReq, pDnode); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_DNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } mndReleaseDnode(pMnode, pDnode); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMCfgDnodeReq *pCfg = pReq->rpcMsg.pCont; - pCfg->dnodeId = htonl(pCfg->dnodeId); + SMnode *pMnode = pReq->pMnode; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCfg->dnodeId); + SMCfgDnodeReq cfgReq = {0}; + if (tDeserializeSMCfgDnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, cfgReq.dnodeId); if (pDnode == NULL) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - mError("dnode:%d, failed to config since %s ", pCfg->dnodeId, terrstr()); + mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); return -1; } @@ -565,15 +608,15 @@ static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { mndReleaseDnode(pMnode, pDnode); SDCfgDnodeReq *pCfgDnode = rpcMallocCont(sizeof(SDCfgDnodeReq)); - pCfgDnode->dnodeId = htonl(pCfg->dnodeId); - memcpy(pCfgDnode->config, pCfg->config, TSDB_DNODE_CONFIG_LEN); + pCfgDnode->dnodeId = htonl(cfgReq.dnodeId); + memcpy(pCfgDnode->config, cfgReq.config, TSDB_DNODE_CONFIG_LEN); SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pCfgDnode, .contLen = sizeof(SDCfgDnodeReq), .ahandle = pReq->rpcMsg.ahandle}; - mInfo("dnode:%d, app:%p config:%s req send to dnode", pCfg->dnodeId, rpcMsg.ahandle, pCfg->config); + mInfo("dnode:%d, app:%p config:%s req send to dnode", cfgReq.dnodeId, rpcMsg.ahandle, cfgReq.config); mndSendReqToDnode(pMnode, &epSet, &rpcMsg); return 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b3fc0e1da0..3527f103db 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -15,9 +15,11 @@ #define _DEFAULT_SOURCE #include "mndFunc.h" +#include "mndAuth.h" #include "mndShow.h" #include "mndSync.h" #include "mndTrans.h" +#include "mndUser.h" #define SDB_FUNC_VER 1 #define SDB_FUNC_RESERVE_SIZE 64 @@ -201,8 +203,8 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pC goto CREATE_FUNC_OVER; } - memcpy(func.pComment, pCreate->pCont, pCreate->commentSize); - memcpy(func.pCode, pCreate->pCont + pCreate->commentSize, func.codeSize); + memcpy(func.pComment, pCreate->pComment, pCreate->commentSize); + memcpy(func.pCode, pCreate->pCode, func.codeSize); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_FUNC_OVER; @@ -261,164 +263,202 @@ DROP_FUNC_OVER: } static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SFuncObj *pFunc = NULL; + SCreateFuncReq createReq = {0}; - SCreateFuncReq *pCreate = pReq->rpcMsg.pCont; - pCreate->outputLen = htonl(pCreate->outputLen); - pCreate->bufSize = htonl(pCreate->bufSize); - pCreate->signature = htobe64(pCreate->signature); - pCreate->commentSize = htonl(pCreate->commentSize); - pCreate->codeSize = htonl(pCreate->codeSize); + if (tDeserializeSCreateFuncReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_FUNC_OVER; + } - mDebug("func:%s, start to create", pCreate->name); + mDebug("func:%s, start to create", createReq.name); - SFuncObj *pFunc = mndAcquireFunc(pMnode, pCreate->name); + pFunc = mndAcquireFunc(pMnode, createReq.name); if (pFunc != NULL) { - mndReleaseFunc(pMnode, pFunc); - if (pCreate->igExists) { - mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); - return 0; + if (createReq.igExists) { + mDebug("func:%s, already exist, ignore exist is set", createReq.name); + code = 0; + goto CREATE_FUNC_OVER; } else { terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } } else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) { - mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->name[0] == 0) { + if (createReq.name[0] == 0) { terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->commentSize <= 0 || pCreate->commentSize > TSDB_FUNC_COMMENT_LEN) { + if (createReq.commentSize <= 0 || createReq.commentSize > TSDB_FUNC_COMMENT_LEN) { terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->codeSize <= 0 || pCreate->codeSize > TSDB_FUNC_CODE_LEN) { + if (createReq.codeSize <= 0 || createReq.codeSize > TSDB_FUNC_CODE_LEN) { terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->pCont[0] == 0) { + if (createReq.pCode[0] == 0) { terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->bufSize <= 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { + if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - int32_t code = mndCreateFunc(pMnode, pReq, pCreate); - if (code != 0) { - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto CREATE_FUNC_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckFuncAuth(pUser)) { + goto CREATE_FUNC_OVER; + } + + code = mndCreateFunc(pMnode, pReq, &createReq); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_FUNC_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("func:%s, failed to create since %s", createReq.name, terrstr()); + } + + mndReleaseFunc(pMnode, pFunc); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SDropFuncReq *pDrop = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SFuncObj *pFunc = NULL; + SDropFuncReq dropReq = {0}; - mDebug("func:%s, start to drop", pDrop->name); - - if (pDrop->name[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; - mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); - return -1; + if (tDeserializeSDropFuncReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_FUNC_OVER; } - SFuncObj *pFunc = mndAcquireFunc(pMnode, pDrop->name); + mDebug("func:%s, start to drop", dropReq.name); + + if (dropReq.name[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; + goto DROP_FUNC_OVER; + } + + pFunc = mndAcquireFunc(pMnode, dropReq.name); if (pFunc == NULL) { - if (pDrop->igNotExists) { - mDebug("func:%s, not exist, ignore not exist is set", pDrop->name); - return 0; + if (dropReq.igNotExists) { + mDebug("func:%s, not exist, ignore not exist is set", dropReq.name); + code = 0; + goto DROP_FUNC_OVER; } else { terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; - mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); - return -1; + goto DROP_FUNC_OVER; } } - int32_t code = mndDropFunc(pMnode, pReq, pFunc); - mndReleaseFunc(pMnode, pFunc); - - if (code != 0) { - mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto DROP_FUNC_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckFuncAuth(pUser)) { + goto DROP_FUNC_OVER; + } + + code = mndDropFunc(pMnode, pReq, pFunc); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_FUNC_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("func:%s, failed to drop since %s", dropReq.name, terrstr()); + } + + mndReleaseFunc(pMnode, pFunc); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { - int32_t code = -1; - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SRetrieveFuncReq retrieveReq = {0}; + SRetrieveFuncRsp retrieveRsp = {0}; - SRetrieveFuncReq *pRetrieve = pReq->rpcMsg.pCont; - pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); - if (pRetrieve->numOfFuncs <= 0 || pRetrieve->numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { + if (tDeserializeSRetrieveFuncReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &retrieveReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto RETRIEVE_FUNC_OVER; + } + + if (retrieveReq.numOfFuncs <= 0 || retrieveReq.numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE; - return -1; + goto RETRIEVE_FUNC_OVER; } - int32_t fsize = sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN + TSDB_FUNC_COMMENT_LEN; - int32_t size = sizeof(SRetrieveFuncRsp) + fsize * pRetrieve->numOfFuncs; - - SRetrieveFuncRsp *pRetrieveRsp = rpcMallocCont(size); - if (pRetrieveRsp == NULL) { + retrieveRsp.numOfFuncs = retrieveReq.numOfFuncs; + retrieveRsp.pFuncInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncInfo)); + if (retrieveRsp.pFuncInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FUNC_RETRIEVE_OVER; + goto RETRIEVE_FUNC_OVER; } - pRetrieveRsp->numOfFuncs = htonl(pRetrieve->numOfFuncs); - char *pOutput = pRetrieveRsp->pFuncInfos; - - for (int32_t i = 0; i < pRetrieve->numOfFuncs; ++i) { - char funcName[TSDB_FUNC_NAME_LEN] = {0}; - memcpy(funcName, pRetrieve->pFuncNames + i * TSDB_FUNC_NAME_LEN, TSDB_FUNC_NAME_LEN); + for (int32_t i = 0; i < retrieveReq.numOfFuncs; ++i) { + char *funcName = taosArrayGet(retrieveReq.pFuncNames, i); SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); if (pFunc == NULL) { terrno = TSDB_CODE_MND_INVALID_FUNC; - mError("func:%s, failed to retrieve since %s", funcName, terrstr()); - goto FUNC_RETRIEVE_OVER; + goto RETRIEVE_FUNC_OVER; } - SFuncInfo *pFuncInfo = (SFuncInfo *)pOutput; - memcpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); - pFuncInfo->funcType = pFunc->funcType; - pFuncInfo->scriptType = pFunc->scriptType; - pFuncInfo->outputType = pFunc->outputType; - pFuncInfo->outputLen = htonl(pFunc->outputLen); - pFuncInfo->bufSize = htonl(pFunc->bufSize); - pFuncInfo->signature = htobe64(pFunc->signature); - pFuncInfo->commentSize = htonl(pFunc->commentSize); - pFuncInfo->codeSize = htonl(pFunc->codeSize); - memcpy(pFuncInfo->pCont, pFunc->pComment, pFunc->commentSize); - memcpy(pFuncInfo->pCont + pFunc->commentSize, pFunc->pCode, pFunc->codeSize); - pOutput += (sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize); + SFuncInfo funcInfo = {0}; + memcpy(funcInfo.name, pFunc->name, TSDB_FUNC_NAME_LEN); + funcInfo.funcType = pFunc->funcType; + funcInfo.scriptType = pFunc->scriptType; + funcInfo.outputType = pFunc->outputType; + funcInfo.outputLen = pFunc->outputLen; + funcInfo.bufSize = pFunc->bufSize; + funcInfo.signature = pFunc->signature; + funcInfo.commentSize = pFunc->commentSize; + funcInfo.codeSize = pFunc->codeSize; + memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize); + memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize); + taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo); mndReleaseFunc(pMnode, pFunc); } - pReq->pCont = pRetrieveRsp; - pReq->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); + int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &retrieveRsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto RETRIEVE_FUNC_OVER; + } + + tSerializeSRetrieveFuncRsp(pRsp, contLen, &retrieveRsp); + + pReq->pCont = pRsp; + pReq->contLen = contLen; code = 0; -FUNC_RETRIEVE_OVER: - if (code != 0) rpcFreeCont(pRetrieveRsp); +RETRIEVE_FUNC_OVER: + taosArrayDestroy(retrieveReq.pFuncNames); + taosArrayDestroy(retrieveRsp.pFuncInfos); return code; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 1e0164a572..eb9ba49dd2 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -15,9 +15,11 @@ #define _DEFAULT_SOURCE #include "mndMnode.h" +#include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" +#include "mndUser.h" #define TSDB_MNODE_VER_NUMBER 1 #define TSDB_MNODE_RESERVE_SIZE 64 @@ -379,40 +381,57 @@ CREATE_MNODE_OVER: } static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMCreateMnodeReq *pCreate = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SMnodeObj *pObj = NULL; + SDnodeObj *pDnode = NULL; + SUserObj *pUser = NULL; + SMCreateMnodeReq createReq = {0}; - pCreate->dnodeId = htonl(pCreate->dnodeId); + if (tDeserializeSMCreateDropMnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_MNODE_OVER; + } - mDebug("mnode:%d, start to create", pCreate->dnodeId); + mDebug("mnode:%d, start to create", createReq.dnodeId); - SMnodeObj *pObj = mndAcquireMnode(pMnode, pCreate->dnodeId); + pObj = mndAcquireMnode(pMnode, createReq.dnodeId); if (pObj != NULL) { - mndReleaseMnode(pMnode, pObj); - mError("mnode:%d, mnode already exist", pObj->id); terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST; - return -1; + goto CREATE_MNODE_OVER; } else if (terrno != TSDB_CODE_MND_MNODE_NOT_EXIST) { - mError("qnode:%d, failed to create mnode since %s", pCreate->dnodeId, terrstr()); - return -1; + goto CREATE_MNODE_OVER; } - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); if (pDnode == NULL) { - mError("mnode:%d, dnode not exist", pCreate->dnodeId); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - return -1; + goto CREATE_MNODE_OVER; } - int32_t code = mndCreateMnode(pMnode, pReq, pDnode, pCreate); + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto CREATE_MNODE_OVER; + } + + if (mndCheckNodeAuth(pUser)) { + goto CREATE_MNODE_OVER; + } + + code = mndCreateMnode(pMnode, pReq, pDnode, &createReq); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_MNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("mnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); + } + + mndReleaseMnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); + mndReleaseUser(pMnode, pUser); - if (code != 0) { - mError("mnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); - return -1; - } - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return code; } static int32_t mndSetDropMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { @@ -534,32 +553,51 @@ DROP_MNODE_OVER: } static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMDropMnodeReq *pDrop = pReq->rpcMsg.pCont; - pDrop->dnodeId = htonl(pDrop->dnodeId); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SMnodeObj *pObj = NULL; + SMDropMnodeReq dropReq = {0}; - mDebug("mnode:%d, start to drop", pDrop->dnodeId); + if (tDeserializeSMCreateDropMnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_MNODE_OVER; + } - if (pDrop->dnodeId <= 0) { + mDebug("mnode:%d, start to drop", dropReq.dnodeId); + + if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_SDB_APP_ERROR; - mError("mnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + goto DROP_MNODE_OVER; } - SMnodeObj *pObj = mndAcquireMnode(pMnode, pDrop->dnodeId); + pObj = mndAcquireMnode(pMnode, dropReq.dnodeId); if (pObj == NULL) { - mError("mnode:%d, not exist", pDrop->dnodeId); - return -1; + goto DROP_MNODE_OVER; } - int32_t code = mndDropMnode(pMnode, pReq, pObj); - if (code != 0) { - mError("mnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto DROP_MNODE_OVER; } - sdbRelease(pMnode->pSdb, pObj); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckNodeAuth(pUser)) { + goto DROP_MNODE_OVER; + } + + code = mndDropMnode(pMnode, pReq, pObj); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_MNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("mnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); + } + + mndReleaseMnode(pMnode, pObj); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d63ade4320..f798928d9c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -343,17 +343,21 @@ static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) { static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; - char *batchReqStr = pReq->rpcMsg.pCont; + SClientHbBatchReq batchReq = {0}; - tDeserializeSClientHbBatchReq(batchReqStr, &batchReq); + if (tDeserializeSClientHbBatchReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &batchReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + SArray *pArray = batchReq.reqs; - int sz = taosArrayGetSize(pArray); + int32_t sz = taosArrayGetSize(pArray); SClientHbBatchRsp batchRsp = {0}; batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); for (int i = 0; i < sz; i++) { - SClientHbReq* pHbReq = taosArrayGet(pArray, i); + SClientHbReq *pHbReq = taosArrayGet(pArray, i); if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) { int32_t kvNum = taosHashGetSize(pHbReq->info); if (NULL == pHbReq->info || kvNum <= 0) { @@ -364,13 +368,13 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { void *pIter = taosHashIterate(pHbReq->info, NULL); while (pIter != NULL) { - SKv* kv = pIter; - + SKv *kv = pIter; + switch (kv->key) { case HEARTBEAT_KEY_DBINFO: { - void *rspMsg = NULL; + void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateDBInfo(pMnode, (SDbVgVersion *)kv->value, kv->valueLen/sizeof(SDbVgVersion), &rspMsg, &rspLen); + mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv); @@ -378,9 +382,9 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { break; } case HEARTBEAT_KEY_STBINFO: { - void *rspMsg = NULL; + void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateStbInfo(pMnode, (SSTableMetaVersion *)kv->value, kv->valueLen/sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv); @@ -392,7 +396,7 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { hbRsp.status = TSDB_CODE_MND_APP_ERROR; break; } - + pIter = taosHashIterate(pHbReq->info, pIter); } @@ -407,15 +411,14 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } taosArrayDestroyEx(pArray, tFreeClientHbReq); - int32_t tlen = tSerializeSClientHbBatchRsp(NULL, &batchRsp); - void* buf = rpcMallocCont(tlen); - void* abuf = buf; - tSerializeSClientHbBatchRsp(&abuf, &batchRsp); - + int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp); + void *buf = rpcMallocCont(tlen); + tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); + int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps); for (int32_t i = 0; i < rspNum; ++i) { SClientHbRsp *rsp = taosArrayGet(batchRsp.rsps, i); - int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info): 0; + int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info) : 0; for (int32_t n = 0; n < kvNum; ++n) { SKv *kv = taosArrayGet(rsp->info, n); tfree(kv->value); diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index e29ab5a5bd..db1ea2ab17 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -15,9 +15,11 @@ #define _DEFAULT_SOURCE #include "mndQnode.h" +#include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" +#include "mndUser.h" #define TSDB_QNODE_VER_NUMBER 1 #define TSDB_QNODE_RESERVE_SIZE 64 @@ -257,40 +259,57 @@ CREATE_QNODE_OVER: } static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMCreateQnodeReq *pCreate = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SQnodeObj *pObj = NULL; + SDnodeObj *pDnode = NULL; + SUserObj *pUser = NULL; + SMCreateQnodeReq createReq = {0}; - pCreate->dnodeId = htonl(pCreate->dnodeId); + if (tDeserializeSMCreateDropQSBNodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_QNODE_OVER; + } - mDebug("qnode:%d, start to create", pCreate->dnodeId); + mDebug("qnode:%d, start to create", createReq.dnodeId); - SQnodeObj *pObj = mndAcquireQnode(pMnode, pCreate->dnodeId); + pObj = mndAcquireQnode(pMnode, createReq.dnodeId); if (pObj != NULL) { - mError("qnode:%d, qnode already exist", pObj->id); terrno = TSDB_CODE_MND_QNODE_ALREADY_EXIST; - mndReleaseQnode(pMnode, pObj); - return -1; + goto CREATE_QNODE_OVER; } else if (terrno != TSDB_CODE_MND_QNODE_NOT_EXIST) { - mError("qnode:%d, failed to create qnode since %s", pCreate->dnodeId, terrstr()); - return -1; + goto CREATE_QNODE_OVER; } - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); if (pDnode == NULL) { - mError("qnode:%d, dnode not exist", pCreate->dnodeId); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - return -1; + goto CREATE_QNODE_OVER; } - int32_t code = mndCreateQnode(pMnode, pReq, pDnode, pCreate); + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto CREATE_QNODE_OVER; + } + + if (mndCheckNodeAuth(pUser)) { + goto CREATE_QNODE_OVER; + } + + code = mndCreateQnode(pMnode, pReq, pDnode, &createReq); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_QNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("qnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); + } + + mndReleaseQnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); + mndReleaseUser(pMnode, pUser); - if (code != 0) { - mError("qnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); - return -1; - } - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return code; } static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { @@ -352,33 +371,51 @@ DROP_QNODE_OVER: } static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMDropQnodeReq *pDrop = pReq->rpcMsg.pCont; - pDrop->dnodeId = htonl(pDrop->dnodeId); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SQnodeObj *pObj = NULL; + SMDropQnodeReq dropReq = {0}; - mDebug("qnode:%d, start to drop", pDrop->dnodeId); + if (tDeserializeSMCreateDropQSBNodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_QNODE_OVER; + } - if (pDrop->dnodeId <= 0) { + mDebug("qnode:%d, start to drop", dropReq.dnodeId); + + if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_SDB_APP_ERROR; - mError("qnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + goto DROP_QNODE_OVER; } - SQnodeObj *pObj = mndAcquireQnode(pMnode, pDrop->dnodeId); + pObj = mndAcquireQnode(pMnode, dropReq.dnodeId); if (pObj == NULL) { - mError("qnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + goto DROP_QNODE_OVER; } - int32_t code = mndDropQnode(pMnode, pReq, pObj); - if (code != 0) { - sdbRelease(pMnode->pSdb, pObj); + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto DROP_QNODE_OVER; + } + + if (mndCheckNodeAuth(pUser)) { + goto DROP_QNODE_OVER; + } + + code = mndDropQnode(pMnode, pReq, pObj); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_QNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); - return -1; } - sdbRelease(pMnode->pSdb, pObj); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + mndReleaseQnode(pMnode, pObj); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index e2ddcee0e9..3b470fc769 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -118,27 +118,28 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { static int32_t mndProcessShowReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowReq *pShowReq = pReq->rpcMsg.pCont; - int8_t type = pShowReq->type; - int16_t payloadLen = htonl(pShowReq->payloadLen); + int32_t code = -1; + SShowReq showReq = {0}; - if (type <= TSDB_MGMT_TABLE_START || type >= TSDB_MGMT_TABLE_MAX) { - terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show-meta req since %s", terrstr()); - return -1; + if (tDeserializeSShowReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &showReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto SHOW_OVER; } - ShowMetaFp metaFp = pMgmt->metaFps[type]; + if (showReq.type <= TSDB_MGMT_TABLE_START || showReq.type >= TSDB_MGMT_TABLE_MAX) { + terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; + goto SHOW_OVER; + } + + ShowMetaFp metaFp = pMgmt->metaFps[showReq.type]; if (metaFp == NULL) { terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show-meta req:%s since %s", mndShowStr(type), terrstr()); - return -1; + goto SHOW_OVER; } - SShowObj *pShow = mndCreateShowObj(pMnode, pShowReq); + SShowObj *pShow = mndCreateShowObj(pMnode, &showReq); if (pShow == NULL) { - mError("failed to process show-meta req:%s since %s", mndShowStr(type), terrstr()); - return -1; + goto SHOW_OVER; } int32_t size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; @@ -146,26 +147,30 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) { if (pRsp == NULL) { mndReleaseShowObj(pShow, true); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:0x%" PRIx64 ", failed to process show-meta req:%s since malloc rsp error", pShow->id, - mndShowStr(type)); - return -1; + goto SHOW_OVER; } - int32_t code = (*metaFp)(pReq, pShow, &pRsp->tableMeta); - mDebug("show:0x%" PRIx64 ", get meta finished, numOfRows:%d cols:%d type:%s, result:%s", pShow->id, pShow->numOfRows, - pShow->numOfColumns, mndShowStr(type), tstrerror(code)); + code = (*metaFp)(pReq, pShow, &pRsp->tableMeta); + mDebug("show:0x%" PRIx64 ", get meta finished, numOfRows:%d cols:%d showReq.type:%s, result:%s", pShow->id, + pShow->numOfRows, pShow->numOfColumns, mndShowStr(showReq.type), tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { pReq->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; pReq->pCont = pRsp; pRsp->showId = htobe64(pShow->id); mndReleaseShowObj(pShow, false); - return TSDB_CODE_SUCCESS; } else { rpcFreeCont(pRsp); mndReleaseShowObj(pShow, true); - return code; } + +SHOW_OVER: + if (code != 0) { + mError("failed to process show-meta req since %s", terrstr()); + } + + tFreeSShowReq(&showReq); + return code; } static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { @@ -175,10 +180,13 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { int32_t size = 0; int32_t rowsRead = 0; - SRetrieveTableReq *pRetrieve = pReq->rpcMsg.pCont; - int64_t showId = htobe64(pRetrieve->showId); + SRetrieveTableReq retrieveReq = {0}; + if (tDeserializeSRetrieveTableReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &retrieveReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - SShowObj *pShow = mndAcquireShowObj(pMnode, showId); + SShowObj *pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); if (pShow == NULL) { terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); @@ -202,7 +210,7 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { pShow->numOfReads = pShow->numOfRows; } - if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { rowsToRead = pShow->numOfRows - pShow->numOfReads; } @@ -226,7 +234,7 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { } // if free flag is set, client wants to clean the resources - if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { rowsRead = (*retrieveFp)(pReq, pShow, pRsp->data, rowsToRead); } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index cbba659719..dd699364c0 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -15,9 +15,11 @@ #define _DEFAULT_SOURCE #include "mndSnode.h" +#include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" +#include "mndUser.h" #define TSDB_SNODE_VER_NUMBER 1 #define TSDB_SNODE_RESERVE_SIZE 64 @@ -258,40 +260,58 @@ CREATE_SNODE_OVER: } static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMCreateSnodeReq *pCreate = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SSnodeObj *pObj = NULL; + SDnodeObj *pDnode = NULL; + SUserObj *pUser = NULL; + SMCreateSnodeReq createReq = {0}; - pCreate->dnodeId = htonl(pCreate->dnodeId); + if (tDeserializeSMCreateDropQSBNodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_SNODE_OVER; + } - mDebug("snode:%d, start to create", pCreate->dnodeId); + mDebug("snode:%d, start to create", createReq.dnodeId); - SSnodeObj *pObj = mndAcquireSnode(pMnode, pCreate->dnodeId); + pObj = mndAcquireSnode(pMnode, createReq.dnodeId); if (pObj != NULL) { - mError("snode:%d, snode already exist", pObj->id); terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; - mndReleaseSnode(pMnode, pObj); - return -1; + goto CREATE_SNODE_OVER; } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) { - mError("snode:%d, failed to create snode since %s", pCreate->dnodeId, terrstr()); - return -1; + goto CREATE_SNODE_OVER; } - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); if (pDnode == NULL) { - mError("snode:%d, dnode not exist", pCreate->dnodeId); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + goto CREATE_SNODE_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto CREATE_SNODE_OVER; + } + + if (mndCheckNodeAuth(pUser)) { + goto CREATE_SNODE_OVER; + } + + code = mndCreateSnode(pMnode, pReq, pDnode, &createReq); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_SNODE_OVER: + if(code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr()); return -1; } - int32_t code = mndCreateSnode(pMnode, pReq, pDnode, pCreate); + mndReleaseSnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); + mndReleaseUser(pMnode, pUser); - if (code != 0) { - mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); - return -1; - } - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return code; } static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { @@ -354,33 +374,51 @@ DROP_SNODE_OVER: } static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMDropSnodeReq *pDrop = pReq->rpcMsg.pCont; - pDrop->dnodeId = htonl(pDrop->dnodeId); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SSnodeObj *pObj = NULL; + SMDropSnodeReq dropReq = {0}; - mDebug("snode:%d, start to drop", pDrop->dnodeId); + if (tDeserializeSMCreateDropQSBNodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_SNODE_OVER; + } - if (pDrop->dnodeId <= 0) { + mDebug("snode:%d, start to drop", dropReq.dnodeId); + + if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_SDB_APP_ERROR; - mError("snode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + goto DROP_SNODE_OVER; } - SSnodeObj *pObj = mndAcquireSnode(pMnode, pDrop->dnodeId); + pObj = mndAcquireSnode(pMnode, dropReq.dnodeId); if (pObj == NULL) { - mError("snode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + goto DROP_SNODE_OVER; } - int32_t code = mndDropSnode(pMnode, pReq, pObj); - if (code != 0) { - sdbRelease(pMnode->pSdb, pObj); + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto DROP_SNODE_OVER; + } + + if (mndCheckNodeAuth(pUser)) { + goto DROP_SNODE_OVER; + } + + code = mndDropSnode(pMnode, pReq, pObj); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_SNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); - return -1; } - sdbRelease(pMnode->pSdb, pObj); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + mndReleaseSnode(pMnode, pObj); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9b0bad9827..344eab38b9 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1523,4 +1523,4 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index c5bebb3249..2a3e0008a2 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -55,7 +55,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg); static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, - const SMqConsumerEp *pSub); + const SMqConsumerEp *pConsumerEp); static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp); @@ -107,9 +107,9 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) { SMqMVRebReq req = { - .vgId = pConsumerEp->vgId, - .oldConsumerId = pConsumerEp->oldConsumerId, - .newConsumerId = pConsumerEp->consumerId, + .vgId = pConsumerEp->vgId, + .oldConsumerId = pConsumerEp->oldConsumerId, + .newConsumerId = pConsumerEp->consumerId, }; int32_t tlen = tEncodeSMqMVRebReq(NULL, &req); @@ -133,6 +133,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume } static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp) { + ASSERT(pConsumerEp->oldConsumerId != -1); int32_t vgId = pConsumerEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); @@ -146,7 +147,7 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC action.epSet = mndGetVgroupEpset(pMnode, pVgObj); action.pCont = buf; action.contLen = sizeof(SMsgHead) + tlen; - action.msgType = TDMT_VND_MQ_SET_CONN; + action.msgType = TDMT_VND_MQ_REB; mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -159,8 +160,7 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) { SMqSetCVgReq req = {0}; - req.oldConsumerId = pConsumerEp->consumerId; - req.newConsumerId = -1; + req.consumerId = pConsumerEp->consumerId; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); void *buf = malloc(sizeof(SMsgHead) + tlen); @@ -218,9 +218,9 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { } ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); - //TODO + // TODO int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus); - mInfo("try to get sub ep, old val: %d", hbStatus); + mTrace("try to get sub ep, old val: %d", hbStatus); atomic_store_32(&pConsumer->hbStatus, 0); /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ @@ -230,7 +230,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { rsp.consumerId = consumerId; rsp.epoch = pConsumer->epoch; if (epoch != rsp.epoch) { - mInfo("old epoch %d, new epoch %d", epoch, rsp.epoch); + mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, rsp.epoch); SArray *pTopics = pConsumer->currentTopics; int sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -311,7 +311,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { while (1) { pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); if (pIter == NULL) break; - int32_t hbStatus = atomic_fetch_add_32(&pConsumer->hbStatus, 1); + int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); if (hbStatus > MND_SUBSCRIBE_REBALANCE_CNT) { int32_t old = atomic_val_compare_exchange_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE, MQ_CONSUMER_STATUS__LOST); @@ -324,10 +324,6 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->lostConsumers, &pConsumer->consumerId); } - /*pRebMsg->consumerId = pConsumer->consumerId;*/ - /*SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_DO_REBALANCE, .pCont = pRebMsg, .contLen = - * sizeof(SMqDoRebalanceMsg)};*/ - /*pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);*/ } } int32_t status = atomic_load_32(&pConsumer->status); @@ -402,68 +398,86 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { int32_t vgEachConsumer = vgNum / consumerNum; int32_t imbalanceVg = vgNum % consumerNum; int32_t imbalanceSolved = 0; - /*SArray *unassignedVgStash = taosArrayInit(0, sizeof(SMqConsumerEp));*/ - /*SArray *unassignedConsumerIdx = taosArrayInit(0, sizeof(int32_t));*/ // iterate all consumers, set unassignedVgStash for (int i = 0; i < consumerNum; i++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); - int vgThisConsumerAfterRb; - if (i < imbalanceVg) vgThisConsumerAfterRb = vgEachConsumer + 1; - else vgThisConsumerAfterRb = vgEachConsumer; + int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); + int vgThisConsumerAfterRb; + if (i < imbalanceVg) + vgThisConsumerAfterRb = vgEachConsumer + 1; + else + vgThisConsumerAfterRb = vgEachConsumer; - mInfo("mq consumer:%ld, connectted vgroup change from %d %d", pSubConsumer->consumerId, vgThisConsumerBeforeRb, vgThisConsumerAfterRb); + mInfo("mq consumer:%ld, connectted vgroup number change from %d to %d", pSubConsumer->consumerId, + vgThisConsumerBeforeRb, vgThisConsumerAfterRb); - while(taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) { + while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) { SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo); ASSERT(pConsumerEp != NULL); ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId); taosArrayPush(pSub->unassignedVg, pConsumerEp); } - SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId); - int32_t status = atomic_load_32(&pRebConsumer->status); - if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || - (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || - (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST) - ) { - pRebConsumer->epoch++; - if (vgThisConsumerAfterRb != 0) { - atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE); - } else { - atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE); - } - - mInfo("mq consumer:%ld, status change from %d %d", pRebConsumer->consumerId, status, pRebConsumer->status); - - SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer); - sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); - mndTransAppendRedolog(pTrans, pConsumerRaw); + SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId); + int32_t status = atomic_load_32(&pRebConsumer->status); + if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || + (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || + (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)) { + pRebConsumer->epoch++; + if (vgThisConsumerAfterRb != 0) { + atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE); + } else { + atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE); } - mndReleaseConsumer(pMnode, pRebConsumer); + + mInfo("mq consumer:%ld, status change from %d to %d", pRebConsumer->consumerId, status, pRebConsumer->status); + + SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer); + sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); + mndTransAppendRedolog(pTrans, pConsumerRaw); + } + mndReleaseConsumer(pMnode, pRebConsumer); } - //assign to vgroup + // assign to vgroup if (taosArrayGetSize(pSub->unassignedVg) != 0) { for (int i = 0; i < consumerNum; i++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); - int vgThisConsumerAfterRb; - if (i < imbalanceVg) vgThisConsumerAfterRb = vgEachConsumer + 1; - else vgThisConsumerAfterRb = vgEachConsumer; + int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); + int vgThisConsumerAfterRb; + if (i < imbalanceVg) + vgThisConsumerAfterRb = vgEachConsumer + 1; + else + vgThisConsumerAfterRb = vgEachConsumer; - while(taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) { - SMqConsumerEp* pConsumerEp = taosArrayPop(pSub->unassignedVg); + while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) { + SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg); ASSERT(pConsumerEp != NULL); pConsumerEp->oldConsumerId = pConsumerEp->consumerId; pConsumerEp->consumerId = pSubConsumer->consumerId; taosArrayPush(pSubConsumer->vgInfo, pConsumerEp); - mInfo("mq consumer:%ld , assign vgroup %d, previously assigned to consumer %ld", pSubConsumer->consumerId, pConsumerEp->vgId, pConsumerEp->oldConsumerId); + if (pConsumerEp->oldConsumerId == -1) { + char *topic; + char *cgroup; + mndSplitSubscribeKey(pSub->key, &topic, &cgroup); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); + mInfo("mq set conn: assign vgroup %d of topic %s to consumer %ld", pConsumerEp->vgId, topic, + pConsumerEp->consumerId); + + mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); + mndReleaseTopic(pMnode, pTopic); + free(topic); + free(cgroup); + } else { + mInfo("mq rebalance: assign vgroup %d, from consumer %ld to consumer %ld", pConsumerEp->vgId, + pConsumerEp->oldConsumerId, pConsumerEp->consumerId); + + mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); + } } } } @@ -478,10 +492,12 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { } if (mndTransPrepare(pMnode, pTrans) != 0) { mError("mq-rebalance-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + taosHashCleanup(pReq->rebSubHash); mndTransDrop(pTrans); return -1; } + taosHashCleanup(pReq->rebSubHash); mndTransDrop(pTrans); return 0; } @@ -707,11 +723,6 @@ static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSub mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql); return -1; } - /*if (pArray && taosArrayGetSize(pArray) != 1) {*/ - /*terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;*/ - /*mError("unsupport topic: %s, sql: %s, plan level: %ld", pTopic->name, pTopic->sql, taosArrayGetSize(pArray));*/ - /*return -1;*/ - /*}*/ SMqConsumerEp consumerEp = {0}; consumerEp.status = 0; @@ -733,13 +744,13 @@ static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSub static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, const SMqConsumerEp *pConsumerEp) { + ASSERT(pConsumerEp->oldConsumerId == -1); int32_t vgId = pConsumerEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); SMqSetCVgReq req = { .vgId = vgId, - .oldConsumerId = pConsumerEp->oldConsumerId, - .newConsumerId = pConsumerEp->consumerId, + .consumerId = pConsumerEp->consumerId, .sql = pTopic->sql, .logicalPlan = pTopic->logicalPlan, .physicalPlan = pTopic->physicalPlan, @@ -892,7 +903,7 @@ SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *cgroup, const c SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); free(key); if (pSub == NULL) { - /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ + terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; } return pSub; } @@ -901,7 +912,7 @@ SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key) { SSdb *pSdb = pMnode->pSdb; SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); if (pSub == NULL) { - /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ + terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; } return pSub; } @@ -1030,8 +1041,13 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { pConsumerEp->oldConsumerId = pConsumerEp->consumerId; pConsumerEp->consumerId = consumerId; taosArrayPush(mqSubConsumer.vgInfo, pConsumerEp); - mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); - atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE); + if (pConsumerEp->oldConsumerId == -1) { + mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); + } else { + mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); + } + // do not set status active to trigger rebalance + /*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/ } SSdbRaw *pRaw = mndSubActionEncode(pSub); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index ea3cc7c2c6..d9444bbdbd 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -60,8 +60,8 @@ void mndCleanupTopic(SMnode *pMnode) {} SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1; - int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; + int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1; + int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; int32_t size = sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + MND_TOPIC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); if (pRaw == NULL) goto TOPIC_ENCODE_OVER; @@ -127,7 +127,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char)); - SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); pTopic->logicalPlan = calloc(len + 1, sizeof(char)); @@ -248,12 +248,21 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj); if (pTopicRaw == NULL) return -1; if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1; + /*STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);*/ + /*mndTransAppendRedolog(pTrans, pTopicRaw);*/ + /*if (mndTransPrepare(pMnode, pTrans) != 0) {*/ + /*mError("mq-createTopic-trans:%d, failed to prepare since %s", pTrans->id, terrstr());*/ + /*mndTransDrop(pTrans);*/ + /*return -1;*/ + /*}*/ + /*mndTransDrop(pTrans);*/ + /*return 0;*/ return sdbWrite(pMnode->pSdb, pTopicRaw); } static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - char *msgStr = pMsg->rpcMsg.pCont; + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; SCMCreateTopicReq createTopicReq = {0}; tDeserializeSCMCreateTopicReq(msgStr, &createTopicReq); @@ -288,13 +297,13 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { int32_t code = mndCreateTopic(pMnode, pMsg, &createTopicReq, pDb); mndReleaseDb(pMnode, pDb); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { terrno = code; mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); return -1; } - return TSDB_CODE_SUCCESS; + return 0; } static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, SMqTopicObj *pTopic) { return 0; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6686c0887f..3230074add 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -542,12 +542,18 @@ static void mndTransSendRpcRsp(STrans *pTrans) { } if (sendRsp && pTrans->rpcHandle != NULL) { + void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); + if (rpcCont != NULL) { + memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); + } + free(pTrans->rpcRsp); + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, .ahandle = pTrans->rpcAHandle, - .pCont = pTrans->rpcRsp, + .pCont = rpcCont, .contLen = pTrans->rpcRspLen}; rpcSendResponse(&rspMsg); pTrans->rpcHandle = NULL; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index b54200f114..5c290c9c4f 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -302,7 +302,7 @@ static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq) { SUserObj *pOperUser = NULL; SCreateUserReq createReq = {0}; - if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, &createReq) == NULL) goto CREATE_USER_OVER; + if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) goto CREATE_USER_OVER; mDebug("user:%s, start to create", createReq.user); @@ -402,7 +402,7 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) { SUserObj newUser = {0}; SAlterUserReq alterReq = {0}; - if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, &alterReq) == NULL) goto ALTER_USER_OVER; + if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) goto ALTER_USER_OVER; mDebug("user:%s, start to alter", alterReq.user); @@ -537,7 +537,7 @@ static int32_t mndProcessDropUserReq(SMnodeMsg *pReq) { SUserObj *pOperUser = NULL; SDropUserReq dropReq = {0}; - if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, &dropReq) == NULL) goto DROP_USER_OVER; + if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) goto DROP_USER_OVER; mDebug("user:%s, start to drop", dropReq.user); @@ -583,7 +583,7 @@ static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) { SGetUserAuthReq authReq = {0}; SGetUserAuthRsp authRsp = {0}; - if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, &authReq) == NULL) goto GET_AUTH_OVER; + if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) goto GET_AUTH_OVER; mTrace("user:%s, start to get auth", authReq.user); @@ -614,15 +614,14 @@ static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) { sdbRelease(pSdb, pDb); } - int32_t contLen = tSerializeSGetUserAuthRsp(NULL, &authRsp); + int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto GET_AUTH_OVER; } - void *pBuf = pRsp; - tSerializeSGetUserAuthRsp(&pBuf, &authRsp); + tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); pReq->pCont = pRsp; pReq->contLen = contLen; diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 315f23f798..2bc4f40614 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -56,10 +56,13 @@ TEST_F(MndTestAcct, 03_Drop_Acct) { } TEST_F(MndTestAcct, 04_Show_Acct) { - int32_t contLen = sizeof(SShowReq); + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_ACCT; - SShowReq* pReq = (SShowReq*)rpcMallocCont(contLen); - pReq->type = TSDB_MGMT_TABLE_ACCT; + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/bnode/bnode.cpp b/source/dnode/mnode/impl/test/bnode/bnode.cpp index 0b54a9bf4a..d7d15df35a 100644 --- a/source/dnode/mnode/impl/test/bnode/bnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/bnode.cpp @@ -52,10 +52,12 @@ TEST_F(MndTestBnode, 01_Show_Bnode) { TEST_F(MndTestBnode, 02_Create_Bnode) { { - int32_t contLen = sizeof(SMCreateBnodeReq); + SMCreateBnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -63,10 +65,12 @@ TEST_F(MndTestBnode, 02_Create_Bnode) { } { - int32_t contLen = sizeof(SMCreateBnodeReq); + SMCreateBnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -83,10 +87,12 @@ TEST_F(MndTestBnode, 02_Create_Bnode) { } { - int32_t contLen = sizeof(SMCreateBnodeReq); + SMCreateBnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -96,11 +102,13 @@ TEST_F(MndTestBnode, 02_Create_Bnode) { TEST_F(MndTestBnode, 03_Drop_Bnode) { { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9019; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9019); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -113,10 +121,12 @@ TEST_F(MndTestBnode, 03_Drop_Bnode) { } { - int32_t contLen = sizeof(SMCreateBnodeReq); + SMCreateBnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -135,10 +145,12 @@ TEST_F(MndTestBnode, 03_Drop_Bnode) { } { - int32_t contLen = sizeof(SMDropBnodeReq); + SMDropBnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -154,10 +166,12 @@ TEST_F(MndTestBnode, 03_Drop_Bnode) { } { - int32_t contLen = sizeof(SMDropBnodeReq); + SMDropBnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -168,10 +182,12 @@ TEST_F(MndTestBnode, 03_Drop_Bnode) { TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMCreateBnodeReq); + SMCreateBnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); @@ -181,10 +197,12 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { { // continue send message, bnode is creating - int32_t contLen = sizeof(SMCreateBnodeReq); + SMCreateBnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -193,10 +211,12 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { { // continue send message, bnode is creating - int32_t contLen = sizeof(SMDropBnodeReq); + SMDropBnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -212,10 +232,12 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateBnodeReq); + SMCreateBnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -230,10 +252,12 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMDropBnodeReq); + SMDropBnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); @@ -243,10 +267,12 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { { // continue send message, bnode is dropping - int32_t contLen = sizeof(SMCreateBnodeReq); + SMCreateBnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -255,10 +281,12 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { { // continue send message, bnode is dropping - int32_t contLen = sizeof(SMDropBnodeReq); + SMDropBnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -274,10 +302,12 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateBnodeReq); + SMCreateBnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 2d1574467e..a3efad8aa2 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -53,29 +53,31 @@ TEST_F(MndTestDb, 01_ShowDb) { TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d1"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -125,18 +127,20 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("master", 9); { - int32_t contLen = sizeof(SAlterDbReq); + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d1"); + alterdbReq.totalBlocks = 12; + alterdbReq.daysToKeep0 = 300; + alterdbReq.daysToKeep1 = 400; + alterdbReq.daysToKeep2 = 500; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.quorum = 2; + alterdbReq.cacheLastRow = 1; - SAlterDbReq* pReq = (SAlterDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); - pReq->totalBlocks = htonl(12); - pReq->daysToKeep0 = htonl(300); - pReq->daysToKeep1 = htonl(400); - pReq->daysToKeep2 = htonl(500); - pReq->fsyncPeriod = htonl(4000); - pReq->walLevel = 2; - pReq->quorum = 2; - pReq->cacheLastRow = 1; + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -194,18 +198,20 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { CheckInt8(0); // update { - int32_t contLen = sizeof(SDropDbReq); + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, "1.d1"); - SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SDropDbRsp* pDrop = (SDropDbRsp*)pRsp->pCont; - pDrop->uid = htobe64(pDrop->uid); - EXPECT_STREQ(pDrop->db, "1.d1"); + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pRsp->pCont, pRsp->contLen, &dropdbRsp); + EXPECT_STREQ(dropdbRsp.db, "1.d1"); } test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); @@ -217,29 +223,31 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -256,73 +264,74 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { uint64_t d2_uid = 0; { - int32_t contLen = sizeof(SUseDbReq); + SUseDbReq usedbReq = {0}; + strcpy(usedbReq.db, "1.d2"); + usedbReq.vgVersion = -1; - SUseDbReq* pReq = (SUseDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); - pReq->vgVersion = htonl(-1); + int32_t contLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSUseDbReq(pReq, contLen, &usedbReq); SRpcMsg* pMsg = test.SendReq(TDMT_MND_USE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - SUseDbRsp* pRsp = (SUseDbRsp*)pMsg->pCont; - EXPECT_STREQ(pRsp->db, "1.d2"); - pRsp->uid = htobe64(pRsp->uid); - d2_uid = pRsp->uid; - pRsp->vgVersion = htonl(pRsp->vgVersion); - pRsp->vgNum = htonl(pRsp->vgNum); - pRsp->hashMethod = pRsp->hashMethod; - EXPECT_EQ(pRsp->vgVersion, 1); - EXPECT_EQ(pRsp->vgNum, 2); - EXPECT_EQ(pRsp->hashMethod, 1); + SUseDbRsp usedbRsp = {0}; + tDeserializeSUseDbRsp(pMsg->pCont, pMsg->contLen, &usedbRsp); + EXPECT_STREQ(usedbRsp.db, "1.d2"); + EXPECT_EQ(usedbRsp.vgVersion, 1); + EXPECT_EQ(usedbRsp.vgNum, 2); + EXPECT_EQ(usedbRsp.hashMethod, 1); + d2_uid = usedbRsp.uid; { - SVgroupInfo* pInfo = &pRsp->vgroupInfo[0]; - pInfo->vgId = htonl(pInfo->vgId); - pInfo->hashBegin = htonl(pInfo->hashBegin); - pInfo->hashEnd = htonl(pInfo->hashEnd); + SVgroupInfo* pInfo = (SVgroupInfo*)taosArrayGet(usedbRsp.pVgroupInfos, 0); + pInfo->vgId = pInfo->vgId; + pInfo->hashBegin = pInfo->hashBegin; + pInfo->hashEnd = pInfo->hashEnd; EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, 0); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); EXPECT_EQ(pInfo->epset.inUse, 0); EXPECT_EQ(pInfo->epset.numOfEps, 1); SEp* pAddr = &pInfo->epset.eps[0]; - pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } { - SVgroupInfo* pInfo = &pRsp->vgroupInfo[1]; - pInfo->vgId = htonl(pInfo->vgId); - pInfo->hashBegin = htonl(pInfo->hashBegin); - pInfo->hashEnd = htonl(pInfo->hashEnd); + SVgroupInfo* pInfo = (SVgroupInfo*)taosArrayGet(usedbRsp.pVgroupInfos, 1); + pInfo->vgId = pInfo->vgId; + pInfo->hashBegin = pInfo->hashBegin; + pInfo->hashEnd = pInfo->hashEnd; EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); EXPECT_EQ(pInfo->epset.inUse, 0); EXPECT_EQ(pInfo->epset.numOfEps, 1); SEp* pAddr = &pInfo->epset.eps[0]; - pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } + + tFreeSUsedbRsp(&usedbRsp); } { - int32_t contLen = sizeof(SDropDbReq); + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, "1.d2"); - SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SDropDbRsp* pDrop = (SDropDbRsp*)pRsp->pCont; - pDrop->uid = htobe64(pDrop->uid); - EXPECT_STREQ(pDrop->db, "1.d2"); - EXPECT_EQ(pDrop->uid, d2_uid); + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pRsp->pCont, pRsp->contLen, &dropdbRsp); + EXPECT_STREQ(dropdbRsp.db, "1.d2"); + EXPECT_EQ(dropdbRsp.uid, d2_uid); } } diff --git a/source/dnode/mnode/impl/test/dnode/dnode.cpp b/source/dnode/mnode/impl/test/dnode/dnode.cpp index 1c0cfb7bfc..f575556345 100644 --- a/source/dnode/mnode/impl/test/dnode/dnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/dnode.cpp @@ -75,11 +75,14 @@ TEST_F(MndTestDnode, 01_ShowDnode) { } TEST_F(MndTestDnode, 02_ConfigDnode) { - int32_t contLen = sizeof(SMCfgDnodeReq); + SMCfgDnodeReq cfgReq = {0}; + cfgReq.dnodeId = 1; + strcpy(cfgReq.config, "ddebugflag"); + strcpy(cfgReq.value, "131"); - SMCfgDnodeReq* pReq = (SMCfgDnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); - strcpy(pReq->config, "ddebugflag 131"); + int32_t contLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCfgDnodeReq(pReq, contLen, &cfgReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONFIG_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -88,11 +91,13 @@ TEST_F(MndTestDnode, 02_ConfigDnode) { TEST_F(MndTestDnode, 03_Create_Dnode) { { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, ""); + createReq.port = 9024; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, ""); - pReq->port = htonl(9024); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -100,11 +105,13 @@ TEST_F(MndTestDnode, 03_Create_Dnode) { } { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = -1; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(-1); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -112,11 +119,13 @@ TEST_F(MndTestDnode, 03_Create_Dnode) { } { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 123456; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(123456); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -124,11 +133,13 @@ TEST_F(MndTestDnode, 03_Create_Dnode) { } { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9024; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9024); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -136,11 +147,13 @@ TEST_F(MndTestDnode, 03_Create_Dnode) { } { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9024; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9024); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -172,10 +185,12 @@ TEST_F(MndTestDnode, 03_Create_Dnode) { TEST_F(MndTestDnode, 04_Drop_Dnode) { { - int32_t contLen = sizeof(SDropDnodeReq); + SDropDnodeReq dropReq = {0}; + dropReq.dnodeId = -3; - SDropDnodeReq* pReq = (SDropDnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(-3); + int32_t contLen = tSerializeSDropDnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -183,10 +198,12 @@ TEST_F(MndTestDnode, 04_Drop_Dnode) { } { - int32_t contLen = sizeof(SDropDnodeReq); + SDropDnodeReq dropReq = {0}; + dropReq.dnodeId = 5; - SDropDnodeReq* pReq = (SDropDnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(5); + int32_t contLen = tSerializeSDropDnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -194,10 +211,12 @@ TEST_F(MndTestDnode, 04_Drop_Dnode) { } { - int32_t contLen = sizeof(SDropDnodeReq); + SDropDnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SDropDnodeReq* pReq = (SDropDnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSDropDnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -205,10 +224,12 @@ TEST_F(MndTestDnode, 04_Drop_Dnode) { } { - int32_t contLen = sizeof(SDropDnodeReq); + SDropDnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SDropDnodeReq* pReq = (SDropDnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSDropDnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -235,11 +256,13 @@ TEST_F(MndTestDnode, 04_Drop_Dnode) { TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9025; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9025); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -247,11 +270,13 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { } { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9026; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9026); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -259,11 +284,13 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { } { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9027; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9027); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 9d31ec7314..f34f77de0c 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -43,123 +43,147 @@ TEST_F(MndTestFunc, 01_Show_Func) { TEST_F(MndTestFunc, 02_Create_Func) { { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, ""); + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, ""); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME); } { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); } { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN + 1; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN + 1); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); } { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); } { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN + 1; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); - pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN - 1); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); } { - int32_t contLen = sizeof(SCreateFuncReq) + 24; + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); - pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); - pReq->pCont[0] = 0; SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); } { - int32_t contLen = sizeof(SCreateFuncReq) + 24; + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN; + createReq.pCode[0] = 'a'; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); - pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); - pReq->pCont[0] = 'a'; SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); } { - int32_t contLen = sizeof(SCreateFuncReq) + 24; + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN; + createReq.pCode[0] = 'a'; + createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); - pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); - pReq->pCont[0] = 'a'; - pReq->bufSize = htonl(TSDB_FUNC_BUF_SIZE + 1); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); } for (int32_t i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SCreateFuncReq); - int32_t commentSize = TSDB_FUNC_COMMENT_LEN; - int32_t codeSize = TSDB_FUNC_CODE_LEN; - contLen = (contLen + codeSize + commentSize); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN; + createReq.pCode[0] = 'a'; + createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; + createReq.igExists = 0; + if (i == 2) createReq.igExists = 1; + createReq.funcType = 1; + createReq.scriptType = 2; + createReq.outputType = TSDB_DATA_TYPE_SMALLINT; + createReq.outputLen = 12; + createReq.bufSize = 4; + createReq.signature = 5; + for (int32_t i = 0; i < createReq.commentSize - 1; ++i) { + createReq.pComment[i] = 'm'; + } + for (int32_t i = 0; i < createReq.codeSize - 1; ++i) { + createReq.pCode[i] = 'd'; + } - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->igExists = 0; - if (i == 2) pReq->igExists = 1; - pReq->funcType = 1; - pReq->scriptType = 2; - pReq->outputType = TSDB_DATA_TYPE_SMALLINT; - pReq->outputLen = htonl(12); - pReq->bufSize = htonl(4); - pReq->signature = htobe64(5); - pReq->commentSize = htonl(commentSize); - pReq->codeSize = htonl(codeSize); - for (int32_t i = 0; i < commentSize - 1; ++i) { - pReq->pCont[i] = 'm'; - } - for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { - pReq->pCont[i] = 'd'; - } + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -187,27 +211,26 @@ TEST_F(MndTestFunc, 02_Create_Func) { TEST_F(MndTestFunc, 03_Retrieve_Func) { { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 1; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f1"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(1); - strcpy(pReq->pFuncNames, "f1"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; - pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp); + EXPECT_EQ(retrieveRsp.numOfFuncs, 1); + EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos)); - SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); - pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); - pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); - pFuncInfo->signature = htobe64(pFuncInfo->signature); - pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); - pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); EXPECT_STREQ(pFuncInfo->name, "f1"); EXPECT_EQ(pFuncInfo->funcType, 1); @@ -219,9 +242,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); - char* pComment = pFuncInfo->pCont; - char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; - char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { comments[i] = 'm'; } @@ -229,17 +250,20 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { codes[i] = 'd'; } - EXPECT_STREQ(pComment, comments); - EXPECT_STREQ(pCode, codes); + EXPECT_STREQ(comments, pFuncInfo->pComment); + EXPECT_STREQ(codes, pFuncInfo->pCode); + taosArrayDestroy(retrieveRsp.pFuncInfos); } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 0; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 0; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -247,12 +271,17 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1; + retrieveReq.pFuncNames = taosArrayInit(TSDB_FUNC_MAX_RETRIEVE + 1, TSDB_FUNC_NAME_LEN); + for (int32_t i = 0; i < TSDB_FUNC_MAX_RETRIEVE + 1; ++i) { + taosArrayPush(retrieveReq.pFuncNames, "1"); + } - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -260,13 +289,15 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 1; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f2"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); - strcpy(pReq->pFuncNames, "f2"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -274,28 +305,27 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { } { - int32_t contLen = sizeof(SCreateFuncReq); - int32_t commentSize = 1024; - int32_t codeSize = 9527; - contLen = (contLen + codeSize + commentSize); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f2"); + createReq.commentSize = 1024; + createReq.codeSize = 9527; + createReq.igExists = 1; + createReq.funcType = 2; + createReq.scriptType = 3; + createReq.outputType = TSDB_DATA_TYPE_BINARY; + createReq.outputLen = 24; + createReq.bufSize = 6; + createReq.signature = 18; + for (int32_t i = 0; i < createReq.commentSize - 1; ++i) { + createReq.pComment[i] = 'p'; + } + for (int32_t i = 0; i < createReq.codeSize - 1; ++i) { + createReq.pCode[i] = 'q'; + } - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f2"); - pReq->igExists = 1; - pReq->funcType = 2; - pReq->scriptType = 3; - pReq->outputType = TSDB_DATA_TYPE_BINARY; - pReq->outputLen = htonl(24); - pReq->bufSize = htonl(6); - pReq->signature = htobe64(18); - pReq->commentSize = htonl(commentSize); - pReq->codeSize = htonl(codeSize); - for (int32_t i = 0; i < commentSize - 1; ++i) { - pReq->pCont[i] = 'p'; - } - for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { - pReq->pCont[i] = 'q'; - } + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -309,27 +339,26 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 1; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f2"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(1); - strcpy(pReq->pFuncNames, "f2"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; - pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp); + EXPECT_EQ(retrieveRsp.numOfFuncs, 1); + EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos)); - SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); - pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); - pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); - pFuncInfo->signature = htobe64(pFuncInfo->signature); - pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); - pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); EXPECT_STREQ(pFuncInfo->name, "f2"); EXPECT_EQ(pFuncInfo->funcType, 2); @@ -341,47 +370,43 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->commentSize, 1024); EXPECT_EQ(pFuncInfo->codeSize, 9527); - char* pComment = pFuncInfo->pCont; - char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; - char* comments = (char*)calloc(1, 1024); + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; for (int32_t i = 0; i < 1024 - 1; ++i) { comments[i] = 'p'; } - char* codes = (char*)calloc(1, 9527); + char codes[TSDB_FUNC_CODE_LEN] = {0}; for (int32_t i = 0; i < 9527 - 1; ++i) { codes[i] = 'q'; } - EXPECT_STREQ(pComment, comments); - EXPECT_STREQ(pCode, codes); - free(comments); - free(codes); + + EXPECT_STREQ(comments, pFuncInfo->pComment); + EXPECT_STREQ(codes, pFuncInfo->pCode); + taosArrayDestroy(retrieveRsp.pFuncInfos); } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 2; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 2; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f2"); + taosArrayPush(retrieveReq.pFuncNames, "f1"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); - strcpy(pReq->pFuncNames, "f2"); - strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f1"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; - pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp); + EXPECT_EQ(retrieveRsp.numOfFuncs, 2); + EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos)); { - SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); - pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); - pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); - pFuncInfo->signature = htobe64(pFuncInfo->signature); - pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); - pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); - + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); EXPECT_STREQ(pFuncInfo->name, "f2"); EXPECT_EQ(pFuncInfo->funcType, 2); EXPECT_EQ(pFuncInfo->scriptType, 3); @@ -392,29 +417,21 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->commentSize, 1024); EXPECT_EQ(pFuncInfo->codeSize, 9527); - char* pComment = pFuncInfo->pCont; - char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; - char* comments = (char*)calloc(1, 1024); + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; for (int32_t i = 0; i < 1024 - 1; ++i) { comments[i] = 'p'; } - char* codes = (char*)calloc(1, 9527); + char codes[TSDB_FUNC_CODE_LEN] = {0}; for (int32_t i = 0; i < 9527 - 1; ++i) { codes[i] = 'q'; } - EXPECT_STREQ(pComment, comments); - EXPECT_STREQ(pCode, codes); - free(comments); - free(codes); + + EXPECT_STREQ(comments, pFuncInfo->pComment); + EXPECT_STREQ(codes, pFuncInfo->pCode); } { - SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos + sizeof(SFuncInfo) + 1024 + 9527); - pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); - pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); - pFuncInfo->signature = htobe64(pFuncInfo->signature); - pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); - pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 1); EXPECT_STREQ(pFuncInfo->name, "f1"); EXPECT_EQ(pFuncInfo->funcType, 1); EXPECT_EQ(pFuncInfo->scriptType, 2); @@ -425,9 +442,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); - char* pComment = pFuncInfo->pCont; - char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; - char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { comments[i] = 'm'; } @@ -435,33 +450,38 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { codes[i] = 'd'; } - EXPECT_STREQ(pComment, comments); - EXPECT_STREQ(pCode, codes); + EXPECT_STREQ(comments, pFuncInfo->pComment); + EXPECT_STREQ(codes, pFuncInfo->pCode); } + taosArrayDestroy(retrieveRsp.pFuncInfos); } - { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 2; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + { + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 2; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f2"); + taosArrayPush(retrieveReq.pFuncNames, "f3"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); - strcpy(pReq->pFuncNames, "f2"); - strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f3"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); - } + } } TEST_F(MndTestFunc, 04_Drop_Func) { { - int32_t contLen = sizeof(SDropFuncReq); + SDropFuncReq dropReq = {0}; + strcpy(dropReq.name, ""); - SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, ""); + int32_t contLen = tSerializeSDropFuncReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropFuncReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -469,10 +489,12 @@ TEST_F(MndTestFunc, 04_Drop_Func) { } { - int32_t contLen = sizeof(SDropFuncReq); + SDropFuncReq dropReq = {0}; + strcpy(dropReq.name, "f3"); - SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f3"); + int32_t contLen = tSerializeSDropFuncReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropFuncReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -480,11 +502,13 @@ TEST_F(MndTestFunc, 04_Drop_Func) { } { - int32_t contLen = sizeof(SDropFuncReq); + SDropFuncReq dropReq = {0}; + strcpy(dropReq.name, "f3"); + dropReq.igNotExists = 1; - SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f3"); - pReq->igNotExists = 1; + int32_t contLen = tSerializeSDropFuncReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropFuncReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -492,10 +516,13 @@ TEST_F(MndTestFunc, 04_Drop_Func) { } { - int32_t contLen = sizeof(SDropFuncReq); + SDropFuncReq dropReq = {0}; + strcpy(dropReq.name, "f1"); + dropReq.igNotExists = 1; - SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); + int32_t contLen = tSerializeSDropFuncReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropFuncReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -517,5 +544,5 @@ TEST_F(MndTestFunc, 04_Drop_Func) { test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); - CheckBinary("f2", TSDB_FUNC_NAME_LEN); + CheckBinary("f2", TSDB_FUNC_NAME_LEN); } diff --git a/source/dnode/mnode/impl/test/mnode/mnode.cpp b/source/dnode/mnode/impl/test/mnode/mnode.cpp index c69c86eb1c..fd8c83e26a 100644 --- a/source/dnode/mnode/impl/test/mnode/mnode.cpp +++ b/source/dnode/mnode/impl/test/mnode/mnode.cpp @@ -60,10 +60,12 @@ TEST_F(MndTestMnode, 01_ShowDnode) { TEST_F(MndTestMnode, 02_Create_Mnode_Invalid_Id) { { - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -73,10 +75,12 @@ TEST_F(MndTestMnode, 02_Create_Mnode_Invalid_Id) { TEST_F(MndTestMnode, 03_Create_Mnode_Invalid_Id) { { - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -87,11 +91,13 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Invalid_Id) { TEST_F(MndTestMnode, 04_Create_Mnode) { { // create dnode - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9029; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9029); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -105,10 +111,12 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { { // create mnode - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -132,10 +140,12 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { { // drop mnode - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -154,10 +164,12 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { { // drop mnode - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -168,10 +180,12 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -181,10 +195,12 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { { // continue send message, mnode is creating - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -193,10 +209,12 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { { // continue send message, mnode is creating - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -212,10 +230,12 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -230,10 +250,12 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); @@ -243,10 +265,12 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { { // continue send message, mnode is dropping - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -255,10 +279,12 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { { // continue send message, mnode is dropping - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -274,10 +300,12 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index ccf13d5d66..5b7bcd47b8 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -96,35 +96,35 @@ TEST_F(MndTestProfile, 03_ConnectMsg_Show) { } TEST_F(MndTestProfile, 04_HeartBeatMsg) { - SClientHbBatchReq batchReq; + SClientHbBatchReq batchReq = {0}; batchReq.reqs = taosArrayInit(0, sizeof(SClientHbReq)); SClientHbReq req = {0}; req.connKey = {.connId = 123, .hbType = HEARTBEAT_TYPE_MQ}; req.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - SKv kv; + SKv kv = {0}; kv.key = 123; kv.value = (void*)"bcd"; kv.valueLen = 4; taosHashPut(req.info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); taosArrayPush(batchReq.reqs, &req); - int32_t tlen = tSerializeSClientHbBatchReq(NULL, &batchReq); - - void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen); - void* bufCopy = buf; - tSerializeSClientHbBatchReq(&bufCopy, &batchReq); + int32_t tlen = tSerializeSClientHbBatchReq(NULL, 0, &batchReq); + void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen); + tSerializeSClientHbBatchReq(buf, tlen, &batchReq); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, buf, tlen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - char* pRspChar = (char*)pMsg->pCont; + SClientHbBatchRsp rsp = {0}; - tDeserializeSClientHbBatchRsp(pRspChar, &rsp); + tDeserializeSClientHbBatchRsp(pMsg->pCont, pMsg->contLen, &rsp); int sz = taosArrayGetSize(rsp.rsps); ASSERT_EQ(sz, 0); - //SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0); - //EXPECT_EQ(pRsp->connKey.connId, 123); - //EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ); - //EXPECT_EQ(pRsp->status, 0); + + // SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0); + // EXPECT_EQ(pRsp->connKey.connId, 123); + // EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ); + // EXPECT_EQ(pRsp->status, 0); #if 0 int32_t contLen = sizeof(SHeartBeatReq); diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 8a9e087e7f..d4e308268a 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -52,10 +52,12 @@ TEST_F(MndTestQnode, 01_Show_Qnode) { TEST_F(MndTestQnode, 02_Create_Qnode) { { - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -63,10 +65,12 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { } { - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -83,10 +87,12 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { } { - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -96,11 +102,13 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { TEST_F(MndTestQnode, 03_Drop_Qnode) { { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9015; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9015); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -113,10 +121,12 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { } { - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -135,10 +145,12 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { } { - int32_t contLen = sizeof(SMDropQnodeReq); + SMDropQnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -154,10 +166,12 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { } { - int32_t contLen = sizeof(SMDropQnodeReq); + SMDropQnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -168,10 +182,12 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); @@ -181,10 +197,12 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { { // continue send message, qnode is creating - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -193,10 +211,12 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { { // continue send message, qnode is creating - int32_t contLen = sizeof(SMDropQnodeReq); + SMDropQnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -212,10 +232,12 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -230,10 +252,12 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMDropQnodeReq); + SMDropQnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); @@ -243,11 +267,12 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { { // continue send message, qnode is dropping - int32_t contLen = sizeof(SMCreateQnodeReq); - - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -255,10 +280,12 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { { // continue send message, qnode is dropping - int32_t contLen = sizeof(SMDropQnodeReq); + SMDropQnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -274,11 +301,12 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateQnodeReq); - - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); if (pRsp->code == 0) break; diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index bc31630ffe..e7e17d65c6 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -26,11 +26,13 @@ class MndTestShow : public ::testing::Test { Testbase MndTestShow::test; TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { - int32_t contLen = sizeof(SShowReq); + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_MAX; - SShowReq* pReq = (SShowReq*)rpcMallocCont(contLen); - pReq->type = TSDB_MGMT_TABLE_MAX; - strcpy(pReq->db, ""); + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -38,11 +40,13 @@ TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { } TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { - int32_t contLen = sizeof(SShowReq); + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_START; - SShowReq* pReq = (SShowReq*)rpcMallocCont(sizeof(SShowReq)); - pReq->type = TSDB_MGMT_TABLE_START; - strcpy(pReq->db, ""); + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index 3a38b9ede6..a38fb5d604 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -52,10 +52,12 @@ TEST_F(MndTestSnode, 01_Show_Snode) { TEST_F(MndTestSnode, 02_Create_Snode) { { - int32_t contLen = sizeof(SMCreateSnodeReq); + SMCreateSnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -63,10 +65,12 @@ TEST_F(MndTestSnode, 02_Create_Snode) { } { - int32_t contLen = sizeof(SMCreateSnodeReq); + SMCreateSnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -83,10 +87,12 @@ TEST_F(MndTestSnode, 02_Create_Snode) { } { - int32_t contLen = sizeof(SMCreateSnodeReq); + SMCreateSnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -96,11 +102,13 @@ TEST_F(MndTestSnode, 02_Create_Snode) { TEST_F(MndTestSnode, 03_Drop_Snode) { { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9017; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9017); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -113,10 +121,12 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { } { - int32_t contLen = sizeof(SMCreateSnodeReq); + SMCreateSnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -135,10 +145,12 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { } { - int32_t contLen = sizeof(SMDropSnodeReq); + SMDropSnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -154,10 +166,12 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { } { - int32_t contLen = sizeof(SMDropSnodeReq); + SMDropSnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -168,10 +182,12 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMCreateSnodeReq); + SMCreateSnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); @@ -181,10 +197,12 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { { // continue send message, snode is creating - int32_t contLen = sizeof(SMCreateSnodeReq); + SMCreateSnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -193,10 +211,12 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { { // continue send message, snode is creating - int32_t contLen = sizeof(SMDropSnodeReq); + SMDropSnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -212,10 +232,12 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateSnodeReq); + SMCreateSnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -230,10 +252,12 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMDropSnodeReq); + SMDropSnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); @@ -243,10 +267,12 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { { // continue send message, snode is dropping - int32_t contLen = sizeof(SMCreateSnodeReq); + SMCreateSnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -255,10 +281,12 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { { // continue send message, snode is dropping - int32_t contLen = sizeof(SMDropSnodeReq); + SMDropSnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -274,10 +302,12 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateSnodeReq); + SMCreateSnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index c5ab44a014..87df50ef0e 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -38,39 +38,43 @@ class MndTestStb : public ::testing::Test { Testbase MndTestStb::test; void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, dbname); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, dbname); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); *pContLen = contLen; return pReq; } void* MndTestStb::BuildDropDbReq(const char* dbname, int32_t* pContLen) { - int32_t contLen = sizeof(SDropDbReq); + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, dbname); - SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, dbname); + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); *pContLen = contLen; return pReq; diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 0c9f560fce..8a62ed639a 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -67,10 +67,9 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { strcpy(createReq.user, "u1"); strcpy(createReq.pass, "p1"); - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -101,10 +100,12 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { { - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -118,10 +119,12 @@ TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { KillThenRestartServer(); { - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -136,11 +139,13 @@ TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { { - int32_t contLen = sizeof(SCreateDnodeReq); + SCreateDnodeReq createReq = {0}; + strcpy(createReq.fqdn, "localhost"); + createReq.port = 9020; - SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); - strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9020); + int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -153,10 +158,12 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { } { - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); @@ -173,10 +180,12 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateQnodeReq); + SMCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 5d8c958f15..d8ce599be1 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -49,10 +49,9 @@ TEST_F(MndTestUser, 02_Create_User) { strcpy(createReq.user, ""); strcpy(createReq.pass, "p1"); - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -64,10 +63,9 @@ TEST_F(MndTestUser, 02_Create_User) { strcpy(createReq.user, "u1"); strcpy(createReq.pass, ""); - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -79,10 +77,9 @@ TEST_F(MndTestUser, 02_Create_User) { strcpy(createReq.user, "root"); strcpy(createReq.pass, "1"); - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -94,10 +91,9 @@ TEST_F(MndTestUser, 02_Create_User) { strcpy(createReq.user, "u1"); strcpy(createReq.pass, "p1"); - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -121,10 +117,9 @@ TEST_F(MndTestUser, 02_Create_User) { SDropUserReq dropReq = {0}; strcpy(dropReq.user, "u1"); - int32_t contLen = tSerializeSDropUserReq(NULL, &dropReq); + int32_t contLen = tSerializeSDropUserReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSDropUserReq(&pBuf, &dropReq); + tSerializeSDropUserReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -141,10 +136,9 @@ TEST_F(MndTestUser, 02_Create_User) { strcpy(createReq.pass, "p1"); createReq.superUser = 1; - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -168,10 +162,9 @@ TEST_F(MndTestUser, 02_Create_User) { SDropUserReq dropReq = {0}; strcpy(dropReq.user, "u2"); - int32_t contLen = tSerializeSDropUserReq(NULL, &dropReq); + int32_t contLen = tSerializeSDropUserReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSDropUserReq(&pBuf, &dropReq); + tSerializeSDropUserReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -190,10 +183,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(createReq.pass, "p1"); createReq.superUser = 1; - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -210,10 +202,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.user, ""); strcpy(alterReq.pass, "p1"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -226,10 +217,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.user, "u3"); strcpy(alterReq.pass, ""); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -242,10 +232,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.user, "u4"); strcpy(alterReq.pass, "1"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -258,10 +247,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.user, "u3"); strcpy(alterReq.pass, "1"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -275,10 +263,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.pass, "1"); alterReq.superUser = 1; - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -291,10 +278,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.user, "u3"); strcpy(alterReq.pass, "1"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -307,10 +293,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.user, "u3"); strcpy(alterReq.pass, "1"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -324,10 +309,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.pass, "1"); strcpy(alterReq.dbname, "d1"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -335,29 +319,31 @@ TEST_F(MndTestUser, 03_Alter_User) { } { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -371,10 +357,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.pass, "1"); strcpy(alterReq.dbname, "1.d2"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -388,10 +373,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.pass, "1"); strcpy(alterReq.dbname, "1.d2"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -401,17 +385,16 @@ TEST_F(MndTestUser, 03_Alter_User) { { SGetUserAuthReq authReq = {0}; strcpy(authReq.user, "u3"); - int32_t contLen = tSerializeSGetUserAuthReq(NULL, &authReq); + int32_t contLen = tSerializeSGetUserAuthReq(NULL, 0, &authReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSGetUserAuthReq(&pBuf, &authReq); + tSerializeSGetUserAuthReq(pReq, contLen, &authReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_GET_USER_AUTH, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); SGetUserAuthRsp authRsp = {0}; - tDeserializeSGetUserAuthRsp(pRsp->pCont, &authRsp); + tDeserializeSGetUserAuthRsp(pRsp->pCont, pRsp->contLen, &authRsp); EXPECT_STREQ(authRsp.user, "u3"); EXPECT_EQ(authRsp.superAuth, 1); int32_t numOfReadDbs = taosHashGetSize(authRsp.readDbs); @@ -433,10 +416,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(alterReq.pass, "1"); strcpy(alterReq.dbname, "1.d2"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -447,10 +429,9 @@ TEST_F(MndTestUser, 03_Alter_User) { SDropUserReq dropReq = {0}; strcpy(dropReq.user, "u3"); - int32_t contLen = tSerializeSDropUserReq(NULL, &dropReq); + int32_t contLen = tSerializeSDropUserReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSDropUserReq(&pBuf, &dropReq); + tSerializeSDropUserReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -467,10 +448,9 @@ TEST_F(MndTestUser, 05_Drop_User) { SDropUserReq dropReq = {0}; strcpy(dropReq.user, ""); - int32_t contLen = tSerializeSDropUserReq(NULL, &dropReq); + int32_t contLen = tSerializeSDropUserReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSDropUserReq(&pBuf, &dropReq); + tSerializeSDropUserReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -481,10 +461,9 @@ TEST_F(MndTestUser, 05_Drop_User) { SDropUserReq dropReq = {0}; strcpy(dropReq.user, "u4"); - int32_t contLen = tSerializeSDropUserReq(NULL, &dropReq); + int32_t contLen = tSerializeSDropUserReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSDropUserReq(&pBuf, &dropReq); + tSerializeSDropUserReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -496,10 +475,9 @@ TEST_F(MndTestUser, 05_Drop_User) { strcpy(createReq.user, "u1"); strcpy(createReq.pass, "p1"); - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -510,10 +488,9 @@ TEST_F(MndTestUser, 05_Drop_User) { SDropUserReq dropReq = {0}; strcpy(dropReq.user, "u1"); - int32_t contLen = tSerializeSDropUserReq(NULL, &dropReq); + int32_t contLen = tSerializeSDropUserReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSDropUserReq(&pBuf, &dropReq); + tSerializeSDropUserReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -533,10 +510,9 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { strcpy(createReq.user, "u1"); strcpy(createReq.pass, "p1"); - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -548,10 +524,9 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { strcpy(createReq.user, "u2"); strcpy(createReq.pass, "p2"); - int32_t contLen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -583,10 +558,9 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { strcpy(alterReq.user, "u1"); strcpy(alterReq.pass, "p2"); - int32_t contLen = tSerializeSAlterUserReq(NULL, &alterReq); + int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSAlterUserReq(&pBuf, &alterReq); + tSerializeSAlterUserReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -616,10 +590,9 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { SDropUserReq dropReq = {0}; strcpy(dropReq.user, "u1"); - int32_t contLen = tSerializeSDropUserReq(NULL, &dropReq); + int32_t contLen = tSerializeSDropUserReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - void* pBuf = pReq; - tSerializeSDropUserReq(&pBuf, &dropReq); + tSerializeSDropUserReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index bd626154e6..a516f423bb 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -35,155 +35,14 @@ extern "C" { #endif -#define TQ_BUFFER_SIZE 8 +typedef struct STQ STQ; -typedef struct STqRspHandle { - void* handle; - void* ahandle; -} STqRspHandle; - -typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; - -typedef struct STqTaskItem { - int8_t status; - int64_t offset; - void* dst; - qTaskInfo_t task; - STqReadHandle* pReadHandle; - SSubQueryMsg* pQueryMsg; -} STqTaskItem; - -// new version -typedef struct STqBuffer { - int64_t firstOffset; - int64_t lastOffset; - STqTaskItem output[TQ_BUFFER_SIZE]; -} STqBuffer; - -typedef struct STqTopicHandle { - char topicName[TSDB_TOPIC_FNAME_LEN]; - char* sql; - char* logicalPlan; - char* physicalPlan; - int64_t committedOffset; - int64_t currentOffset; - STqBuffer buffer; - SWalReadHandle* pReadhandle; -} STqTopicHandle; - -typedef struct STqConsumerHandle { - int64_t consumerId; - int64_t epoch; - char cgroup[TSDB_TOPIC_FNAME_LEN]; - SArray* topics; // SArray -} STqConsumerHandle; - -typedef struct STqMemRef { +// memory allocator provided by vnode +typedef struct { SMemAllocatorFactory* pAllocatorFactory; SMemAllocator* pAllocator; } STqMemRef; -typedef struct STqSerializedHead { - int16_t ver; - int16_t action; - int32_t checksum; - int64_t ssize; - char content[]; -} STqSerializedHead; - -typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); -typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj); -typedef void (*FTqDelete)(void*); - -#define TQ_BUCKET_MASK 0xFF -#define TQ_BUCKET_SIZE 256 - -#define TQ_PAGE_SIZE 4096 -// key + offset + size -#define TQ_IDX_SIZE 24 -// 4096 / 24 -#define TQ_MAX_IDX_ONE_PAGE 170 -// 24 * 170 -#define TQ_IDX_PAGE_BODY_SIZE 4080 -// 4096 - 4080 -#define TQ_IDX_PAGE_HEAD_SIZE 16 - -#define TQ_ACTION_CONST 0 -#define TQ_ACTION_INUSE 1 -#define TQ_ACTION_INUSE_CONT 2 -#define TQ_ACTION_INTXN 3 - -#define TQ_SVER 0 - -// TODO: inplace mode is not implemented -#define TQ_UPDATE_INPLACE 0 -#define TQ_UPDATE_APPEND 1 - -#define TQ_DUP_INTXN_REWRITE 0 -#define TQ_DUP_INTXN_REJECT 2 - -static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } - -static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } - -static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; - -#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE - -typedef struct STqMetaHandle { - int64_t key; - int64_t offset; - int64_t serializedSize; - void* valueInUse; - void* valueInTxn; -} STqMetaHandle; - -typedef struct STqMetaList { - STqMetaHandle handle; - struct STqMetaList* next; - // struct STqMetaList* inTxnPrev; - // struct STqMetaList* inTxnNext; - struct STqMetaList* unpersistPrev; - struct STqMetaList* unpersistNext; -} STqMetaList; - -typedef struct STqMetaStore { - STqMetaList* bucket[TQ_BUCKET_SIZE]; - // a table head - STqMetaList* unpersistHead; - // topics that are not connectted - STqMetaList* unconnectTopic; - - // TODO:temporaral use, to be replaced by unified tfile - int fileFd; - // TODO:temporaral use, to be replaced by unified tfile - int idxFd; - - char* dirPath; - int32_t tqConfigFlag; - FTqSerialize pSerializer; - FTqDeserialize pDeserializer; - FTqDelete pDeleter; -} STqMetaStore; - -typedef struct STQ { - // the collection of groups - // the handle of meta kvstore - char* path; - STqCfg* tqConfig; - STqMemRef tqMemRef; - STqMetaStore* tqMeta; - SWal* pWal; - SMeta* pMeta; -} STQ; - -typedef struct STqMgmt { - int8_t inited; - tmr_h timer; -} STqMgmt; - -static STqMgmt tqMgmt; - // init once int tqInit(); void tqCleanUp(); @@ -192,7 +51,7 @@ void tqCleanUp(); STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); void tqClose(STQ*); -// void* will be replace by a msg type +// required by vnode int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7549772613..394076f433 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -35,13 +35,14 @@ typedef struct SDnode SDnode; typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef struct STqCfg { +typedef struct { // TODO int32_t reserved; } STqCfg; -typedef struct SVnodeCfg { +typedef struct { int32_t vgId; + uint64_t dbId; SDnode *pDnode; STfs *pTfs; uint64_t wsize; @@ -67,9 +68,9 @@ typedef struct { SendReqToDnodeFp sendReqToDnodeFp; } SVnodeOpt; -typedef struct STqReadHandle { +typedef struct { int64_t ver; - uint64_t tbUid; + int64_t tbUid; SHashObj *tbIdHash; const SSubmitMsg *pMsg; SSubmitBlk *pBlock; @@ -199,7 +200,7 @@ int32_t vnodeCompact(SVnode *pVnode); int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); -/* ------------------------- TQ QUERY -------------------------- */ +/* ------------------------- TQ READ --------------------------- */ STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); @@ -207,12 +208,12 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA pReadHandle->pColIdList = pColIdList; } -// static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, const SArray* pTableIdList) { -// pHandle->tbUid = pTableIdList; +// static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, int64_t tbUid) { +// pHandle->tbUid = tbUid; //} static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { - pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { return -1; } diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index cefe9eff72..344ad992f0 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -20,6 +20,7 @@ #include "tlog.h" #include "tq.h" #include "trpc.h" + #ifdef __cplusplus extern "C" { #endif @@ -63,8 +64,148 @@ extern int32_t tqDebugFlag; } \ } -int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); -const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); +#define TQ_BUFFER_SIZE 8 + +#define TQ_BUCKET_MASK 0xFF +#define TQ_BUCKET_SIZE 256 + +#define TQ_PAGE_SIZE 4096 +// key + offset + size +#define TQ_IDX_SIZE 24 +// 4096 / 24 +#define TQ_MAX_IDX_ONE_PAGE 170 +// 24 * 170 +#define TQ_IDX_PAGE_BODY_SIZE 4080 +// 4096 - 4080 +#define TQ_IDX_PAGE_HEAD_SIZE 16 + +#define TQ_ACTION_CONST 0 +#define TQ_ACTION_INUSE 1 +#define TQ_ACTION_INUSE_CONT 2 +#define TQ_ACTION_INTXN 3 + +#define TQ_SVER 0 + +// TODO: inplace mode is not implemented +#define TQ_UPDATE_INPLACE 0 +#define TQ_UPDATE_APPEND 1 + +#define TQ_DUP_INTXN_REWRITE 0 +#define TQ_DUP_INTXN_REJECT 2 + +static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } + +static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } + +static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; + +#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE + +typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; + +typedef struct { + int16_t ver; + int16_t action; + int32_t checksum; + int64_t ssize; + char content[]; +} STqSerializedHead; + +typedef int32_t (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); +typedef int32_t (*FTqDeserialize)(void* self, const STqSerializedHead* pHead, void** ppObj); +typedef void (*FTqDelete)(void*); + +typedef struct { + int64_t key; + int64_t offset; + int64_t serializedSize; + void* valueInUse; + void* valueInTxn; +} STqMetaHandle; + +typedef struct STqMetaList { + STqMetaHandle handle; + struct STqMetaList* next; + // struct STqMetaList* inTxnPrev; + // struct STqMetaList* inTxnNext; + struct STqMetaList* unpersistPrev; + struct STqMetaList* unpersistNext; +} STqMetaList; + +typedef struct { + STQ* pTq; + STqMetaList* bucket[TQ_BUCKET_SIZE]; + // a table head + STqMetaList* unpersistHead; + // topics that are not connectted + STqMetaList* unconnectTopic; + + // TODO:temporaral use, to be replaced by unified tfile + int fileFd; + // TODO:temporaral use, to be replaced by unified tfile + int idxFd; + + char* dirPath; + int32_t tqConfigFlag; + FTqSerialize pSerializer; + FTqDeserialize pDeserializer; + FTqDelete pDeleter; +} STqMetaStore; + +struct STQ { + // the collection of groups + // the handle of meta kvstore + char* path; + STqCfg* tqConfig; + STqMemRef tqMemRef; + STqMetaStore* tqMeta; + SWal* pWal; + SMeta* pMeta; +}; + +typedef struct { + int8_t inited; + tmr_h timer; +} STqMgmt; + +static STqMgmt tqMgmt; + +typedef struct { + int8_t status; + int64_t offset; + qTaskInfo_t task; + STqReadHandle* pReadHandle; +} STqTaskItem; + +// new version +typedef struct { + int64_t firstOffset; + int64_t lastOffset; + STqTaskItem output[TQ_BUFFER_SIZE]; +} STqBuffer; + +typedef struct { + char topicName[TSDB_TOPIC_FNAME_LEN]; + char* sql; + char* logicalPlan; + char* physicalPlan; + char* qmsg; + int64_t persistedOffset; + int64_t committedOffset; + int64_t currentOffset; + STqBuffer buffer; + SWalReadHandle* pReadhandle; +} STqTopic; + +typedef struct { + int64_t consumerId; + int64_t epoch; + char cgroup[TSDB_TOPIC_FNAME_LEN]; + SArray* topics; // SArray +} STqConsumer; + +int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**); +int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**); static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } diff --git a/source/dnode/vnode/src/inc/tqMetaStore.h b/source/dnode/vnode/src/inc/tqMetaStore.h index 3bf9bb7138..eb203b7117 100644 --- a/source/dnode/vnode/src/inc/tqMetaStore.h +++ b/source/dnode/vnode/src/inc/tqMetaStore.h @@ -23,8 +23,8 @@ extern "C" { #endif -STqMetaStore* tqStoreOpen(const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, FTqDelete pDeleter, - int32_t tqConfigFlag); +STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, + FTqDelete pDeleter, int32_t tqConfigFlag); int32_t tqStoreClose(STqMetaStore*); // int32_t tqStoreDelete(TqMetaStore*); // int32_t tqStoreCommitAll(TqMetaStore*); diff --git a/source/dnode/vnode/src/inc/tqOffset.h b/source/dnode/vnode/src/inc/tqOffset.h new file mode 100644 index 0000000000..b58de26f68 --- /dev/null +++ b/source/dnode/vnode/src/inc/tqOffset.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TQ_OFFSET_H_ +#define _TD_TQ_OFFSET_H_ + +#include "tqInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STqOffsetCfg STqOffsetCfg; +typedef struct STqOffsetStore STqOffsetStore; + +STqOffsetStore* STqOffsetOpen(STqOffsetCfg*); +void STqOffsetClose(STqOffsetStore*); + +int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey); +int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset); +int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey); +int32_t tqOffsetPersistAll(STqOffsetStore* pStore); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TQ_OFFSET_H_*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index af4c9da05f..79e733692c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -50,7 +50,8 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl // TODO: error code of buffer pool } #endif - pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, free, 0); + pTq->tqMeta = + tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, free, 0); if (pTq->tqMeta == NULL) { free(pTq); #if 0 @@ -76,19 +77,89 @@ int tqPushMsg(STQ* pTq, void* p, int64_t version) { return 0; } -int tqCommit(STQ* pTq) { - // do nothing - return 0; +int tqCommit(STQ* pTq) { return tqStorePersist(pTq->tqMeta); } + +int32_t tqGetTopicHandleSize(const STqTopic* pTopic) { + return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->logicalPlan) + strlen(pTopic->physicalPlan) + + strlen(pTopic->qmsg) + sizeof(int64_t) * 3; } -int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** ppHead) { - int32_t num = taosArrayGetSize(pConsumer->topics); - int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + - num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN); +int32_t tqGetConsumerHandleSize(const STqConsumer* pConsumer) { + int num = taosArrayGetSize(pConsumer->topics); + int32_t sz = 0; + for (int i = 0; i < num; i++) { + STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); + sz += tqGetTopicHandleSize(pTopic); + } + return sz; +} + +static FORCE_INLINE int32_t tEncodeSTqTopic(void** buf, const STqTopic* pTopic) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pTopic->topicName); + /*tlen += taosEncodeString(buf, pTopic->sql);*/ + /*tlen += taosEncodeString(buf, pTopic->logicalPlan);*/ + /*tlen += taosEncodeString(buf, pTopic->physicalPlan);*/ + tlen += taosEncodeString(buf, pTopic->qmsg); + tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset); + tlen += taosEncodeFixedI64(buf, pTopic->committedOffset); + tlen += taosEncodeFixedI64(buf, pTopic->currentOffset); + return tlen; +} + +static FORCE_INLINE const void* tDecodeSTqTopic(const void* buf, STqTopic* pTopic) { + buf = taosDecodeStringTo(buf, pTopic->topicName); + /*buf = taosDecodeString(buf, &pTopic->sql);*/ + /*buf = taosDecodeString(buf, &pTopic->logicalPlan);*/ + /*buf = taosDecodeString(buf, &pTopic->physicalPlan);*/ + buf = taosDecodeString(buf, &pTopic->qmsg); + buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset); + buf = taosDecodeFixedI64(buf, &pTopic->committedOffset); + buf = taosDecodeFixedI64(buf, &pTopic->currentOffset); + return buf; +} + +static FORCE_INLINE int32_t tEncodeSTqConsumer(void** buf, const STqConsumer* pConsumer) { + int32_t sz; + + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeFixedI64(buf, pConsumer->epoch); + tlen += taosEncodeString(buf, pConsumer->cgroup); + sz = taosArrayGetSize(pConsumer->topics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); + tlen += tEncodeSTqTopic(buf, pTopic); + } + return tlen; +} + +static FORCE_INLINE const void* tDecodeSTqConsumer(const void* buf, STqConsumer* pConsumer) { + int32_t sz; + + buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeFixedI64(buf, &pConsumer->epoch); + buf = taosDecodeStringTo(buf, pConsumer->cgroup); + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->topics = taosArrayInit(sz, sizeof(STqTopic)); + if (pConsumer->topics == NULL) return NULL; + for (int32_t i = 0; i < sz; i++) { + STqTopic pTopic; + buf = tDecodeSTqTopic(buf, &pTopic); + taosArrayPush(pConsumer->topics, &pTopic); + } + return buf; +} + +int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead) { + int32_t sz = tEncodeSTqConsumer(NULL, pConsumer); + if (sz > (*ppHead)->ssize) { - void* tmpPtr = realloc(*ppHead, sz); + void* tmpPtr = realloc(*ppHead, sizeof(STqSerializedHead) + sz); if (tmpPtr == NULL) { free(*ppHead); + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } *ppHead = tmpPtr; @@ -96,44 +167,42 @@ int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** } void* ptr = (*ppHead)->content; - *(int64_t*)ptr = pConsumer->consumerId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = pConsumer->epoch; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - memcpy(ptr, pConsumer->topics, TSDB_TOPIC_FNAME_LEN); - ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN); - *(int32_t*)ptr = num; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - for (int32_t i = 0; i < num; i++) { - STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); - memcpy(ptr, pTopic->topicName, TSDB_TOPIC_FNAME_LEN); - ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN); - *(int64_t*)ptr = pTopic->committedOffset; - POINTER_SHIFT(ptr, sizeof(int64_t)); - } + void* abuf = ptr; + tEncodeSTqConsumer(&abuf, pConsumer); return 0; } -const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle** ppConsumer) { - STqConsumerHandle* pConsumer = *ppConsumer; - const void* ptr = pHead->content; - pConsumer->consumerId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - pConsumer->epoch = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - memcpy(pConsumer->cgroup, ptr, TSDB_TOPIC_FNAME_LEN); - ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN); - int32_t sz = *(int32_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - pConsumer->topics = taosArrayInit(sz, sizeof(STqTopicHandle)); - for (int32_t i = 0; i < sz; i++) { - /*STqTopicHandle* topicHandle = */ - /*taosArrayPush(pConsumer->topics, );*/ +int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsumer** ppConsumer) { + const void* str = pHead->content; + *ppConsumer = calloc(1, sizeof(STqConsumer)); + if (*ppConsumer == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; + } + if (tDecodeSTqConsumer(str, *ppConsumer) == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; + } + STqConsumer* pConsumer = *ppConsumer; + int32_t sz = taosArrayGetSize(pConsumer->topics); + for (int32_t i = 0; i < sz; i++) { + STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); + pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); + if (pTopic->pReadhandle == NULL) { + ASSERT(false); + } + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { + pTopic->buffer.output[i].status = 0; + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); + SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pMeta}; + pTopic->buffer.output[i].pReadHandle = pReadHandle; + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); + } } - return NULL; -} + return 0; +} int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; @@ -144,7 +213,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; - STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); + /*printf("vg %d get consume req\n", pReq->head.vgId);*/ + + STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); if (pConsumer == NULL) { pMsg->pCont = NULL; pMsg->contLen = 0; @@ -155,9 +226,10 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { int sz = taosArrayGetSize(pConsumer->topics); for (int i = 0; i < sz; i++) { - STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); + STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); // TODO: support multiple topic in one req if (strcmp(pTopic->topicName, pReq->topic) != 0) { + ASSERT(false); continue; } @@ -171,7 +243,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { - pTopic->committedOffset = pReq->offset-1; + pTopic->committedOffset = pReq->offset - 1; } rsp.committedOffset = pTopic->committedOffset; @@ -181,6 +253,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (fetchOffset <= pTopic->committedOffset) { fetchOffset = pTopic->committedOffset + 1; } + /*printf("vg %d fetch Offset %ld\n", pReq->head.vgId, fetchOffset);*/ int8_t pos; int8_t skip = 0; SWalHead* pHead; @@ -231,7 +304,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { break; } } - //TODO copy + // TODO copy rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; rsp.rspOffset = fetchOffset; @@ -265,15 +338,12 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } void* abuf = buf; tEncodeSMqConsumeRsp(&abuf, &rsp); + if (rsp.pBlockData) { - taosArrayDestroyEx(rsp.pBlockData, (void(*)(void*))tDeleteSSDataBlock); + taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); rsp.pBlockData = NULL; - /*for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) {*/ - /*SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i);*/ - /*tDeleteSSDataBlock(pBlock);*/ - /*}*/ - /*taosArrayDestroy(rsp.pBlockData);*/ } + pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; @@ -285,8 +355,9 @@ int32_t tqProcessRebReq(STQ* pTq, char* msg) { SMqMVRebReq req = {0}; tDecodeSMqMVRebReq(msg, &req); - STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); + STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); ASSERT(pConsumer); + pConsumer->consumerId = req.newConsumerId; tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); tqHandleCommit(pTq->tqMeta, req.newConsumerId); tqHandlePurge(pTq->tqMeta, req.oldConsumerId); @@ -297,28 +368,22 @@ int32_t tqProcessRebReq(STQ* pTq, char* msg) { int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { SMqSetCVgReq req = {0}; tDecodeSMqSetCVgReq(msg, &req); - - STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); + + /*printf("vg %d set to consumer from %ld to %ld\n", req.vgId, req.oldConsumerId, req.newConsumerId);*/ + STqConsumer* pConsumer = calloc(1, sizeof(STqConsumer)); if (pConsumer == NULL) { - pConsumer = calloc(sizeof(STqConsumerHandle), 1); - if (pConsumer == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - } else { - tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.newConsumerId); - tqHandlePurge(pTq->tqMeta, req.oldConsumerId); - terrno = TSDB_CODE_SUCCESS; - return 0; + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; } + strcpy(pConsumer->cgroup, req.cgroup); - pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle)); - pConsumer->consumerId = req.newConsumerId; + pConsumer->topics = taosArrayInit(0, sizeof(STqTopic)); + pConsumer->consumerId = req.consumerId; pConsumer->epoch = 0; - STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); + STqTopic* pTopic = calloc(1, sizeof(STqTopic)); if (pTopic == NULL) { + taosArrayDestroy(pConsumer->topics); free(pConsumer); return -1; } @@ -326,6 +391,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { pTopic->sql = req.sql; pTopic->logicalPlan = req.logicalPlan; pTopic->physicalPlan = req.physicalPlan; + pTopic->qmsg = req.qmsg; pTopic->committedOffset = -1; pTopic->currentOffset = -1; @@ -333,157 +399,18 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { pTopic->buffer.lastOffset = -1; pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); if (pTopic->pReadhandle == NULL) { + ASSERT(false); } for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); - SReadHandle handle = { .reader = pReadHandle, .meta = pTq->pMeta }; + SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pMeta}; pTopic->buffer.output[i].pReadHandle = pReadHandle; pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); } taosArrayPush(pConsumer->topics, pTopic); - tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.newConsumerId); + tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer); + tqHandleCommit(pTq->tqMeta, req.consumerId); terrno = TSDB_CODE_SUCCESS; return 0; } - -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { - STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); - if (pReadHandle == NULL) { - return NULL; - } - pReadHandle->pVnodeMeta = pMeta; - pReadHandle->pMsg = NULL; - pReadHandle->ver = -1; - pReadHandle->pColIdList = NULL; - pReadHandle->sver = -1; - pReadHandle->pSchema = NULL; - pReadHandle->pSchemaWrapper = NULL; - return pReadHandle; -} - -void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) { - pReadHandle->pMsg = pMsg; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter); - pReadHandle->ver = ver; - memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); -} - -bool tqNextDataBlock(STqReadHandle* pHandle) { - while (1) { - if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { - return false; - } - if (pHandle->pBlock == NULL) return false; - - pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid); - /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/ - ASSERT(pHandle->tbIdHash); - void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); - if (ret != NULL) { - pHandle->pBlock->tid = htonl(pHandle->pBlock->tid); - pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion); - pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen); - pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen); - pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows); - return true; - } - } - return false; -} - -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { - /*int32_t sversion = pHandle->pBlock->sversion;*/ - /*SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);*/ - pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList); - pBlockInfo->rows = pHandle->pBlock->numOfRows; - pBlockInfo->uid = pHandle->pBlock->uid; - return 0; -} - -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { - /*int32_t sversion = pHandle->pBlock->sversion;*/ - // TODO set to real sversion - int32_t sversion = 0; - if (pHandle->sver != sversion) { - pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->pBlock->uid, sversion); - - tb_uid_t quid; - STbCfg* pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->pBlock->uid); - if (pTbCfg->type == META_CHILD_TABLE) { - quid = pTbCfg->ctbCfg.suid; - } else { - quid = pHandle->pBlock->uid; - } - pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true); - pHandle->sver = sversion; - } - - STSchema* pTschema = pHandle->pSchema; - SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; - - int32_t numOfRows = pHandle->pBlock->numOfRows; - int32_t numOfCols = pHandle->pSchema->numOfCols; - int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); - - //TODO: stable case - if (colNumNeed > pSchemaWrapper->nCols) { - colNumNeed = pSchemaWrapper->nCols; - } - - SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); - if (pArray == NULL) { - return NULL; - } - - int j = 0; - for (int32_t i = 0; i < colNumNeed; i++) { - int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); - while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { - j++; - } - SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; - SColumnInfoData colInfo = {0}; - int sz = numOfRows * pColSchema->bytes; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = colId; - colInfo.info.type = pColSchema->type; - - colInfo.pData = calloc(1, sz); - if (colInfo.pData == NULL) { - // TODO free - taosArrayDestroy(pArray); - return NULL; - } - taosArrayPush(pArray, &colInfo); - } - - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pTschema); - STSRow* row; - // int32_t kvIdx = 0; - int32_t curRow = 0; - tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter); - while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { - tdSTSRowIterReset(&iter, row); - // get all wanted col of that block - for (int32_t i = 0; i < colNumNeed; i++) { - SColumnInfoData* pColData = taosArrayGet(pArray, i); - STColumn* pCol = schemaColAt(pTschema, i); - // TODO - ASSERT(pCol->colId == pColData->info.colId); - // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); - SCellVal sVal = {0}; - if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) { - // TODO: reach end - break; - } - memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes); - } - curRow++; - } - return pArray; -} diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index d220966ba6..d40cc2294f 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -68,20 +68,21 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { return lseek(fd, offset, SEEK_SET); } -STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter, - int32_t tqConfigFlag) { - STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); +STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, + FTqDelete deleter, int32_t tqConfigFlag) { + STqMetaStore* pMeta = calloc(1, sizeof(STqMetaStore)); if (pMeta == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } - memset(pMeta, 0, sizeof(STqMetaStore)); + pMeta->pTq = pTq; // concat data file name and index file name size_t pathLen = strlen(path); pMeta->dirPath = malloc(pathLen + 1); if (pMeta->dirPath == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + free(pMeta); return NULL; } strcpy(pMeta->dirPath, path); @@ -103,12 +104,11 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial } pMeta->idxFd = idxFd; - pMeta->unpersistHead = malloc(sizeof(STqMetaList)); + pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } - memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; strcpy(name, path); @@ -145,12 +145,11 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial ASSERT(idxBuf.head.writeOffset == idxRead); // loop read every entry for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - STqMetaList* pNode = malloc(sizeof(STqMetaList)); + STqMetaList* pNode = calloc(1, sizeof(STqMetaList)); if (pNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; // TODO: free memory } - memset(pNode, 0, sizeof(STqMetaList)); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); lseek(fileFd, pNode->handle.offset, SEEK_SET); @@ -169,25 +168,25 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial } if (serializedObj->action == TQ_ACTION_INUSE) { if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); + pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse); } else { pNode->handle.valueInUse = TQ_DELETE_TOKEN; } } else if (serializedObj->action == TQ_ACTION_INTXN) { if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn); + pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInTxn); } else { pNode->handle.valueInTxn = TQ_DELETE_TOKEN; } } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) { if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); + pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse); } else { pNode->handle.valueInUse = TQ_DELETE_TOKEN; } STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); if (ptr->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn); + pMeta->pDeserializer(pTq, ptr, &pNode->handle.valueInTxn); } else { pNode->handle.valueInTxn = TQ_DELETE_TOKEN; } @@ -289,7 +288,6 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) { return 0; } -// TODO: wrap in tfile int32_t tqStorePersist(STqMetaStore* pMeta) { STqIdxPageBuf idxBuf; int64_t* bufPtr = (int64_t*)idxBuf.buffer; @@ -303,7 +301,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { pSHead->ver = TQ_SVER; pSHead->checksum = 0; pSHead->ssize = sizeof(STqSerializedHead); - int allocatedSize = sizeof(STqSerializedHead); + /*int allocatedSize = sizeof(STqSerializedHead);*/ int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); tqReadLastPage(pMeta->idxFd, &idxBuf); @@ -418,14 +416,14 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu pNode = pNode->next; } } - STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); + STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } - memset(pNewNode, 0, sizeof(STqMetaList)); pNewNode->handle.key = key; pNewNode->handle.valueInUse = value; + pNewNode->next = pMeta->bucket[bucketKey]; // put into unpersist list pNewNode->unpersistPrev = pMeta->unpersistHead; pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext; @@ -490,12 +488,11 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va pNode = pNode->next; } } - STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); + STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } - memset(pNewNode, 0, sizeof(STqMetaList)); pNewNode->handle.key = key; pNewNode->handle.valueInTxn = value; pNewNode->next = pMeta->bucket[bucketKey]; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c new file mode 100644 index 0000000000..4115cb7313 --- /dev/null +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#define _DEFAULT_SOURCE + +#include "tqOffset.h" + +enum ETqOffsetPersist { + TQ_OFFSET_PERSIST__LAZY = 1, + TQ_OFFSET_PERSIST__EAGER, +}; + +struct STqOffsetCfg { + int8_t persistPolicy; +}; + +struct STqOffsetStore { + STqOffsetCfg cfg; + SHashObj* pHash; // SHashObj +}; + +STqOffsetStore* STqOffsetOpen(STqOffsetCfg* pCfg) { + STqOffsetStore* pStore = malloc(sizeof(STqOffsetStore)); + if (pStore == NULL) { + return NULL; + } + memcpy(&pStore->cfg, pCfg, sizeof(STqOffsetCfg)); + pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); + return pStore; +} + diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c new file mode 100644 index 0000000000..a83e289fec --- /dev/null +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#define _DEFAULT_SOURCE + +#include "vnode.h" + +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { + STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); + if (pReadHandle == NULL) { + return NULL; + } + pReadHandle->pVnodeMeta = pMeta; + pReadHandle->pMsg = NULL; + pReadHandle->ver = -1; + pReadHandle->pColIdList = NULL; + pReadHandle->sver = -1; + pReadHandle->pSchema = NULL; + pReadHandle->pSchemaWrapper = NULL; + return pReadHandle; +} + +void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) { + pReadHandle->pMsg = pMsg; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter); + pReadHandle->ver = ver; + memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); +} + +bool tqNextDataBlock(STqReadHandle* pHandle) { + while (1) { + if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { + return false; + } + if (pHandle->pBlock == NULL) return false; + + pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid); + /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/ + ASSERT(pHandle->tbIdHash); + void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); + if (ret != NULL) { + pHandle->pBlock->tid = htonl(pHandle->pBlock->tid); + pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion); + pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen); + pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen); + pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows); + return true; + } + } + return false; +} + +int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { + /*int32_t sversion = pHandle->pBlock->sversion;*/ + /*SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);*/ + pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList); + pBlockInfo->rows = pHandle->pBlock->numOfRows; + pBlockInfo->uid = pHandle->pBlock->uid; + return 0; +} + +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { + /*int32_t sversion = pHandle->pBlock->sversion;*/ + // TODO set to real sversion + int32_t sversion = 0; + if (pHandle->sver != sversion) { + pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->pBlock->uid, sversion); + + tb_uid_t quid; + STbCfg* pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->pBlock->uid); + if (pTbCfg->type == META_CHILD_TABLE) { + quid = pTbCfg->ctbCfg.suid; + } else { + quid = pHandle->pBlock->uid; + } + pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true); + pHandle->sver = sversion; + } + + STSchema* pTschema = pHandle->pSchema; + SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; + + int32_t numOfRows = pHandle->pBlock->numOfRows; + int32_t numOfCols = pHandle->pSchema->numOfCols; + int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); + + // TODO: stable case + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + + SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); + if (pArray == NULL) { + return NULL; + } + + int j = 0; + for (int32_t i = 0; i < colNumNeed; i++) { + int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); + while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { + j++; + } + SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; + SColumnInfoData colInfo = {0}; + int sz = numOfRows * pColSchema->bytes; + colInfo.info.bytes = pColSchema->bytes; + colInfo.info.colId = colId; + colInfo.info.type = pColSchema->type; + + colInfo.pData = calloc(1, sz); + if (colInfo.pData == NULL) { + // TODO free + taosArrayDestroy(pArray); + return NULL; + } + taosArrayPush(pArray, &colInfo); + } + + STSRowIter iter = {0}; + tdSTSRowIterInit(&iter, pTschema); + STSRow* row; + // int32_t kvIdx = 0; + int32_t curRow = 0; + tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter); + while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { + tdSTSRowIterReset(&iter, row); + // get all wanted col of that block + for (int32_t i = 0; i < colNumNeed; i++) { + SColumnInfoData* pColData = taosArrayGet(pArray, i); + STColumn* pCol = schemaColAt(pTschema, i); + // TODO + ASSERT(pCol->colId == pColData->info.colId); + // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); + SCellVal sVal = {0}; + if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) { + // TODO: reach end + break; + } + memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes); + } + curRow++; + } + return pArray; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 9b44e14c9b..26c313f421 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -18,7 +18,7 @@ #define TSDB_MAX_SUBBLOCKS 8 typedef struct { - STable * pTable; + STable *pTable; SSkipListIterator *pIter; } SCommitIter; @@ -34,11 +34,11 @@ typedef struct { bool isLFileSame; TSKEY minKey; TSKEY maxKey; - SArray * aBlkIdx; // SBlockIdx array - STable * pTable; - SArray * aSupBlk; // Table super-block array - SArray * aSubBlk; // table sub-block array - SDataCols * pDataCols; + SArray *aBlkIdx; // SBlockIdx array + STable *pTable; + SArray *aSupBlk; // Table super-block array + SArray *aSubBlk; // table sub-block array + SDataCols *pDataCols; } SCommitH; #define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5) @@ -90,7 +90,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { SDiskID did; SDFileSet nSet = {0}; - STsdbFS * pfs = REPO_FS(pRepo); + STsdbFS *pfs = REPO_FS(pRepo); int level; ASSERT(pSet->fid >= pRtn->minFid); @@ -135,12 +135,13 @@ int tsdbPrepareCommit(STsdb *pTsdb) { pTsdb->imem = pTsdb->mem; pTsdb->mem = NULL; + return 0; } int tsdbCommit(STsdb *pRepo) { STsdbMemTable *pMem = pRepo->imem; SCommitH commith = {0}; - SDFileSet * pSet = NULL; + SDFileSet *pSet = NULL; int fid; if (pRepo->imem == NULL) return 0; @@ -303,7 +304,7 @@ static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { } static int tsdbNextCommitFid(SCommitH *pCommith) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdbCfg *pCfg = REPO_CFG(pRepo); int fid = TSDB_IVLD_FID; @@ -336,7 +337,7 @@ static void tsdbDestroyCommitH(SCommitH *pCommith) { } static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdbCfg *pCfg = REPO_CFG(pRepo); ASSERT(pSet == NULL || pSet->fid == fid); @@ -391,12 +392,12 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { } static int tsdbCreateCommitIters(SCommitH *pCommith) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbMemTable * pMem = pRepo->imem; + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbMemTable *pMem = pRepo->imem; SSkipListIterator *pSlIter; - SCommitIter * pCommitIter; - SSkipListNode * pNode; - STbData * pTbData; + SCommitIter *pCommitIter; + SSkipListNode *pNode; + STbData *pTbData; pCommith->niters = SL_SIZE(pMem->pSlIdx); pCommith->iters = (SCommitIter *)calloc(pCommith->niters, sizeof(SCommitIter)); @@ -452,7 +453,7 @@ static void tsdbResetCommitFile(SCommitH *pCommith) { static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { SDiskID did; - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(fid, &(pCommith->rtn)), &did) < 0) { @@ -583,7 +584,7 @@ int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray uint32_t tlen; SBlockInfo *pBlkInfo; int64_t offset; - SBlock * pBlock; + SBlock *pBlock; memset(pIdx, 0, sizeof(*pIdx)); @@ -1130,7 +1131,7 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf) { - STsdbCfg * pCfg = REPO_CFG(pRepo); + STsdbCfg *pCfg = REPO_CFG(pRepo); SBlockData *pBlockData; int64_t offset = 0; int rowsToWrite = pDataCols->numOfRows; @@ -1147,7 +1148,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * // Get # of cols not all NULL(not including key column) int nColsNotAllNull = 0; for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column - SDataCol * pDataCol = pDataCols->cols + ncol; + SDataCol *pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it @@ -1188,7 +1189,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * // All not NULL columns finish if (ncol != 0 && tcol >= nColsNotAllNull) break; - SDataCol * pDataCol = pDataCols->cols + ncol; + SDataCol *pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + tcol; if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; @@ -1212,7 +1213,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * } #endif - void * tptr; + void *tptr; // Make room if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { @@ -1278,7 +1279,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols); - tsdbDebug("vgId:%d uid:%"PRId64" a block of data is written to file %s, offset %" PRId64 + tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); @@ -1294,9 +1295,9 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo } static int tsdbWriteBlockInfo(SCommitH *pCommih) { - SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); + SDFile *pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); SBlockIdx blkIdx; - STable * pTable = TSDB_COMMIT_TABLE(pCommih); + STable *pTable = TSDB_COMMIT_TABLE(pCommih); if (tsdbWriteBlockInfoImpl(pHeadf, pTable, pCommih->aSupBlk, pCommih->aSubBlk, (void **)(&(TSDB_COMMIT_BUF(pCommih))), &blkIdx) < 0) { @@ -1316,11 +1317,11 @@ static int tsdbWriteBlockInfo(SCommitH *pCommih) { } static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); SMergeInfo mInfo; int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); - SDFile * pDFile; + SDFile *pDFile; bool isLast; SBlock block; @@ -1349,16 +1350,16 @@ static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLi } static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; - SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; TSKEY keyLimit; int16_t colId = PRIMARYKEY_TIMESTAMP_COL_ID; SMergeInfo mInfo; SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; SBlock block, supBlock; - SDFile * pDFile; + SDFile *pDFile; if (bidx == nBlocks - 1) { keyLimit = pCommith->maxKey; @@ -1474,10 +1475,10 @@ static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool isLastOneBlock) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdbCfg *pCfg = REPO_CFG(pRepo); SBlock block; - SDFile * pDFile; + SDFile *pDFile; bool isLast; int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); @@ -1598,7 +1599,7 @@ static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { } static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdbCfg *pCfg = REPO_CFG(pRepo); int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 6bbf3b959d..c748907d6c 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -29,6 +29,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { cfg.vgId = pVnodeCfg->vgId; cfg.pDnode = pVnodeCfg->pDnode; cfg.pTfs = pVnodeCfg->pTfs; + cfg.dbId = pVnodeCfg->dbId; } // Validate options diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index a45a9d5a72..f541834bec 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -93,6 +93,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { if (pTbCfg->type == META_CHILD_TABLE) { pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid); if (pStbCfg == NULL) { + code = TSDB_CODE_VND_TB_NOT_EXIST; goto _exit; } @@ -116,9 +117,11 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { msgLen = sizeof(STableMetaRsp) + sizeof(SSchema) * (nCols + nTagCols); pTbMetaMsg = (STableMetaRsp *)rpcMallocCont(msgLen); if (pTbMetaMsg == NULL) { + code = TSDB_CODE_VND_OUT_OF_MEMORY; goto _exit; } + pTbMetaMsg->dbId = htobe64(pVnode->config.dbId); memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName)); strcpy(pTbMetaMsg->tbName, pReq->tbName); if (pTbCfg->type == META_CHILD_TABLE) { diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 4d048bc6e2..46e60329ae 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -16,19 +16,6 @@ #include "tq.h" #include "vnd.h" -#if 0 -int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { - switch (pMsg->msgType) { - case TDMT_VND_MQ_SET_CUR: - if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) { - // TODO: handle error - } - break; - } - return 0; -} -#endif - int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { SRpcMsg *pMsg; @@ -36,11 +23,12 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; - taosEncodeFixedU64(&pBuf, ver); + taosEncodeFixedI64(&pBuf, ver); if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { + /*ASSERT(false);*/ // TODO: handle error } } @@ -55,7 +43,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vCreateTbReq; SVCreateTbBatchReq vCreateTbBatchReq; - void * ptr = vnodeMalloc(pVnode, pMsg->contLen); + void *ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -64,8 +52,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { memcpy(ptr, pMsg->pCont, pMsg->contLen); // todo: change the interface here - uint64_t ver; - taosDecodeFixedU64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); + int64_t ver; + taosDecodeFixedI64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) { // TODO: handle error } @@ -132,7 +120,6 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } break; case TDMT_VND_MQ_REB: { if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(ptr, sizeof(SMsgHead))) < 0) { - } } break; default: diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 1d23f333b2..049b69991f 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -13,3 +13,4 @@ add_subdirectory(function) add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) +add_subdirectory(nodes) \ No newline at end of file diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 8f24374387..5857437f9d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -27,7 +27,7 @@ extern "C" { #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 #define CTG_DEFAULT_CACHE_DB_NUMBER 20 -#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 10000 +#define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000 #define CTG_DEFAULT_RENT_SECOND 10 #define CTG_DEFAULT_RENT_SLOT_SIZE 10 @@ -47,9 +47,19 @@ enum { CTG_RENT_STABLE, }; +enum { + CTG_ACT_UPDATE_VG = 0, + CTG_ACT_UPDATE_TBL, + CTG_ACT_REMOVE_DB, + CTG_ACT_REMOVE_STB, + CTG_ACT_REMOVE_TBL, + CTG_ACT_MAX +}; + typedef struct SCtgDebug { bool lockDebug; bool cacheDebug; + bool apiDebug; uint32_t showCachePeriodSec; } SCtgDebug; @@ -65,7 +75,7 @@ typedef struct SCtgDBCache { SRWLatch vgLock; uint64_t dbId; int8_t deleted; - SDBVgroupInfo *vgInfo; + SDBVgInfo *vgInfo; SCtgTbMetaCache tbCache; } SCtgDBCache; @@ -85,7 +95,6 @@ typedef struct SCtgRentMgmt { typedef struct SCatalog { uint64_t clusterId; - SRWLatch dbLock; SHashObj *dbCache; //key:dbname, value:SCtgDBCache SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; @@ -96,7 +105,8 @@ typedef struct SCtgApiStat { } SCtgApiStat; typedef struct SCtgRuntimeStat { - + uint64_t qNum; + uint64_t qDoneNum; } SCtgRuntimeStat; typedef struct SCtgCacheStat { @@ -109,15 +119,70 @@ typedef struct SCatalogStat { SCtgCacheStat cache; } SCatalogStat; +typedef struct SCtgUpdateVgMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t dbId; + SDBVgInfo* dbInfo; +} SCtgUpdateVgMsg; + +typedef struct SCtgUpdateTblMsg { + SCatalog* pCtg; + STableMetaOutput* output; +} SCtgUpdateTblMsg; + +typedef struct SCtgRemoveDBMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t dbId; +} SCtgRemoveDBMsg; + +typedef struct SCtgRemoveStbMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + uint64_t dbId; + uint64_t suid; +} SCtgRemoveStbMsg; + +typedef struct SCtgMetaAction { + int32_t act; + void *data; +} SCtgMetaAction; + +typedef struct SCtgQNode { + SCtgMetaAction action; + struct SCtgQNode *next; +} SCtgQNode; + typedef struct SCatalogMgmt { bool exit; SRWLatch lock; + SRWLatch qlock; + SCtgQNode *head; + SCtgQNode *tail; + tsem_t sem; + uint64_t qRemainNum; + pthread_t updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; } SCatalogMgmt; typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); +typedef int32_t (*ctgActFunc)(SCtgMetaAction *); + +typedef struct SCtgAction { + int32_t actId; + char name[32]; + ctgActFunc func; +} SCtgAction; + +#define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.qRemainNum, 1) +#define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.qRemainNum, 1) + +#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1) +#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) @@ -130,20 +195,26 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0) #define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE)) -#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) +#define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) -#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgError(param, ...) qError("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCatalog, __VA_ARGS__) +#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) +#define CTG_DB_NOT_EXIST(code) (code == TSDB_CODE_MND_DB_NOT_EXIST) + +#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCtg, __VA_ARGS__) #define CTG_LOCK_DEBUG(...) do { if (gCTGDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) #define CTG_CACHE_DEBUG(...) do { if (gCTGDebug.cacheDebug) { qDebug(__VA_ARGS__); } } while (0) +#define CTG_API_DEBUG(...) do { if (gCTGDebug.apiDebug) { qDebug(__VA_ARGS__); } } while (0) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 +#define CTG_IS_LOCKED(_lock) atomic_load_32((_lock)) + #define CTG_LOCK(type, _lock) do { \ if (CTG_READ == (type)) { \ assert(atomic_load_32((_lock)) >= 0); \ @@ -181,8 +252,8 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define CTG_API_ENTER() do { CTG_LOCK(CTG_READ, &ctgMgmt.lock); if (atomic_load_8(&ctgMgmt.exit)) { CTG_RET(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) -#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_RET(__code); } while (0) +#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0) +#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8(&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6ecff87a89..f1c2395de4 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -18,9 +18,75 @@ #include "tname.h" #include "catalogInt.h" -SCatalogMgmt ctgMgmt = {0}; +int32_t ctgActUpdateVg(SCtgMetaAction *action); +int32_t ctgActUpdateTbl(SCtgMetaAction *action); +int32_t ctgActRemoveDB(SCtgMetaAction *action); +int32_t ctgActRemoveStb(SCtgMetaAction *action); +int32_t ctgActRemoveTbl(SCtgMetaAction *action); +SCatalogMgmt gCtgMgmt = {0}; SCtgDebug gCTGDebug = {0}; +SCtgAction gCtgAction[CTG_ACT_MAX] = {{ + CTG_ACT_UPDATE_VG, + "update vgInfo", + ctgActUpdateVg + }, + { + CTG_ACT_UPDATE_TBL, + "update tbMeta", + ctgActUpdateTbl + }, + { + CTG_ACT_REMOVE_DB, + "remove DB", + ctgActRemoveDB + }, + { + CTG_ACT_REMOVE_STB, + "remove stbMeta", + ctgActRemoveStb + }, + { + CTG_ACT_REMOVE_TBL, + "remove tbMeta", + ctgActRemoveTbl + } +}; + +int32_t ctgDbgEnableDebug(char *option) { + if (0 == strcasecmp(option, "lock")) { + gCTGDebug.lockDebug = true; + qDebug("lock debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "cache")) { + gCTGDebug.cacheDebug = true; + qDebug("cache debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "api")) { + gCTGDebug.apiDebug = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + + qError("invalid debug option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + +int32_t ctgDbgGetStatNum(char *option, void *res) { + if (0 == strcasecmp(option, "runtime.qDoneNum")) { + *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum); + return TSDB_CODE_SUCCESS; + } + + qError("invalid stat option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) { return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; @@ -44,25 +110,25 @@ int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) { return num; } -int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type) { - if (NULL == pCatalog || NULL == pCatalog->dbCache) { +int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) { + if (NULL == pCtg || NULL == pCtg->dbCache) { return 0; } switch (type) { case CTG_DBG_DB_NUM: - return (int32_t)taosHashGetSize(pCatalog->dbCache); + return (int32_t)taosHashGetSize(pCtg->dbCache); case CTG_DBG_DB_RENT_NUM: - return ctgDbgGetRentNum(&pCatalog->dbRent); + return ctgDbgGetRentNum(&pCtg->dbRent); case CTG_DBG_STB_RENT_NUM: - return ctgDbgGetRentNum(&pCatalog->stbRent); + return ctgDbgGetRentNum(&pCtg->stbRent); default: break; } SCtgDBCache *dbCache = NULL; int32_t num = 0; - void *pIter = taosHashIterate(pCatalog->dbCache, NULL); + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { dbCache = (SCtgDBCache *)pIter; switch (type) { @@ -76,7 +142,7 @@ int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type) { ctgError("invalid type:%d", type); break; } - pIter = taosHashIterate(pCatalog->dbCache, pIter); + pIter = taosHashIterate(pCtg->dbCache, pIter); } return num; @@ -108,79 +174,57 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { -void ctgDbgShowClusterCache(struct SCatalog* pCatalog) { - if (NULL == pCatalog) { +void ctgDbgShowClusterCache(SCatalog* pCtg) { + if (NULL == pCtg) { return; } - CTG_CACHE_DEBUG("## cluster %"PRIx64" %p cache Info ##", pCatalog->clusterId, pCatalog); - CTG_CACHE_DEBUG("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_META_NUM), - ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_STB_RENT_NUM)); + CTG_CACHE_DEBUG("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); + CTG_CACHE_DEBUG("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), + ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); - ctgDbgShowDBCache(pCatalog->dbCache); -} - -int32_t ctgInitDBCache(struct SCatalog* pCatalog) { - if (NULL == pCatalog->dbCache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - if (NULL == cache) { - ctgError("taosHashInit %d failed", CTG_DEFAULT_CACHE_DB_NUMBER); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache, NULL, cache)) { - taosHashCleanup(cache); - } - } - - return TSDB_CODE_SUCCESS; + ctgDbgShowDBCache(pCtg->dbCache); } -int32_t ctgInitTbMetaCache(struct SCatalog* pCatalog, SCtgDBCache *dbCache) { - if (NULL == dbCache->tbCache.metaCache) { - if (dbCache->deleted) { - ctgInfo("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - SHashObj *metaCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == metaCache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.metaCache, NULL, metaCache)) { - taosHashCleanup(metaCache); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgInitStbCache(struct SCatalog* pCatalog, SCtgDBCache *dbCache) { - if (NULL == dbCache->tbCache.stbCache) { - if (dbCache->deleted) { - ctgInfo("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } +void ctgPopAction(SCtgMetaAction **action) { + SCtgQNode *orig = gCtgMgmt.head; - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == cache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } + SCtgQNode *node = gCtgMgmt.head->next; + gCtgMgmt.head = gCtgMgmt.head->next; - if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.stbCache, NULL, cache)) { - taosHashCleanup(cache); - } + CTG_QUEUE_SUB(); + + tfree(orig); + + *action = &node->action; +} + + +int32_t ctgPushAction(SCtgMetaAction *action) { + SCtgQNode *node = calloc(1, sizeof(SCtgQNode)); + if (NULL == node) { + qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + CTG_RET(TSDB_CODE_CTG_MEM_ERROR); } + + node->action = *action; + + CTG_LOCK(CTG_WRITE, &gCtgMgmt.qlock); + gCtgMgmt.tail->next = node; + gCtgMgmt.tail = node; + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.qlock); + + CTG_QUEUE_ADD(); + CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum); + + tsem_post(&gCtgMgmt.sem); return TSDB_CODE_SUCCESS; } - void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { if (NULL == mgmt->slots) { return; @@ -214,93 +258,183 @@ void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_UNLOCK(CTG_WRITE, &cache->metaLock); } +void ctgFreeVgInfo(SDBVgInfo *vgInfo) { + if (NULL == vgInfo) { + return; + } + + if (vgInfo->vgHash) { + taosHashCleanup(vgInfo->vgHash); + vgInfo->vgHash = NULL; + } + + tfree(vgInfo); +} + void ctgFreeDbCache(SCtgDBCache *dbCache) { if (NULL == dbCache) { return; } - atomic_store_8(&dbCache->deleted, 1); - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (dbCache->vgInfo) { - - if (dbCache->vgInfo->vgHash) { - taosHashCleanup(dbCache->vgInfo->vgHash); - dbCache->vgInfo->vgHash = NULL; - } - - tfree(dbCache->vgInfo); - } + ctgFreeVgInfo (dbCache->vgInfo); CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); ctgFreeTableMetaCache(&dbCache->tbCache); } -void ctgFreeHandle(struct SCatalog* pCatalog) { - ctgFreeMetaRent(&pCatalog->dbRent); - ctgFreeMetaRent(&pCatalog->stbRent); - if (pCatalog->dbCache) { - void *pIter = taosHashIterate(pCatalog->dbCache, NULL); +void ctgFreeHandle(SCatalog* pCtg) { + ctgFreeMetaRent(&pCtg->dbRent); + ctgFreeMetaRent(&pCtg->stbRent); + + if (pCtg->dbCache) { + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; + atomic_store_8(&dbCache->deleted, 1); + ctgFreeDbCache(dbCache); - - pIter = taosHashIterate(pCatalog->dbCache, pIter); + + pIter = taosHashIterate(pCtg->dbCache, pIter); } - taosHashCleanup(pCatalog->dbCache); + taosHashCleanup(pCtg->dbCache); } - free(pCatalog); + free(pCtg); } -int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbFName, SCtgDBCache **dbCache, bool *inCache) { - if (NULL == pCatalog->dbCache) { + +int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { + CTG_LOCK(CTG_READ, &dbCache->vgLock); + + if (dbCache->deleted) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + + *inCache = false; + return TSDB_CODE_SUCCESS; + } + + + if (NULL == dbCache->vgInfo) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + *inCache = false; + ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); + return TSDB_CODE_SUCCESS; + } + + *inCache = true; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (dbCache->deleted) { + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + return TSDB_CODE_SUCCESS; +} + +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { + taosHashRelease(pCtg->dbCache, dbCache); +} + +void ctgReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); +} + +void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); +} + + +int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { + SCtgDBCache *dbCache = NULL; + if (acquire) { + dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); + } else { + dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); + } + + if (NULL == dbCache) { + *pCache = NULL; + ctgDebug("db not in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + if (dbCache->deleted) { + if (acquire) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + ctgDebug("db is removing from cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); +} + +int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); +} + + +int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { + if (NULL == pCtg->dbCache) { + *pCache = NULL; *inCache = false; ctgWarn("empty db cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } - SCtgDBCache *cache = NULL; - - while (true) { - cache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - - if (NULL == cache) { - *inCache = false; - ctgWarn("not in db vgroup cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &cache->vgLock); - if (NULL == cache->vgInfo) { - CTG_UNLOCK(CTG_READ, &cache->vgLock); - taosHashRelease(pCatalog->dbCache, cache); - ctgWarn("db cache vgInfo is NULL, dbFName:%s", dbFName); - - continue; - } - - break; + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + *pCache = NULL; + *inCache = false; + return TSDB_CODE_SUCCESS; + } + + ctgAcquireVgInfo(pCtg, dbCache, inCache); + if (!(*inCache)) { + ctgReleaseDBCache(pCtg, dbCache); + + *pCache = NULL; + return TSDB_CODE_SUCCESS; } - *dbCache = cache; + *pCache = dbCache; *inCache = true; - ctgDebug("Got db vgroup from cache, dbFName:%s", dbFName); + ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } -int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { +int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { char *msg = NULL; int32_t msgLen = 0; - ctgDebug("try to get db vgroup from mnode, db:%s", input->db); + ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)](input, &msg, 0, &msgLen); if (code) { @@ -318,6 +452,11 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { + if (CTG_DB_NOT_EXIST(rpcRsp.code)) { + ctgDebug("db not exist in mnode, dbFName:%s", input->db); + return rpcRsp.code; + } + ctgError("error rsp for use db, code:%s, db:%s", tstrerror(rpcRsp.code), input->db); CTG_ERR_RET(rpcRsp.code); } @@ -328,31 +467,32 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp CTG_ERR_RET(code); } + ctgDebug("Got db vgInfo from mnode, dbFName:%s", input->db); + return TSDB_CODE_SUCCESS; } -int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, char* tbName, int32_t *exist) { - if (NULL == pCatalog->dbCache) { +int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { + if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { *exist = 0; - ctgWarn("db not exist in cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } - size_t sz = 0; CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == tbMeta) { - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); *exist = 0; ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -361,7 +501,7 @@ int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, cha *exist = 1; - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -369,29 +509,22 @@ int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, cha } -int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { - if (NULL == pCatalog->dbCache) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { + if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); *pTableMeta = NULL; - SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, db, strlen(db)); + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { *exist = 0; - ctgWarn("no db cache, dbFName:%s, tbName:%s", db, pTableName->tname); - return TSDB_CODE_SUCCESS; - } - - if (NULL == dbCache->tbCache.metaCache) { - *exist = 0; - taosHashRelease(pCatalog->dbCache, dbCache); - ctgWarn("empty tbmeta cache, dbFName:%s, tbName:%s", db, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -402,8 +535,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN if (NULL == *pTableMeta) { *exist = 0; - taosHashRelease(pCatalog->dbCache, dbCache); - ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", db, pTableName->tname); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -412,8 +545,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { - taosHashRelease(pCatalog->dbCache, dbCache); - ctgDebug("Got tbmeta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, db, pTableName->tname); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got tbl from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -422,8 +555,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); - ctgError("stable not in stbCache, suid:%"PRIx64, tbMeta->suid); + ctgReleaseDBCache(pCtg, dbCache); + ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); tfree(*pTableMeta); *exist = 0; return TSDB_CODE_SUCCESS; @@ -431,17 +564,17 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN if ((*stbMeta)->suid != tbMeta->suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); tfree(*pTableMeta); ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); + int32_t metaSize = CTG_META_SIZE(*stbMeta); *pTableMeta = realloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); ctgError("realloc size[%d] failed", metaSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -450,50 +583,53 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", db, pTableName->tname); + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableName, int32_t *tbType) { - if (NULL == pCatalog->dbCache) { +int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType) { + if (NULL == pCtg->dbCache) { ctgWarn("empty db cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char dbName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbName); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); - SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - ctgInfo("db not in cache, dbFName:%s", dbName); return TSDB_CODE_SUCCESS; } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == pTableMeta) { - ctgWarn("tbmeta not in cache, dbFName:%s, tbName:%s", dbName, pTableName->tname); - taosHashRelease(pCatalog->dbCache, dbCache); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); + ctgReleaseDBCache(pCtg, dbCache); return TSDB_CODE_SUCCESS; } *tbType = atomic_load_8(&pTableMeta->tableType); - taosHashRelease(dbCache->tbCache.metaCache, dbCache); - taosHashRelease(pCatalog->dbCache, dbCache); + taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbName, pTableName->tname, *tbType); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, pTableName->tname, *tbType); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; @@ -539,15 +675,15 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransport return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - return ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); + return ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); } -int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { +int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -607,12 +743,12 @@ int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { return TSDB_CODE_SUCCESS; } -int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray** vgroupList) { +int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { SHashObj *vgroupHash = NULL; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + int32_t vgNum = taosHashGetSize(vgHash); vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { @@ -620,23 +756,23 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + void *pIter = taosHashIterate(vgHash, NULL); while (pIter) { vgInfo = pIter; if (NULL == taosArrayPush(vgList, vgInfo)) { ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); + taosHashCancelIterate(vgHash, pIter); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - pIter = taosHashIterate(dbInfo->vgHash, pIter); + pIter = taosHashIterate(vgHash, pIter); vgInfo = NULL; } - *vgroupList = vgList; - vgList = NULL; + *pList = vgList; - ctgDebug("Got vg list from DB, vgNum:%d", vgNum); + ctgDebug("Got vgList from cache, vgNum:%d", vgNum); return TSDB_CODE_SUCCESS; @@ -649,7 +785,7 @@ _return: CTG_RET(code); } -int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { int32_t code = 0; int32_t vgNum = taosHashGetSize(dbInfo->vgHash); @@ -693,7 +829,6 @@ int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbIn CTG_RET(code); } -#if 1 int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { return -1; @@ -713,29 +848,6 @@ int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { return 0; } } -#else - -int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { - return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { - return 1; - } else { - return 0; - } -} - -int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { - if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { - return -1; - } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { - return 1; - } else { - return 0; - } -} - -#endif int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; @@ -932,38 +1044,55 @@ int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t si return TSDB_CODE_SUCCESS; } -int32_t ctgAddDBCache(struct SCatalog *pCatalog, const char *dbFName, SCtgDBCache *dbCache) { +int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { int32_t code = 0; - code = taosHashPut(pCatalog->dbCache, dbFName, strlen(dbFName), dbCache, sizeof(SCtgDBCache)); + SCtgDBCache newDBCache = {0}; + newDBCache.dbId = dbId; + + newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.metaCache) { + ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.stbCache) { + ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache)); if (code) { if (HASH_NODE_EXIST(code)) { ctgDebug("db already in cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; + goto _return; } ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - SDbVgVersion vgVersion = {.dbId = dbCache->dbId, .vgVersion = dbCache->vgInfo ? dbCache->vgInfo->vgVersion : -1}; + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - ctgDebug("dbCache added, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbCache->dbId); + ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion))); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); return TSDB_CODE_SUCCESS; _return: - ctgFreeDbCache(dbCache); + ctgFreeDbCache(&newDBCache); CTG_RET(code); } -void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache) { +void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { void *pIter = taosHashIterate(cache->stbCache, NULL); @@ -971,7 +1100,7 @@ void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache uint64_t *suid = NULL; taosHashGetKey(pIter, (void **)&suid, NULL); - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCatalog->stbRent, *suid, ctgSTableVersionCompare)) { + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgSTableVersionCompare)) { ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); } @@ -979,232 +1108,256 @@ void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache } } CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - - ctgFreeTableMetaCache(cache); } -int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, SCtgDBCache *dbCache, const char* dbFName) { - if (taosHashRemove(pCatalog->dbCache, dbFName, strlen(dbFName))) { +int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { + uint64_t dbId = dbCache->dbId; + + ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + atomic_store_8(&dbCache->deleted, 1); + + ctgRemoveStbRent(pCtg, &dbCache->tbCache); + + ctgFreeDbCache(dbCache); + + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionCompare)); + + ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } - atomic_store_8(&dbCache->deleted, 1); - - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (dbCache->vgInfo) { - ctgInfo("cleanup db vgInfo, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); - - if (dbCache->vgInfo->vgHash) { - taosHashCleanup(dbCache->vgInfo->vgHash); - } - - tfree(dbCache->vgInfo); - } - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - - ctgRemoveAndFreeTableMeta(pCatalog, &dbCache->tbCache); - - ctgInfo("db removed from cache, dbFName:%s, uid:%"PRIx64, dbFName, dbCache->dbId); - - CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->dbRent, dbCache->dbId, ctgDbVgVersionCompare)); - - ctgDebug("db removed from rent, dbFName:%s, uid:%"PRIx64, dbFName, dbCache->dbId); + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireDBCache(struct SCatalog* pCatalog, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { +int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - - CTG_LOCK(CTG_WRITE, &pCatalog->dbLock); + ctgGetDBCache(pCtg, dbFName, &dbCache); - while (true) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - if (dbCache) { - // TODO OPEN IT + if (dbCache) { + // TODO OPEN IT #if 0 - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } #else - if (0 == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbId && (dbCache->dbId == 0)) { - dbCache->dbId = dbId; - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#endif - code = ctgValidateAndRemoveDb(pCatalog, dbCache, dbFName); - taosHashRelease(pCatalog->dbCache, dbCache); - dbCache = NULL; - if (code) { - if (TSDB_CODE_CTG_DB_DROPPED == code) { - continue; - } - - CTG_ERR_JRET(code); - } + if (0 == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; } - SCtgDBCache newDBCache = {0}; - newDBCache.dbId = dbId; + if (dbId && (dbCache->dbId == 0)) { + dbCache->dbId = dbId; + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } - CTG_ERR_JRET(ctgAddDBCache(pCatalog, dbFName, &newDBCache)); + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#endif + CTG_ERR_RET(ctgRemoveDB(pCtg, dbCache, dbFName)); + } + + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); + + ctgGetDBCache(pCtg, dbFName, &dbCache); + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { + int32_t code = 0; + SDBVgInfo* dbInfo = *pDbInfo; + + if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } -_return: + bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion}; - if (dbCache) { - taosHashRelease(pCatalog->dbCache, dbCache); + SCtgDBCache *dbCache = NULL; + CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); + SDBVgInfo *vgInfo = NULL; + CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + + if (dbCache->vgInfo) { + if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { + ctgInfo("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + ctgWReleaseVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + ctgFreeVgInfo(dbCache->vgInfo); + } + + dbCache->vgInfo = dbInfo; + + *pDbInfo = NULL; + + ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + + ctgWReleaseVgInfo(dbCache); + + dbCache = NULL; + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); CTG_RET(code); } +int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { + SCtgTbMetaCache *tbCache = &dbCache->tbCache; -int32_t ctgUpdateTbMetaImpl(struct SCatalog *pCatalog, SCtgTbMetaCache *tbCache, char *dbFName, char *tbName, STableMeta *meta, int32_t metaSize) { - CTG_LOCK(CTG_READ, &tbCache->metaLock); + CTG_LOCK(CTG_READ, &tbCache->metaLock); + if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + int8_t origType = 0; + uint64_t origSuid = 0; + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { + origType = orig->tableType; + origSuid = orig->suid; + + if (origType == TSDB_SUPER_TABLE && ((!isStb) || origSuid != meta->suid)) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgSTableVersionCompare); + } + } + + if (isStb) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + } + if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { + if (isStb) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + } + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - return TSDB_CODE_SUCCESS; -} - -int32_t ctgUpdateStbMetaImpl(struct SCatalog *pCatalog, SCtgTbMetaCache *tbCache, char *dbFName, char *tbName, STableMeta *meta, int32_t metaSize) { - bool newAdded = false; - int32_t code = 0; - SSTableMetaVersion metaRent = {.suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); - - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - STableMeta *orig = taosHashAcquire(tbCache->metaCache, tbName, strlen(tbName)); - if (orig) { - if (orig->suid != meta->suid) { - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, db:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - } - - ctgMetaRentRemove(&pCatalog->stbRent, orig->suid, ctgSTableVersionCompare); - } - - taosHashRelease(tbCache->metaCache, orig); + if (!isStb) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, tbCache, dbFName, tbName, meta, metaSize)); - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (taosHashPutExt(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES, &newAdded) != 0) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (isStb && origSuid == meta->suid) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; + } + + STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + + SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); - ctgDebug("update stable to cache, suid:%"PRIx64, meta->suid); - - if (newAdded) { - CTG_ERR_RET(ctgMetaRentAdd(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); - } else { - CTG_ERR_RET(ctgMetaRentUpdate(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion), ctgSTableVersionCompare)); + return TSDB_CODE_SUCCESS; +} + +int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { + *dst = malloc(sizeof(SDBVgInfo)); + if (NULL == *dst) { + qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + memcpy(*dst, src, sizeof(SDBVgInfo)); + + size_t hashSize = taosHashGetSize(src->vgHash); + (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == (*dst)->vgHash) { + qError("taosHashInit %d failed", (int32_t)hashSize); + tfree(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + int32_t *vgId = NULL; + void *pIter = taosHashIterate(src->vgHash, NULL); + while (pIter) { + taosHashGetKey(pIter, (void **)&vgId, NULL); + + if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) { + qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); + taosHashCancelIterate(src->vgHash, pIter); + taosHashCleanup((*dst)->vgHash); + tfree(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + pIter = taosHashIterate(src->vgHash, pIter); + } + + return TSDB_CODE_SUCCESS; - -_return: - - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_RET(code); } -int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { - int32_t code = 0; - SCtgDBCache *dbCache = NULL; - if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - CTG_ERR_RET(ctgInitDBCache(pCatalog)); - - CTG_ERR_JRET(ctgAcquireDBCache(pCatalog, output->dbFName, output->dbId, &dbCache)); - - CTG_ERR_JRET(ctgInitTbMetaCache(pCatalog, dbCache)); - CTG_ERR_JRET(ctgInitStbCache(pCatalog, dbCache)); - - if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); - } - - if (CTG_IS_META_CTABLE(output->metaType)) { - goto _return; - } - - if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { - ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags); - - if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { - CTG_ERR_JRET(ctgUpdateStbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->tbName, output->tbMeta, tbSize)); - } else { - CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->tbName, output->tbMeta, tbSize)); - } - -_return: - - if (dbCache) { - taosHashRelease(pCatalog->dbCache, dbCache); - CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); - } - - CTG_RET(code); -} - -int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache) { +int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { bool inCache = false; + int32_t code = 0; if (!forceUpdate) { - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbFName, dbCache, &inCache)); + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); if (inCache) { return TSDB_CODE_SUCCESS; } - - ctgDebug("failed to get DB vgroupInfo from cache, dbName:%s, load it from mnode, update:%d", dbFName, forceUpdate); } SUseDbOutput DbOut = {0}; @@ -1213,221 +1366,490 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - while (true) { - CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); - CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbFName, DbOut.dbId, DbOut.dbVgroup)); - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbFName, dbCache, &inCache)); + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut)); - if (!inCache) { - ctgWarn("can't get db vgroup from cache, will retry, db:%s", dbFName); - continue; + CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo)); + + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG}; + SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); + ctgFreeVgInfo(DbOut.dbVgroup); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->pCtg = pCtg; + msg->dbId = DbOut.dbId; + msg->dbInfo = DbOut.dbVgroup; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(*pInfo); + tfree(msg); + + *pInfo = DbOut.dbVgroup; + + CTG_RET(code); +} + + +int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { + *pOutput = malloc(sizeof(STableMetaOutput)); + if (NULL == *pOutput) { + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(*pOutput, output, sizeof(STableMetaOutput)); + + if (output->tbMeta) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + (*pOutput)->tbMeta = malloc(metaSize); + if (NULL == (*pOutput)->tbMeta) { + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + tfree(*pOutput); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - break; + memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); } return TSDB_CODE_SUCCESS; } -int32_t ctgValidateAndRemoveStbMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid, bool *removed) { - *removed = false; - - SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); - if (NULL == dbCache) { - ctgInfo("db not exist in dbCache, may be removed, db:%s", dbName); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - if (taosHashRemove(dbCache->tbCache.stbCache, &suid, sizeof(suid))) { - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); - ctgInfo("stb not exist in stbCache, may be removed, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, stbName, strlen(stbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); - ctgError("stb not exist in cache, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - - taosHashRelease(pCatalog->dbCache, dbCache); - - *removed = true; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { +int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable, STableMetaOutput **pOutput) { + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; int32_t code = 0; - CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); - - STableMetaOutput voutput = {0}; - STableMetaOutput moutput = {0}; - STableMetaOutput *output = &voutput; + CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); + SCtgUpdateTblMsg *msg = NULL; + STableMetaOutput moutput = {0}; + STableMetaOutput *output = malloc(sizeof(STableMetaOutput)); + if (NULL == output) { + ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + if (CTG_IS_STABLE(isSTable)) { - ctgDebug("will renew tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode - CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTransporter, pMgmtEps, pTableName, output)); - if (CTG_IS_META_NULL(moutput.metaType)) { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); - } else { - output = &moutput; + if (CTG_IS_META_NULL(output->metaType)) { + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); } } else { - ctgDebug("will renew tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); // if get from vnode failed or no table meta, will not try mnode - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); - if (CTG_IS_META_TABLE(voutput.metaType) && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) { - ctgDebug("will continue to renew tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); - - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); + if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); - voutput.metaType = moutput.metaType; + tfree(output->tbMeta); - tfree(voutput.tbMeta); - voutput.tbMeta = moutput.tbMeta; - moutput.tbMeta = NULL; - } else if (CTG_IS_META_BOTH(voutput.metaType)) { + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, output)); + } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCatalog, voutput.dbFName, voutput.tbName, &exist)); + CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, &moutput)); if (CTG_IS_META_NULL(moutput.metaType)) { - SET_META_TYPE_NULL(voutput.metaType); + SET_META_TYPE_NULL(output->metaType); } - tfree(voutput.tbMeta); - voutput.tbMeta = moutput.tbMeta; + tfree(output->tbMeta); + output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { - tfree(voutput.tbMeta); + tfree(output->tbMeta); - SET_META_TYPE_CTABLE(voutput.metaType); + SET_META_TYPE_CTABLE(output->metaType); } } } if (CTG_IS_META_NULL(output->metaType)) { - ctgError("no tablemeta got, tbNmae:%s", tNameGetTableName(pTableName)); + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pTableName)); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, output)); + if (CTG_IS_META_TABLE(output->metaType)) { + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, output->tbMeta->tableType); + } else { + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); + } + + if (pOutput) { + CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); + } + + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; + msg = malloc(sizeof(SCtgUpdateTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->output = output; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; _return: - tfree(voutput.tbMeta); - tfree(moutput.tbMeta); + tfree(output->tbMeta); + tfree(output); + tfree(msg); CTG_RET(code); } -int32_t ctgGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { +int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; + int32_t code = 0; if (!forceUpdate) { - CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist)); if (exist && CTG_TBTYPE_MATCH(isSTable, (*pTableMeta)->tableType)) { return TSDB_CODE_SUCCESS; } + + tfree(*pTableMeta); } else if (CTG_IS_UNKNOWN_STABLE(isSTable)) { int32_t tbType = 0; - CTG_ERR_RET(ctgGetTableTypeFromCache(pCatalog, pTableName, &tbType)); + CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType)); CTG_SET_STABLE(isSTable, tbType); } - CTG_ERR_RET(ctgRenewTableMetaImpl(pCatalog, pRpc, pMgmtEps, pTableName, isSTable)); + STableMetaOutput *output = NULL; - CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); + if (CTG_IS_META_TABLE(output->metaType)) { + *pTableMeta = output->tbMeta; + goto _return; + } + + if (CTG_IS_META_BOTH(output->metaType)) { + memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + + *pTableMeta = output->tbMeta; + goto _return; + } + + if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { + ctgError("invalid metaType:%d", output->metaType); + tfree(output->tbMeta); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + // HANDLE ONLY CHILD TABLE META + + SName stbName = *pTableName; + strcpy(stbName.tname, output->tbName); + + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); if (0 == exist) { - ctgError("renew tablemeta succeed but get from cache failed, may be deleted, tbName:%s", tNameGetTableName(pTableName)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); + CTG_ERR_JRET(TSDB_CODE_VND_TB_NOT_EXIST); + } + + memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + +_return: + + tfree(output); + + if (*pTableMeta) { + ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); + } + + CTG_RET(code); +} + + + +int32_t ctgActUpdateVg(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateVgMsg *msg = action->data; + + CTG_ERR_JRET(ctgUpdateDBVgInfo(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); + +_return: + + tfree(msg->dbInfo); + tfree(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveDB(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveDBMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + goto _return; } + if (dbCache->dbId != msg->dbId) { + ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); + goto _return; + } + + CTG_ERR_JRET(ctgRemoveDB(pCtg, dbCache, msg->dbFName)); + +_return: + + tfree(msg); + + CTG_RET(code); +} + + +int32_t ctgActUpdateTbl(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateTblMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + STableMetaOutput* output = msg->output; + SCtgDBCache *dbCache = NULL; + + if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { + ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + + CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + } + + if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + } + +_return: + + if (output) { + tfree(output->tbMeta); + tfree(output); + } + + tfree(msg); + + CTG_RET(code); +} + + +int32_t ctgActRemoveStb(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveStbMsg *msg = action->data; + bool removed = false; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId != msg->dbId) { + ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + + ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgSTableVersionCompare)); + + ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + +_return: + + tfree(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveTbl(SCtgMetaAction *action) { + +} + + + +void* ctgUpdateThreadFunc(void* param) { + setThreadName("catalog"); + + qInfo("catalog update thread started"); + + CTG_LOCK(CTG_READ, &gCtgMgmt.lock); + + while (true) { + tsem_wait(&gCtgMgmt.sem); + + if (atomic_load_8(&gCtgMgmt.exit)) { + break; + } + + SCtgMetaAction *action = NULL; + ctgPopAction(&action); + + qDebug("process %s action", gCtgAction[action->act].name); + + (*gCtgAction[action->act].func)(action); + + CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); + } + + CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); + + qInfo("catalog update thread stopped"); + + return NULL; +} + + +int32_t ctgStartUpdateThread() { + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + CTG_ERR_RET(terrno); + } + + pthread_attr_destroy(&thAttr); return TSDB_CODE_SUCCESS; } int32_t catalogInit(SCatalogCfg *cfg) { - if (ctgMgmt.pCluster) { + if (gCtgMgmt.pCluster) { qError("catalog already initialized"); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - atomic_store_8(&ctgMgmt.exit, false); + atomic_store_8(&gCtgMgmt.exit, false); if (cfg) { - memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg)); + memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); - if (ctgMgmt.cfg.maxDBCacheNum == 0) { - ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; + if (gCtgMgmt.cfg.maxDBCacheNum == 0) { + gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; } - if (ctgMgmt.cfg.maxTblCacheNum == 0) { - ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; + if (gCtgMgmt.cfg.maxTblCacheNum == 0) { + gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; } - if (ctgMgmt.cfg.dbRentSec == 0) { - ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; + if (gCtgMgmt.cfg.dbRentSec == 0) { + gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; } - if (ctgMgmt.cfg.stbRentSec == 0) { - ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; + if (gCtgMgmt.cfg.stbRentSec == 0) { + gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } } else { - ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; - ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; - ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; - ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; + gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; + gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; + gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; + gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } - ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - if (NULL == ctgMgmt.pCluster) { + gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + if (NULL == gCtgMgmt.pCluster) { qError("taosHashInit %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stbRentSec); + CTG_ERR_RET(ctgStartUpdateThread()); + + tsem_init(&gCtgMgmt.sem, 0, 0); + + gCtgMgmt.head = calloc(1, sizeof(SCtgQNode)); + if (NULL == gCtgMgmt.head) { + qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + gCtgMgmt.tail = gCtgMgmt.head; + + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { +int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { if (NULL == catalogHandle) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == ctgMgmt.pCluster) { + if (NULL == gCtgMgmt.pCluster) { qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } @@ -1436,7 +1858,7 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { SCatalog *clusterCtg = NULL; while (true) { - SCatalog **ctg = (SCatalog **)taosHashGet(ctgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); + SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); if (ctg && (*ctg)) { *catalogHandle = *ctg; @@ -1452,10 +1874,22 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { clusterCtg->clusterId = clusterId; - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, ctgMgmt.cfg.dbRentSec, CTG_RENT_DB)); - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, ctgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); - code = taosHashPut(ctgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); + clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == clusterCtg->dbCache) { + qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == metaCache) { + qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { if (HASH_NODE_EXIST(code)) { ctgFreeHandle(clusterCtg); @@ -1482,97 +1916,82 @@ _return: CTG_RET(code); } -void catalogFreeHandle(struct SCatalog* pCatalog) { - if (NULL == pCatalog) { +void catalogFreeHandle(SCatalog* pCtg) { + if (NULL == pCtg) { return; } - if (taosHashRemove(ctgMgmt.pCluster, &pCatalog->clusterId, sizeof(pCatalog->clusterId))) { - ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCatalog->clusterId); + if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { + ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCtg->clusterId); return; } - uint64_t clusterId = pCatalog->clusterId; + uint64_t clusterId = pCtg->clusterId; - ctgFreeHandle(pCatalog); + ctgFreeHandle(pCtg); ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); } -int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { - if (NULL == pCatalog || NULL == dbName || NULL == version) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version) { CTG_API_ENTER(); - if (NULL == pCatalog->dbCache) { + if (NULL == pCtg || NULL == dbFName || NULL == version) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + if (NULL == pCtg->dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("empty db cache, dbName:%s", dbName); + ctgInfo("empty db cache, dbFName:%s", dbFName); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgDBCache *db = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); - if (NULL == db) { + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("db not in cache, dbName:%s", dbName); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_LOCK(CTG_READ, &db->vgLock); - - if (NULL == db->vgInfo) { - CTG_UNLOCK(CTG_READ, &db->vgLock); + bool inCache = false; + ctgAcquireVgInfo(pCtg, dbCache, &inCache); + if (!inCache) { + ctgReleaseDBCache(pCtg, dbCache); *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("db not in cache, dbName:%s", dbName); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - *version = db->vgInfo->vgVersion; - CTG_UNLOCK(CTG_READ, &db->vgLock); - - taosHashRelease(pCatalog->dbCache, db); + *version = dbCache->vgInfo->vgVersion; - ctgDebug("Got db vgVersion from cache, dbName:%s, vgVersion:%d", dbName, *version); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SArray** vgroupList) { - if (NULL == pCatalog || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SArray** vgroupList) { CTG_API_ENTER(); - SCtgDBCache* dbCache = NULL; - SVgroupInfo *vgInfo = NULL; + if (NULL == pCtg || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + SCtgDBCache* dbCache = NULL; int32_t code = 0; SArray *vgList = NULL; - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbFName, forceUpdate, &dbCache)); - - int32_t vgNum = (int32_t)taosHashGetSize(dbCache->vgInfo->vgHash); - vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); - if (NULL == vgList) { - ctgError("taosArrayInit %d failed", vgNum); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + SHashObj *vgHash = NULL; + SDBVgInfo *vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, forceUpdate, &dbCache, &vgInfo)); + if (dbCache) { + vgHash = dbCache->vgInfo->vgHash; + } else { + vgHash = vgInfo->vgHash; } - void *pIter = taosHashIterate(dbCache->vgInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - - if (NULL == taosArrayPush(vgList, vgInfo)) { - ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); - taosHashCancelIterate(dbCache->vgInfo->vgHash, pIter); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - pIter = taosHashIterate(dbCache->vgInfo->vgHash, pIter); - vgInfo = NULL; - } + CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, &vgList)); *vgroupList = vgList; vgList = NULL; @@ -1580,248 +1999,268 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* _return: if (dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); } - if (vgList) { - taosArrayDestroy(vgList); - vgList = NULL; + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + tfree(vgInfo); } CTG_API_LEAVE(code); } -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbFName, uint64_t dbId, SDBVgroupInfo* dbInfo) { - int32_t code = 0; - +int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo* dbInfo) { CTG_API_ENTER(); + + int32_t code = 0; - if (NULL == pCatalog || NULL == dbFName || NULL == dbInfo) { + if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion); + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG}; + SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_ERR_JRET(ctgInitDBCache(pCatalog)); - - bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion}; - - SCtgDBCache *dbCache = NULL; - CTG_ERR_JRET(ctgAcquireDBCache(pCatalog, dbFName, dbId, &dbCache)); - - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (dbCache->deleted) { - ctgInfo("db is dropping, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); - CTG_ERR_JRET(TSDB_CODE_CTG_DB_DROPPED); - } - - if (NULL == dbCache->vgInfo) { - dbCache->vgInfo = dbInfo; - } else { - if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { - ctgInfo("db vgVersion is old, dbFName:%s, vgVersion:%d, current:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); - - goto _return; - } - - if (dbCache->vgInfo->vgHash) { - ctgDebug("cleanup db vgHash, dbFName:%s", dbFName); - taosHashCleanup(dbCache->vgInfo->vgHash); - dbCache->vgInfo->vgHash = NULL; - } - - tfree(dbCache->vgInfo); - dbCache->vgInfo = dbInfo; - } - + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + msg->dbInfo = dbInfo; dbInfo = NULL; - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + action.data = msg; - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); + CTG_ERR_JRET(ctgPushAction(&action)); - ctgDebug("dbCache updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_API_LEAVE(code); + _return: - if (dbCache) { - CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); - } - if (dbInfo) { taosHashCleanup(dbInfo->vgHash); - dbInfo->vgHash = NULL; tfree(dbInfo); } + + tfree(msg); CTG_API_LEAVE(code); } -int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbFName, uint64_t dbId) { +int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { + CTG_API_ENTER(); + int32_t code = 0; - if (NULL == pCatalog || NULL == dbFName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + if (NULL == pCtg || NULL == dbFName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - if (NULL == pCatalog->dbCache) { + if (NULL == pCtg->dbCache) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - if (NULL == dbCache) { - ctgInfo("db not exist in dbCache, may be removed, dbFName:%s", dbFName); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; + SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); + CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); } - if (dbCache->dbId != dbId) { - ctgInfo("db id already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, dbFName, dbCache->dbId, dbId); - taosHashRelease(pCatalog->dbCache, dbCache); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); - CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbCache, dbFName)); - _return: - - taosHashRelease(pCatalog->dbCache, dbCache); + + tfree(action.data); CTG_API_LEAVE(code); } -int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid) { +int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid) { + CTG_API_ENTER(); + int32_t code = 0; - bool removed = false; - if (NULL == pCatalog || NULL == dbName || NULL == stbName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + if (NULL == pCtg || NULL == dbFName || NULL == stbName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - if (NULL == pCatalog->dbCache) { + if (NULL == pCtg->dbCache) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - - CTG_ERR_RET(ctgValidateAndRemoveStbMeta(pCatalog, dbName, stbName, suid, &removed)); - if (!removed) { - CTG_API_LEAVE(TSDB_CODE_SUCCESS); + + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; + SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); + CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); } - - ctgInfo("stb removed from cache, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); - CTG_ERR_JRET(ctgMetaRentRemove(&pCatalog->stbRent, suid, ctgSTableVersionCompare)); - - ctgDebug("stb removed from rent, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->stbName, stbName, sizeof(msg->stbName)); + msg->dbId = dbId; + msg->suid = suid; + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + _return: - + + tfree(action.data); + CTG_API_LEAVE(code); } -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); } -int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); } -int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg) { - STableMetaOutput output = {0}; +int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == rspMsg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); + if (NULL == output) { + ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); + } + int32_t code = 0; - CTG_API_ENTER(); + strcpy(output->dbFName, rspMsg->dbFName); + strcpy(output->tbName, rspMsg->tbName); - strcpy(output.dbFName, rspMsg->dbFName); - strcpy(output.tbName, rspMsg->tbName); + output->dbId = rspMsg->dbId; - SET_META_TYPE_TABLE(output.metaType); + SET_META_TYPE_TABLE(output->metaType); - CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output.tbMeta)); + CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); - CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output)); + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; + SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + msg->pCtg = pCtg; + msg->output = output; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + CTG_API_LEAVE(code); + _return: - tfree(output.tbMeta); + tfree(output->tbMeta); + tfree(output); + tfree(msg); CTG_API_LEAVE(code); } -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - CTG_API_LEAVE(ctgRenewTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, isSTable)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTransporter, pMgmtEps, pTableName, isSTable, NULL)); } -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); } -int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgroupList) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - - CTG_API_ENTER(); STableMeta *tbMeta = NULL; int32_t code = 0; SVgroupInfo vgroupInfo = {0}; SCtgDBCache* dbCache = NULL; SArray *vgList = NULL; + SDBVgInfo *vgInfo = NULL; - *pVgroupList = NULL; + *pVgList = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCatalog, pRpc, pMgmtEps, pTableName, false, &tbMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, -1)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbCache)); - // TODO REMOEV THIS .... + SHashObj *vgHash = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, false, &dbCache, &vgInfo)); + + if (dbCache) { + vgHash = dbCache->vgInfo->vgHash; + } else { + vgHash = vgInfo->vgHash; + } + + /* TODO REMOEV THIS .... if (0 == tbMeta->vgId) { SVgroupInfo vgroup = {0}; - catalogGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pTableName, &vgroup); + catalogGetTableHashVgroup(pCtg, pRpc, pMgmtEps, pTableName, &vgroup); tbMeta->vgId = vgroup.vgId; } - // TODO REMOVE THIS .... + // TODO REMOVE THIS ....*/ if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbCache->vgInfo, pVgroupList)); + CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList)); } else { int32_t vgId = tbMeta->vgId; - if (NULL == taosHashGetClone(dbCache->vgInfo->vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { + if (NULL == taosHashGetClone(vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1837,16 +2276,22 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - *pVgroupList = vgList; + *pVgList = vgList; vgList = NULL; } _return: - tfree(tbMeta); if (dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + tfree(tbMeta); + + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + tfree(vgInfo); } if (vgList) { @@ -1858,37 +2303,42 @@ _return: } -int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { - SCtgDBCache* dbCache = NULL; - int32_t code = 0; - +int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { CTG_API_ENTER(); + SCtgDBCache* dbCache = NULL; + int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbCache)); + SDBVgInfo *vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTransporter, pMgmtEps, db, false, &dbCache, &vgInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCatalog, dbCache->vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); _return: if (dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + tfree(vgInfo); } CTG_API_LEAVE(code); } -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { CTG_API_ENTER(); + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + int32_t code = 0; pRsp->pTableMeta = NULL; @@ -1909,7 +2359,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const S SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, name, false, &pTableMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, name, false, &pTableMeta, -1)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); @@ -1937,64 +2387,72 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { CTG_API_ENTER(); + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + //TODO CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num) { - if (NULL == pCatalog || NULL == stables || NULL == num) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgMetaRentGet(&pCatalog->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + if (NULL == pCtg || NULL == stables || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); } -int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num) { - if (NULL == pCatalog || NULL == dbs || NULL == num) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == dbs || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - CTG_API_LEAVE(ctgMetaRentGet(&pCatalog->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } void catalogDestroy(void) { - if (NULL == ctgMgmt.pCluster || atomic_load_8(&ctgMgmt.exit)) { + qInfo("start to destroy catalog"); + + if (NULL == gCtgMgmt.pCluster || atomic_load_8(&gCtgMgmt.exit)) { return; } - atomic_store_8(&ctgMgmt.exit, true); + atomic_store_8(&gCtgMgmt.exit, true); - CTG_LOCK(CTG_WRITE, &ctgMgmt.lock); + tsem_post(&gCtgMgmt.sem); - SCatalog *pCatalog = NULL; - void *pIter = taosHashIterate(ctgMgmt.pCluster, NULL); - while (pIter) { - pCatalog = *(SCatalog **)pIter; - - if (pCatalog) { - catalogFreeHandle(pCatalog); - } - - pIter = taosHashIterate(ctgMgmt.pCluster, pIter); + while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { + usleep(1); } - taosHashCleanup(ctgMgmt.pCluster); - ctgMgmt.pCluster = NULL; + CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); - CTG_UNLOCK(CTG_WRITE, &ctgMgmt.lock); + SCatalog *pCtg = NULL; + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + pCtg = *(SCatalog **)pIter; + + if (pCtg) { + catalogFreeHandle(pCtg); + } + + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); + } + + taosHashCleanup(gCtgMgmt.pCluster); + gCtgMgmt.pCluster = NULL; + + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock); qInfo("catalog destroyed"); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index d0f98e3c2a..86865e5790 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -33,18 +33,31 @@ #include "tep.h" #include "trpc.h" #include "tvariant.h" +#include "catalogInt.h" namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, int32_t *exist); -extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output); extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); +extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); +extern "C" int32_t ctgDbgEnableDebug(char *option); +extern "C" int32_t ctgDbgGetStatNum(char *option, void *res); -void ctgTestSetPrepareTableMeta(); -void ctgTestSetPrepareCTableMeta(); -void ctgTestSetPrepareSTableMeta(); -void ctgTestSetPrepareMultiSTableMeta(); +void ctgTestSetRspTableMeta(); +void ctgTestSetRspCTableMeta(); +void ctgTestSetRspSTableMeta(); +void ctgTestSetRspMultiSTableMeta(); + +extern "C" SCatalogMgmt gCtgMgmt; + +enum { + CTGT_RSP_VGINFO = 1, + CTGT_RSP_TBMETA, + CTGT_RSP_CTBMETA, + CTGT_RSP_STBMETA, + CTGT_RSP_MSTBMETA, +}; bool ctgTestStop = false; bool ctgTestEnableSleep = false; @@ -52,14 +65,14 @@ bool ctgTestDeadLoop = false; int32_t ctgTestPrintNum = 200000; int32_t ctgTestMTRunSec = 5; -int32_t ctgTestCurrentVgVersion = 0; -int32_t ctgTestVgVersion = 1; -int32_t ctgTestVgNum = 10; -int32_t ctgTestColNum = 2; -int32_t ctgTestTagNum = 1; -int32_t ctgTestSVersion = 1; -int32_t ctgTestTVersion = 1; -int32_t ctgTestSuid = 2; +int32_t ctgTestCurrentVgVersion = 0; +int32_t ctgTestVgVersion = 1; +int32_t ctgTestVgNum = 10; +int32_t ctgTestColNum = 2; +int32_t ctgTestTagNum = 1; +int32_t ctgTestSVersion = 1; +int32_t ctgTestTVersion = 1; +int32_t ctgTestSuid = 2; uint64_t ctgTestDbId = 33; uint64_t ctgTestClusterId = 0x1; @@ -68,36 +81,42 @@ char *ctgTestTablename = "table1"; char *ctgTestCTablename = "ctable1"; char *ctgTestSTablename = "stable1"; +int32_t ctgTestRspFunc[10] = {0}; +int32_t ctgTestRspIdx = 0; + void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { - SCreateDbReq *pReq = (SCreateDbReq *)rpcMallocCont(sizeof(SCreateDbReq)); - strcpy(pReq->db, "1.db1"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.db1"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbReq); + rpcMsg.contLen = contLen; rpcMsg.msgType = TDMT_MND_CREATE_DB; SRpcMsg rpcRsp = {0}; - rpcSendRecv(shandle, pEpSet, &rpcMsg, &rpcRsp); ASSERT_EQ(rpcRsp.code, 0); @@ -110,6 +129,8 @@ void ctgTestInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + ctgDbgEnableDebug("api"); + char temp[128] = {0}; sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { @@ -176,11 +197,11 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(s->name, "tag1s"); } -void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) { +void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; - SDBVgroupInfo *dbVgroup = (SDBVgroupInfo *)calloc(1, sizeof(SDBVgroupInfo)); + SDBVgInfo *dbVgroup = (SDBVgInfo *)calloc(1, sizeof(SDBVgInfo)); dbVgroup->vgVersion = vgVersion++; @@ -210,7 +231,6 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) { *pdbVgroup = dbVgroup; } - void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { strcpy(rspMsg->dbFName, ctgTestDbname); sprintf(rspMsg->tbName, "%s", ctgTestSTablename); @@ -248,44 +268,47 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { return; } - -void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - SUseDbRsp *rspMsg = NULL; // todo - - pRsp->code = 0; - pRsp->contLen = sizeof(SUseDbRsp) + ctgTestVgNum * sizeof(SVgroupInfo); - pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (SUseDbRsp *)pRsp->pCont; - strcpy(rspMsg->db, ctgTestDbname); - rspMsg->vgVersion = htonl(ctgTestVgVersion); +void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + SUseDbRsp usedbRsp = {0}; + strcpy(usedbRsp.db, ctgTestDbname); + usedbRsp.vgVersion = ctgTestVgVersion; ctgTestCurrentVgVersion = ctgTestVgVersion; - rspMsg->vgNum = htonl(ctgTestVgNum); - rspMsg->hashMethod = 0; - rspMsg->uid = htobe64(ctgTestDbId); + usedbRsp.vgNum = ctgTestVgNum; + usedbRsp.hashMethod = 0; + usedbRsp.uid = ctgTestDbId; + usedbRsp.pVgroupInfos = taosArrayInit(usedbRsp.vgNum, sizeof(SVgroupInfo)); - SVgroupInfo *vg = NULL; - uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; + uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; for (int32_t i = 0; i < ctgTestVgNum; ++i) { - vg = &rspMsg->vgroupInfo[i]; - - vg->vgId = htonl(i + 1); - vg->hashBegin = htonl(i * hashUnit); - vg->hashEnd = htonl(hashUnit * (i + 1) - 1); - vg->epset.numOfEps = i % TSDB_MAX_REPLICA + 1; - vg->epset.inUse = i % vg->epset.numOfEps; - for (int32_t n = 0; n < vg->epset.numOfEps; ++n) { - SEp *addr = &vg->epset.eps[n]; - strcpy(addr->fqdn, "a0"); - addr->port = htons(n + 22); + SVgroupInfo vg = {0}; + vg.vgId = i + 1; + vg.hashBegin = i * hashUnit; + vg.hashEnd = hashUnit * (i + 1) - 1; + if (i == ctgTestVgNum - 1) { + vg.hashEnd = htonl(UINT32_MAX); } + + vg.epset.numOfEps = i % TSDB_MAX_REPLICA + 1; + vg.epset.inUse = i % vg.epset.numOfEps; + for (int32_t n = 0; n < vg.epset.numOfEps; ++n) { + SEp *addr = &vg.epset.eps[n]; + strcpy(addr->fqdn, "a0"); + addr->port = n + 22; + } + + taosArrayPush(usedbRsp.pVgroupInfos, &vg); } - vg->hashEnd = htonl(UINT32_MAX); + int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); + void *pReq = rpcMallocCont(contLen); + tSerializeSUseDbRsp(pReq, contLen, &usedbRsp); - return; + pRsp->code = 0; + pRsp->contLen = contLen; + pRsp->pCont = pReq; } -void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -321,7 +344,7 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM return; } -void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -364,7 +387,7 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc return; } -void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -372,8 +395,8 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; strcpy(rspMsg->dbFName, ctgTestDbname); - strcpy(rspMsg->tbName, ctgTestSTablename); - strcpy(rspMsg->stbName, ctgTestSTablename); + strcpy(rspMsg->tbName, ctgTestSTablename); + strcpy(rspMsg->stbName, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -407,7 +430,7 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc return; } -void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo static int32_t idx = 1; @@ -453,151 +476,193 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, return; } -void ctgTestPrepareDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareTableMeta(); +void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + switch (ctgTestRspFunc[ctgTestRspIdx]) { + case CTGT_RSP_VGINFO: + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_TBMETA: + ctgTestRspTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_CTBMETA: + ctgTestRspCTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_STBMETA: + ctgTestRspSTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_MSTBMETA: + ctgTestRspMultiSTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + default: + break; + } + + ctgTestRspIdx++; return; } -void ctgTestPrepareDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareCTableMeta(); +void ctgTestRspDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + + ctgTestSetRspTableMeta(); return; } -void ctgTestPrepareDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); +void ctgTestRspDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareSTableMeta(); + ctgTestSetRspCTableMeta(); return; } -void ctgTestPrepareDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); +void ctgTestRspDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareMultiSTableMeta(); + ctgTestSetRspSTableMeta(); return; } -void ctgTestSetPrepareDbVgroups() { +void ctgTestRspDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + + ctgTestSetRspMultiSTableMeta(); + + return; +} + +void ctgTestSetRspDbVgroups() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroups); + stub.set(rpcSendRecv, ctgTestRspDbVgroups); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroups); + stub.set(f.second, ctgTestRspDbVgroups); } } } -void ctgTestSetPrepareTableMeta() { +void ctgTestSetRspTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareTableMeta); + stub.set(rpcSendRecv, ctgTestRspTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareTableMeta); + stub.set(f.second, ctgTestRspTableMeta); } } } -void ctgTestSetPrepareCTableMeta() { +void ctgTestSetRspCTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareCTableMeta); + stub.set(rpcSendRecv, ctgTestRspCTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareCTableMeta); + stub.set(f.second, ctgTestRspCTableMeta); } } } -void ctgTestSetPrepareSTableMeta() { +void ctgTestSetRspSTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareSTableMeta); + stub.set(rpcSendRecv, ctgTestRspSTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareSTableMeta); + stub.set(f.second, ctgTestRspSTableMeta); } } } -void ctgTestSetPrepareMultiSTableMeta() { +void ctgTestSetRspMultiSTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareMultiSTableMeta); + stub.set(rpcSendRecv, ctgTestRspMultiSTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareMultiSTableMeta); + stub.set(f.second, ctgTestRspMultiSTableMeta); } } } -void ctgTestSetPrepareDbVgroupsAndNormalMeta() { +void ctgTestSetRspByIdx() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndNormalMeta); + stub.set(rpcSendRecv, ctgTestRspByIdx); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndNormalMeta); + stub.set(f.second, ctgTestRspByIdx); } } } -void ctgTestSetPrepareDbVgroupsAndChildMeta() { + +void ctgTestSetRspDbVgroupsAndNormalMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndChildMeta); + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndNormalMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndChildMeta); + stub.set(f.second, ctgTestRspDbVgroupsAndNormalMeta); } } } -void ctgTestSetPrepareDbVgroupsAndSuperMeta() { +void ctgTestSetRspDbVgroupsAndChildMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndSuperMeta); + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndChildMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndSuperMeta); + stub.set(f.second, ctgTestRspDbVgroupsAndChildMeta); } } } -void ctgTestSetPrepareDbVgroupsAndMultiSuperMeta() { +void ctgTestSetRspDbVgroupsAndSuperMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndMultiSuperMeta); + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndSuperMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndMultiSuperMeta); + stub.set(f.second, ctgTestRspDbVgroupsAndSuperMeta); + } + } +} + +void ctgTestSetRspDbVgroupsAndMultiSuperMeta() { + static Stub stub; + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndMultiSuperMeta); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRecv$", result); + for (const auto &f : result) { + stub.set(f.second, ctgTestRspDbVgroupsAndMultiSuperMeta); } } } @@ -612,7 +677,7 @@ void *ctgTestGetDbVgroupThread(void *param) { int32_t n = 0; while (!ctgTestStop) { - code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); if (code) { assert(0); } @@ -635,12 +700,12 @@ void *ctgTestGetDbVgroupThread(void *param) { void *ctgTestSetSameDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); + code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); if (code) { assert(0); } @@ -656,16 +721,15 @@ void *ctgTestSetSameDbVgroupThread(void *param) { return NULL; } - void *ctgTestSetDiffDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup); + code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup); if (code) { assert(0); } @@ -681,7 +745,6 @@ void *ctgTestSetDiffDbVgroupThread(void *param) { return NULL; } - void *ctgTestGetCtableMetaThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; @@ -716,14 +779,24 @@ void *ctgTestGetCtableMetaThread(void *param) { void *ctgTestSetCtableMetaThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; int32_t n = 0; - STableMetaOutput output = {0}; + STableMetaOutput *output = NULL; - ctgTestBuildCTableMetaOutput(&output); + SCtgMetaAction action = {0}; + + action.act = CTG_ACT_UPDATE_TBL; while (!ctgTestStop) { - code = ctgUpdateTableMetaCache(pCtg, &output); + output = (STableMetaOutput *)malloc(sizeof(STableMetaOutput)); + ctgTestBuildCTableMetaOutput(output); + + SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)malloc(sizeof(SCtgUpdateTblMsg)); + msg->pCtg = pCtg; + msg->output = output; + action.data = msg; + + code = ctgActUpdateTbl(&action); if (code) { assert(0); } @@ -736,11 +809,10 @@ void *ctgTestSetCtableMetaThread(void *param) { } } - tfree(output.tbMeta); - return NULL; } +#if 0 TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; @@ -749,7 +821,7 @@ TEST(tableMeta, normalTable) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroups(); + ctgTestSetRspDbVgroups(); initQueryModuleMsgHandle(); @@ -770,7 +842,11 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epset.numOfEps, 3); - ctgTestSetPrepareTableMeta(); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + usleep(10000); + } + + ctgTestSetRspTableMeta(); STableMeta *tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); @@ -784,6 +860,16 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -832,6 +918,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(allStbNum, 0); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, childTableCase) { @@ -841,7 +928,7 @@ TEST(tableMeta, childTableCase) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndChildMeta(); + ctgTestSetRspDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -868,6 +955,16 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -914,7 +1011,7 @@ TEST(tableMeta, childTableCase) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); free(stb); stb = NULL; } else { @@ -930,6 +1027,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(allStbNum, 1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, superTableCase) { @@ -937,7 +1035,7 @@ TEST(tableMeta, superTableCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestSetRspDbVgroupsAndSuperMeta(); initQueryModuleMsgHandle(); @@ -966,7 +1064,17 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - ctgTestSetPrepareCTableMeta(); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + + ctgTestSetRspCTableMeta(); tableMeta = NULL; @@ -983,8 +1091,18 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (2 != n) { + usleep(10000); + } else { + break; + } + } + + tableMeta = NULL; - code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); @@ -1016,7 +1134,7 @@ TEST(tableMeta, superTableCase) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - + free(stb); stb = NULL; } else { @@ -1032,6 +1150,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(allStbNum, 1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, rmStbMeta) { @@ -1041,7 +1160,7 @@ TEST(tableMeta, rmStbMeta) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestSetRspDbVgroupsAndSuperMeta(); initQueryModuleMsgHandle(); @@ -1070,16 +1189,38 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - code = catalogRemoveSTableMeta(pCtg, "1.db1", ctgTestSTablename, ctgTestSuid); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + + code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid); ASSERT_EQ(code, 0); + while (true) { + int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); + if (n || m) { + usleep(10000); + } else { + break; + } + } + + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); - + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, updateStbMeta) { @@ -1089,7 +1230,7 @@ TEST(tableMeta, updateStbMeta) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestSetRspDbVgroupsAndSuperMeta(); initQueryModuleMsgHandle(); @@ -1118,6 +1259,16 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + tfree(tableMeta); STableMetaRsp rsp = {0}; @@ -1126,6 +1277,16 @@ TEST(tableMeta, updateStbMeta) { code = catalogUpdateSTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n != 3) { + usleep(10000); + } else { + break; + } + } + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); @@ -1146,19 +1307,26 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); tfree(tableMeta); - + catalogDestroy(); + memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat)); } - - TEST(tableDistVgroup, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; - ctgTestSetPrepareDbVgroupsAndNormalMeta(); + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + ctgTestRspFunc[2] = CTGT_RSP_VGINFO; + + ctgTestSetRspByIdx(); initQueryModuleMsgHandle(); @@ -1174,7 +1342,7 @@ TEST(tableDistVgroup, normalTable) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1182,6 +1350,7 @@ TEST(tableDistVgroup, normalTable) { ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableDistVgroup, childTableCase) { @@ -1190,7 +1359,16 @@ TEST(tableDistVgroup, childTableCase) { SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; - ctgTestSetPrepareDbVgroupsAndChildMeta(); + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_CTBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_VGINFO; + + ctgTestSetRspByIdx(); initQueryModuleMsgHandle(); @@ -1206,7 +1384,7 @@ TEST(tableDistVgroup, childTableCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1214,6 +1392,7 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(vgInfo->epset.numOfEps, 4); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableDistVgroup, superTableCase) { @@ -1222,7 +1401,18 @@ TEST(tableDistVgroup, superTableCase) { SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_STBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_VGINFO; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); @@ -1237,7 +1427,7 @@ TEST(tableDistVgroup, superTableCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1251,6 +1441,7 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(dbVgroup, getSetDbVgroupCase) { @@ -1258,12 +1449,19 @@ TEST(dbVgroup, getSetDbVgroupCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; SArray *vgList = NULL; ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndNormalMeta(); + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + + + ctgTestSetRspByIdx(); + initQueryModuleMsgHandle(); @@ -1279,16 +1477,21 @@ TEST(dbVgroup, getSetDbVgroupCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + usleep(10000); + } + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epset.numOfEps, 3); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1297,15 +1500,26 @@ TEST(dbVgroup, getSetDbVgroupCase) { taosArrayDestroy(vgList); ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); + code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); ASSERT_EQ(code, 0); + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n != 3) { + usleep(10000); + } else { + break; + } + } + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 7); ASSERT_EQ(vgInfo.epset.numOfEps, 2); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1314,6 +1528,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { taosArrayDestroy(vgList); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(multiThread, getSetRmSameDbVgroup) { @@ -1321,13 +1536,13 @@ TEST(multiThread, getSetRmSameDbVgroup) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroups(); + ctgTestSetRspDbVgroups(); initQueryModuleMsgHandle(); @@ -1365,6 +1580,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { sleep(1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(multiThread, getSetRmDiffDbVgroup) { @@ -1372,13 +1588,13 @@ TEST(multiThread, getSetRmDiffDbVgroup) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroups(); + ctgTestSetRspDbVgroups(); initQueryModuleMsgHandle(); @@ -1416,22 +1632,21 @@ TEST(multiThread, getSetRmDiffDbVgroup) { sleep(1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } - - TEST(multiThread, ctableMeta) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndChildMeta(); + ctgTestSetRspDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -1468,16 +1683,15 @@ TEST(multiThread, ctableMeta) { sleep(2); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } - - TEST(rentTest, allRent) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; SDbVgVersion *dbs = NULL; @@ -1486,7 +1700,7 @@ TEST(rentTest, allRent) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndMultiSuperMeta(); + ctgTestSetRspDbVgroupsAndMultiSuperMeta(); initQueryModuleMsgHandle(); @@ -1516,6 +1730,10 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { + usleep(10000); + } + code = catalogGetExpiredDBs(pCtg, &dbs, &num); ASSERT_EQ(code, 0); printf("%d - expired dbNum:%d\n", i, num); @@ -1530,7 +1748,8 @@ TEST(rentTest, allRent) { printf("%d - expired stableNum:%d\n", i, num); if (stable) { for (int32_t n = 0; n < num; ++n) { - printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); + printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, + stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } free(stable); stable = NULL; @@ -1541,8 +1760,11 @@ TEST(rentTest, allRent) { } catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } +#endif + int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 2138e58ccc..78d2feaa83 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -47,10 +47,11 @@ int32_t fmGetHandle(FuncMgtHandle* pHandle) { int32_t fmGetFuncInfo(FuncMgtHandle handle, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { SFuncMgtService* pService = (SFuncMgtService*)handle; - pFuncId = taosHashGet(pService->pFuncNameHashTable, pFuncName, strlen(pFuncName)); - if (NULL == pFuncId) { + void* pVal = taosHashGet(pService->pFuncNameHashTable, pFuncName, strlen(pFuncName)); + if (NULL == pVal) { return TSDB_CODE_FAILED; } + *pFuncId = *(int32_t*)pVal; if (*pFuncId < 0 || *pFuncId >= funcMgtBuiltinsNum) { return TSDB_CODE_FAILED; } diff --git a/source/nodes/CMakeLists.txt b/source/libs/nodes/CMakeLists.txt similarity index 84% rename from source/nodes/CMakeLists.txt rename to source/libs/nodes/CMakeLists.txt index b30534f3f2..9a826e034c 100644 --- a/source/nodes/CMakeLists.txt +++ b/source/libs/nodes/CMakeLists.txt @@ -2,7 +2,7 @@ aux_source_directory(src NODES_SRC) add_library(nodes STATIC ${NODES_SRC}) target_include_directories( nodes - PUBLIC "${CMAKE_SOURCE_DIR}/include/nodes" + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/nodes" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( diff --git a/source/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c similarity index 100% rename from source/nodes/src/nodesCloneFuncs.c rename to source/libs/nodes/src/nodesCloneFuncs.c diff --git a/source/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c similarity index 100% rename from source/nodes/src/nodesCodeFuncs.c rename to source/libs/nodes/src/nodesCodeFuncs.c diff --git a/source/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c similarity index 99% rename from source/nodes/src/nodesEqualFuncs.c rename to source/libs/nodes/src/nodesEqualFuncs.c index 41d1e5b05d..65eb6e9b32 100644 --- a/source/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodes.h" +#include "querynodes.h" #define COMPARE_SCALAR_FIELD(fldname) \ do { \ diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c new file mode 100644 index 0000000000..d1ded390db --- /dev/null +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "querynodes.h" + +typedef enum ETraversalOrder { + TRAVERSAL_PREORDER = 1, + TRAVERSAL_POSTORDER +} ETraversalOrder; + +static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext); + +static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext) { + if (NULL == pNode) { + return DEAL_RES_CONTINUE; + } + + EDealRes res = DEAL_RES_CONTINUE; + + if (TRAVERSAL_PREORDER == order) { + res = walker(pNode, pContext); + if (DEAL_RES_CONTINUE != res) { + return res; + } + } + + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_LIMIT: + // these node types with no subnodes + break; + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + res = walkNode(pOpNode->pLeft, order, walker, pContext); + if (DEAL_RES_ERROR != res) { + res = walkNode(pOpNode->pRight, order, walker, pContext); + } + break; + } + case QUERY_NODE_LOGIC_CONDITION: + res = walkList(((SLogicConditionNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_IS_NULL_CONDITION: + res = walkNode(((SIsNullCondNode*)pNode)->pExpr, order, walker, pContext); + break; + case QUERY_NODE_FUNCTION: + res = walkList(((SFunctionNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + break; // todo + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; + res = walkNode(pJoinTableNode->pLeft, order, walker, pContext); + if (DEAL_RES_ERROR != res) { + res = walkNode(pJoinTableNode->pRight, order, walker, pContext); + } + if (DEAL_RES_ERROR != res) { + res = walkNode(pJoinTableNode->pOnCond, order, walker, pContext); + } + break; + } + case QUERY_NODE_GROUPING_SET: + res = walkList(((SGroupingSetNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_ORDER_BY_EXPR: + res = walkNode(((SOrderByExprNode*)pNode)->pExpr, order, walker, pContext); + break; + case QUERY_NODE_STATE_WINDOW: + res = walkNode(((SStateWindowNode*)pNode)->pCol, order, walker, pContext); + break; + case QUERY_NODE_SESSION_WINDOW: + res = walkNode(((SSessionWindowNode*)pNode)->pCol, order, walker, pContext); + break; + case QUERY_NODE_INTERVAL_WINDOW: { + SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; + res = walkNode(pInterval->pInterval, order, walker, pContext); + if (DEAL_RES_ERROR != res) { + res = walkNode(pInterval->pOffset, order, walker, pContext); + } + if (DEAL_RES_ERROR != res) { + res = walkNode(pInterval->pSliding, order, walker, pContext); + } + if (DEAL_RES_ERROR != res) { + res = walkNode(pInterval->pFill, order, walker, pContext); + } + break; + } + case QUERY_NODE_NODE_LIST: + res = walkList(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext); + break; + case QUERY_NODE_FILL: + res = walkNode(((SFillNode*)pNode)->pValues, order, walker, pContext); + break; + case QUERY_NODE_RAW_EXPR: + res = walkNode(((SRawExprNode*)pNode)->pNode, order, walker, pContext); + break; + default: + break; + } + + if (DEAL_RES_ERROR != res && TRAVERSAL_POSTORDER == order) { + res = walker(pNode, pContext); + } + + return res; +} + +static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext) { + SNode* node; + FOREACH(node, pNodeList) { + if (DEAL_RES_ERROR == walkNode(node, order, walker, pContext)) { + return DEAL_RES_ERROR; + } + } + return DEAL_RES_CONTINUE; +} + +void nodesWalkNode(SNode* pNode, FNodeWalker walker, void* pContext) { + (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); +} + +void nodesWalkList(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { + (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); +} + +void nodesWalkNodePostOrder(SNode* pNode, FNodeWalker walker, void* pContext) { + (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); +} + +void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { + (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext); +} + +static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRewriter rewriter, void* pContext); + +static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewriter rewriter, void* pContext) { + if (NULL == pRawNode || NULL == *pRawNode) { + return DEAL_RES_CONTINUE; + } + + EDealRes res = DEAL_RES_CONTINUE; + + if (TRAVERSAL_PREORDER == order) { + res = rewriter(pRawNode, pContext); + if (DEAL_RES_CONTINUE != res) { + return res; + } + } + + SNode* pNode = *pRawNode; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_LIMIT: + // these node types with no subnodes + break; + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + res = rewriteNode(&(pOpNode->pLeft), order, rewriter, pContext); + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pOpNode->pRight), order, rewriter, pContext); + } + break; + } + case QUERY_NODE_LOGIC_CONDITION: + res = rewriteList(((SLogicConditionNode*)pNode)->pParameterList, order, rewriter, pContext); + break; + case QUERY_NODE_IS_NULL_CONDITION: + res = rewriteNode(&(((SIsNullCondNode*)pNode)->pExpr), order, rewriter, pContext); + break; + case QUERY_NODE_FUNCTION: + res = rewriteList(((SFunctionNode*)pNode)->pParameterList, order, rewriter, pContext); + break; + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + break; // todo + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; + res = rewriteNode(&(pJoinTableNode->pLeft), order, rewriter, pContext); + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pJoinTableNode->pRight), order, rewriter, pContext); + } + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pJoinTableNode->pOnCond), order, rewriter, pContext); + } + break; + } + case QUERY_NODE_GROUPING_SET: + res = rewriteList(((SGroupingSetNode*)pNode)->pParameterList, order, rewriter, pContext); + break; + case QUERY_NODE_ORDER_BY_EXPR: + res = rewriteNode(&(((SOrderByExprNode*)pNode)->pExpr), order, rewriter, pContext); + break; + case QUERY_NODE_STATE_WINDOW: + res = rewriteNode(&(((SStateWindowNode*)pNode)->pCol), order, rewriter, pContext); + break; + case QUERY_NODE_SESSION_WINDOW: + res = rewriteNode(&(((SSessionWindowNode*)pNode)->pCol), order, rewriter, pContext); + break; + case QUERY_NODE_INTERVAL_WINDOW: { + SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; + res = rewriteNode(&(pInterval->pInterval), order, rewriter, pContext); + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pInterval->pOffset), order, rewriter, pContext); + } + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pInterval->pSliding), order, rewriter, pContext); + } + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pInterval->pFill), order, rewriter, pContext); + } + break; + } + case QUERY_NODE_NODE_LIST: + res = rewriteList(((SNodeListNode*)pNode)->pNodeList, order, rewriter, pContext); + break; + case QUERY_NODE_FILL: + res = rewriteNode(&(((SFillNode*)pNode)->pValues), order, rewriter, pContext); + break; + case QUERY_NODE_RAW_EXPR: + res = rewriteNode(&(((SRawExprNode*)pNode)->pNode), order, rewriter, pContext); + break; + default: + break; + } + + if (DEAL_RES_ERROR != res && TRAVERSAL_POSTORDER == order) { + res = rewriter(pRawNode, pContext); + } + + return res; +} + +static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRewriter rewriter, void* pContext) { + SNode** pNode; + FOREACH_FOR_REWRITE(pNode, pNodeList) { + if (DEAL_RES_ERROR == rewriteNode(pNode, order, rewriter, pContext)) { + return DEAL_RES_ERROR; + } + } + return DEAL_RES_CONTINUE; +} + +void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext) { + (void)rewriteNode(pNode, TRAVERSAL_PREORDER, rewriter, pContext); +} + +void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { + (void)rewriteList(pList, TRAVERSAL_PREORDER, rewriter, pContext); +} + +void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { + (void)rewriteNode(pNode, TRAVERSAL_POSTORDER, rewriter, pContext); +} + +void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { + (void)rewriteList(pList, TRAVERSAL_POSTORDER, rewriter, pContext); +} diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c similarity index 88% rename from source/nodes/src/nodesUtilFuncs.c rename to source/libs/nodes/src/nodesUtilFuncs.c index 155eb6effe..5d4ddd0e03 100644 --- a/source/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodes.h" +#include "querynodes.h" #include "nodesShowStmts.h" #include "taoserror.h" @@ -76,7 +76,7 @@ SNode* nodesMakeNode(ENodeType type) { return NULL; } -static bool destroyNode(SNode* pNode, void* pContext) { +static EDealRes destroyNode(SNode* pNode, void* pContext) { switch (nodeType(pNode)) { case QUERY_NODE_VALUE: tfree(((SValueNode*)pNode)->literal); @@ -85,6 +85,7 @@ static bool destroyNode(SNode* pNode, void* pContext) { break; } tfree(pNode); + return DEAL_RES_CONTINUE; } void nodesDestroyNode(SNode* pNode) { @@ -116,9 +117,23 @@ SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode) { pList->pTail->pNext = p; } pList->pTail = p; + ++(pList->length); return pList; } +SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { + if (NULL == pCell->pPrev) { + pList->pHead = pCell->pNext; + } else { + pCell->pPrev->pNext = pCell->pNext; + pCell->pNext->pPrev = pCell->pPrev; + } + SListCell* pNext = pCell->pNext; + tfree(pCell); + --(pList->length); + return pNext; +} + SNode* nodesListGetNode(SNodeList* pList, int32_t index) { SNode* node; FOREACH(node, pList) { @@ -137,6 +152,11 @@ void nodesDestroyList(SNodeList* pList) { tfree(pList); } +bool nodesIsExprNode(const SNode* pNode) { + ENodeType type = nodeType(pNode); + return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type); +} + bool nodesIsArithmeticOp(const SOperatorNode* pOp) { switch (pOp->opType) { case OP_TYPE_ADD: diff --git a/source/nodes/test/CMakeLists.txt b/source/libs/nodes/test/CMakeLists.txt similarity index 100% rename from source/nodes/test/CMakeLists.txt rename to source/libs/nodes/test/CMakeLists.txt diff --git a/source/libs/nodes/test/nodesTest.cpp b/source/libs/nodes/test/nodesTest.cpp new file mode 100644 index 0000000000..c116faf4ce --- /dev/null +++ b/source/libs/nodes/test/nodesTest.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "querynodes.h" + +using namespace std; + +static EDealRes rewriterTest(SNode** pNode, void* pContext) { + EDealRes* pRes = (EDealRes*)pContext; + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + SOperatorNode* pOp = (SOperatorNode*)(*pNode); + if (QUERY_NODE_VALUE != nodeType(pOp->pLeft) || QUERY_NODE_VALUE != nodeType(pOp->pRight)) { + *pRes = DEAL_RES_ERROR; + } + SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + string tmp = to_string(stoi(((SValueNode*)(pOp->pLeft))->literal) + stoi(((SValueNode*)(pOp->pRight))->literal)); + pVal->literal = strdup(tmp.c_str()); + nodesDestroyNode(*pNode); + *pNode = (SNode*)pVal; + } + return DEAL_RES_CONTINUE; +} + +TEST(NodesTest, traverseTest) { + SNode* pRoot = nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode* pOp = (SOperatorNode*)pRoot; + SOperatorNode* pLeft = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + pLeft->pLeft = nodesMakeNode(QUERY_NODE_VALUE); + ((SValueNode*)(pLeft->pLeft))->literal = strdup("10"); + pLeft->pRight = nodesMakeNode(QUERY_NODE_VALUE); + ((SValueNode*)(pLeft->pRight))->literal = strdup("5"); + pOp->pLeft = (SNode*)pLeft; + pOp->pRight = nodesMakeNode(QUERY_NODE_VALUE); + ((SValueNode*)(pOp->pRight))->literal = strdup("3"); + + EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR); + EDealRes res = DEAL_RES_CONTINUE; + nodesRewriteNodePostOrder(&pRoot, rewriterTest, &res); + EXPECT_EQ(res, DEAL_RES_CONTINUE); + EXPECT_EQ(nodeType(pRoot), QUERY_NODE_VALUE); + EXPECT_EQ(string(((SValueNode*)pRoot)->literal), "18"); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h index 82315a5ba4..43bc5349e4 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "nodes.h" +#include "querynodes.h" #include "nodesShowStmts.h" #include "astCreateContext.h" #include "ttoken.h" @@ -55,6 +55,7 @@ SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToke SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); +SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 2e08a3afcc..77d6900acf 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -1,18 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifndef TDENGINE_ASTTOMSG_H #define TDENGINE_ASTTOMSG_H #include "parserInt.h" #include "tmsg.h" - char* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf); -char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); -SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); +char* buildShowMsg(SShowInfo* pShowInfo, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t* outputLen, SParseContext* pCtx, SMsgBuf* pMsgBuf); +char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildDropStableReq(SSqlInfo* pInfo, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf); +char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf); #endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y index 763f4ee38b..eef04b17c3 100644 --- a/source/libs/parser/inc/new_sql.y +++ b/source/libs/parser/inc/new_sql.y @@ -238,6 +238,7 @@ joined_table(A) ::= %type join_type { EJoinType } %destructor join_type { PARSER_DESTRUCTOR_TRACE; } +join_type(A) ::= . { PARSER_TRACE; A = JOIN_TYPE_INNER; } join_type(A) ::= INNER. { PARSER_TRACE; A = JOIN_TYPE_INNER; } /************************************************ query_specification *************************************************/ @@ -315,7 +316,12 @@ fill_mode(A) ::= NEXT. %type group_by_clause_opt { SNodeList* } %destructor group_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } group_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } -group_by_clause_opt(A) ::= GROUP BY expression_list(B). { PARSER_TRACE; A = B; } +group_by_clause_opt(A) ::= GROUP BY group_by_list(B). { PARSER_TRACE; A = B; } + +%type group_by_list { SNodeList* } +%destructor group_by_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +group_by_list(A) ::= expression(B). { PARSER_TRACE; A = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, B))); } +group_by_list(A) ::= group_by_list(B) NK_COMMA expression(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, C))); } having_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } having_clause_opt(A) ::= HAVING search_condition(B). { PARSER_TRACE; A = B; } diff --git a/source/libs/parser/inc/parserImpl.h b/source/libs/parser/inc/parserImpl.h index 183075d465..b55060def7 100644 --- a/source/libs/parser/inc/parserImpl.h +++ b/source/libs/parser/inc/parserImpl.h @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodes.h" +#include "querynodes.h" #include "parser.h" #ifndef _TD_AST_CREATE_FUNCS_H_ @@ -25,6 +25,7 @@ extern "C" { typedef struct SQuery { SNode* pRoot; + // todo reslut meta } SQuery; int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery); diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c index bfcebfd8b8..ce2903f6aa 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/astCreateFuncs.c @@ -242,6 +242,9 @@ SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order CHECK_OUT_OF_MEM(orderByExpr); orderByExpr->pExpr = pExpr; orderByExpr->order = order; + if (NULL_ORDER_DEFAULT == nullOrder) { + nullOrder = (ORDER_ASC == order ? NULL_ORDER_FIRST : NULL_ORDER_LAST); + } orderByExpr->nullOrder = nullOrder; return (SNode*)orderByExpr; } @@ -279,6 +282,15 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { return (SNode*)fill; } +SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) { + SGroupingSetNode* groupingSet = (SGroupingSetNode*)nodesMakeNode(QUERY_NODE_GROUPING_SET); + CHECK_OUT_OF_MEM(groupingSet); + groupingSet->groupingSetType = GP_TYPE_NORMAL; + groupingSet->pParameterList = nodesMakeList(); + nodesListAppend(groupingSet->pParameterList, pNode); + return (SNode*)groupingSet; +} + SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { strncpy(((SExprNode*)pNode)->aliasName, pAlias->z, pAlias->n); return pNode; diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index c34b341de5..2bbf9116c2 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -1,5 +1,20 @@ -#include "parserInt.h" +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include "astGenerator.h" +#include "parserInt.h" #include "parserUtil.h" char* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) { @@ -11,20 +26,19 @@ char* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, createReq.superUser = (int8_t)pUser->type; if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - // pMsg->privilege = (char)pCmd->count; + // pMsg->privilege = (char)pCmd->count; } else { strncpy(createReq.pass, pUser->passwd.z, pUser->passwd.n); } - int32_t tlen = tSerializeSCreateUserReq(NULL, &createReq); + int32_t tlen = tSerializeSCreateUserReq(NULL, 0, &createReq); void* pReq = malloc(tlen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void* pBuf = pReq; - tSerializeSCreateUserReq(&pBuf, &createReq); + tSerializeSCreateUserReq(pReq, tlen, &createReq); *outputLen = tlen; return pReq; } @@ -40,14 +54,11 @@ char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; - createReq.maxUsers = htonl(pAcctOpt->maxUsers); - createReq.maxDbs = htonl(pAcctOpt->maxDbs); - createReq.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); - createReq.maxStreams = htonl(pAcctOpt->maxStreams); - // createReq.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); - createReq.maxStorage = htobe64(pAcctOpt->maxStorage); - // createReq.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); - // createReq.maxConnections = htonl(pAcctOpt->maxConnections); + createReq.maxUsers = pAcctOpt->maxUsers; + createReq.maxDbs = pAcctOpt->maxDbs; + createReq.maxTimeSeries = pAcctOpt->maxTimeSeries; + createReq.maxStreams = pAcctOpt->maxStreams; + createReq.maxStorage = pAcctOpt->maxStorage; if (pAcctOpt->stat.n == 0) { createReq.accessState = -1; @@ -63,20 +74,19 @@ char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, } } - int32_t tlen = tSerializeSCreateAcctReq(NULL, &createReq); + int32_t tlen = tSerializeSCreateAcctReq(NULL, 0, &createReq); void* pReq = malloc(tlen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void* pBuf = pReq; - tSerializeSCreateAcctReq(&pBuf, &createReq); + tSerializeSCreateAcctReq(pReq, tlen, &createReq); *outputLen = tlen; return pReq; } -char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) { +char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgBufLen) { SDropUserReq dropReq = {0}; SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0); @@ -86,39 +96,34 @@ char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* msgLen, int64_t id, char* msgBu strncpy(dropReq.user, pName->z, pName->n); - int32_t tlen = tSerializeSDropUserReq(NULL, &dropReq); + int32_t tlen = tSerializeSDropUserReq(NULL, 0, &dropReq); void* pReq = malloc(tlen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void* pBuf = pReq; - tSerializeSDropUserReq(&pBuf, &dropReq); - *msgLen = tlen; + tSerializeSDropUserReq(pReq, tlen, &dropReq); + *outputLen = tlen; return pReq; } -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf) { - SShowReq* pShowMsg = calloc(1, sizeof(SShowReq)); - if (pShowMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return pShowMsg; - } +char* buildShowMsg(SShowInfo* pShowInfo, int32_t* outputLen, SParseContext* pCtx, SMsgBuf* pMsgBuf) { + SShowReq showReq = {.type = pShowInfo->showType}; - pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { SToken* pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query - strncpy(pShowMsg->payload, pPattern->z, pPattern->n); - pShowMsg->payloadLen = htons(pPattern->n); + showReq.payloadLen = pPattern->n; + showReq.payload = malloc(showReq.payloadLen); + strncpy(showReq.payload, pPattern->z, pPattern->n); } } else { SToken* pEpAddr = &pShowInfo->prefix; assert(pEpAddr->n > 0 && pEpAddr->type > 0); - - tstrncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n + 1); - pShowMsg->payloadLen = htons(pEpAddr->n); + showReq.payloadLen = pEpAddr->n; + showReq.payload = malloc(showReq.payloadLen); + strncpy(showReq.payload, pEpAddr->z, pEpAddr->n); } if (pShowInfo->showType == TSDB_MGMT_TABLE_STB || pShowInfo->showType == TSDB_MGMT_TABLE_VGROUP) { @@ -127,22 +132,32 @@ SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, SMsgBuf* pMsgB if (pShowInfo->prefix.n > 0) { if (pShowInfo->prefix.n >= TSDB_DB_FNAME_LEN) { terrno = buildInvalidOperationMsg(pMsgBuf, "prefix name is too long"); - tfree(pShowMsg); + tFreeSShowReq(&showReq); return NULL; } tNameSetDbName(&n, pCtx->acctId, pShowInfo->prefix.z, pShowInfo->prefix.n); } else if (pCtx->db == NULL || strlen(pCtx->db) == 0) { terrno = buildInvalidOperationMsg(pMsgBuf, "database is not specified"); - tfree(pShowMsg); + tFreeSShowReq(&showReq); return NULL; } else { tNameSetDbName(&n, pCtx->acctId, pCtx->db, strlen(pCtx->db)); } - tNameGetFullDbName(&n, pShowMsg->db); + tNameGetFullDbName(&n, showReq.db); } - return pShowMsg; + int32_t tlen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = malloc(tlen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSShowReq(pReq, tlen, &showReq); + tFreeSShowReq(&showReq); + *outputLen = tlen; + return pReq; } static int32_t setKeepOption(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb, SMsgBuf* pMsgBuf) { @@ -150,35 +165,35 @@ static int32_t setKeepOption(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb, const char* msg2 = "invalid keep value"; const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2"; - pMsg->daysToKeep0 = htonl(-1); - pMsg->daysToKeep1 = htonl(-1); - pMsg->daysToKeep2 = htonl(-1); + pMsg->daysToKeep0 = -1; + pMsg->daysToKeep1 = -1; + pMsg->daysToKeep2 = -1; SArray* pKeep = pCreateDb->keep; if (pKeep != NULL) { size_t s = taosArrayGetSize(pKeep); #ifdef _STORAGE - if (s >= 4 ||s <= 0) { + if (s >= 4 || s <= 0) { #else if (s != 1) { #endif return buildInvalidOperationMsg(pMsgBuf, msg1); } -// tListI* p0 = taosArrayGet(pKeep, 0); -// tVariantListItem* p1 = (s > 1) ? taosArrayGet(pKeep, 1) : p0; -// tVariantListItem* p2 = (s > 2) ? taosArrayGet(pKeep, 2) : p1; -// -// if ((int32_t)p0->pVar.i64 <= 0 || (int32_t)p1->pVar.i64 <= 0 || (int32_t)p2->pVar.i64 <= 0) { -// return buildInvalidOperationMsg(pMsgBuf, msg2); -// } -// if (!(((int32_t)p0->pVar.i64 <= (int32_t)p1->pVar.i64) && ((int32_t)p1->pVar.i64 <= (int32_t)p2->pVar.i64))) { -// return buildInvalidOperationMsg(pMsgBuf, msg3); -// } -// -// pMsg->daysToKeep0 = htonl((int32_t)p0->pVar.i64); -// pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64); -// pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64); + // tListI* p0 = taosArrayGet(pKeep, 0); + // tVariantListItem* p1 = (s > 1) ? taosArrayGet(pKeep, 1) : p0; + // tVariantListItem* p2 = (s > 2) ? taosArrayGet(pKeep, 2) : p1; + // + // if ((int32_t)p0->pVar.i64 <= 0 || (int32_t)p1->pVar.i64 <= 0 || (int32_t)p2->pVar.i64 <= 0) { + // return buildInvalidOperationMsg(pMsgBuf, msg2); + // } + // if (!(((int32_t)p0->pVar.i64 <= (int32_t)p1->pVar.i64) && ((int32_t)p1->pVar.i64 <= (int32_t)p2->pVar.i64))) { + // return buildInvalidOperationMsg(pMsgBuf, msg3); + // } + // + // pMsg->daysToKeep0 = htonl((int32_t)p0->pVar.i64); + // pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64); + // pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64); } return TSDB_CODE_SUCCESS; @@ -189,7 +204,7 @@ static int32_t setTimePrecision(SCreateDbReq* pMsg, const SCreateDbInfo* pCreate pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SToken* pToken = (SToken*) &pCreateDbInfo->precision; + SToken* pToken = (SToken*)&pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -212,21 +227,21 @@ static int32_t setTimePrecision(SCreateDbReq* pMsg, const SCreateDbInfo* pCreate } static void doSetDbOptions(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb) { - pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); - pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); - pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); - pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); - pMsg->minRows = htonl(pCreateDb->minRowsPerBlock); - pMsg->maxRows = htonl(pCreateDb->maxRowsPerBlock); - pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); - pMsg->compression = (int8_t) pCreateDb->compressionLevel; - pMsg->walLevel = (char)pCreateDb->walLevel; - pMsg->replications = pCreateDb->replica; - pMsg->quorum = pCreateDb->quorum; - pMsg->ignoreExist = pCreateDb->ignoreExists; - pMsg->update = pCreateDb->update; - pMsg->cacheLastRow = pCreateDb->cachelast; - pMsg->numOfVgroups = htonl(pCreateDb->numOfVgroups); + pMsg->cacheBlockSize = pCreateDb->cacheBlockSize; + pMsg->totalBlocks = pCreateDb->numOfBlocks; + pMsg->daysPerFile = pCreateDb->daysPerFile; + pMsg->commitTime = (int32_t)pCreateDb->commitTime; + pMsg->minRows = pCreateDb->minRowsPerBlock; + pMsg->maxRows = pCreateDb->maxRowsPerBlock; + pMsg->fsyncPeriod = pCreateDb->fsyncPeriod; + pMsg->compression = (int8_t)pCreateDb->compressionLevel; + pMsg->walLevel = (char)pCreateDb->walLevel; + pMsg->replications = pCreateDb->replica; + pMsg->quorum = pCreateDb->quorum; + pMsg->ignoreExist = pCreateDb->ignoreExists; + pMsg->update = pCreateDb->update; + pMsg->cacheLastRow = pCreateDb->cachelast; + pMsg->numOfVgroups = pCreateDb->numOfVgroups; } int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) { @@ -243,12 +258,104 @@ int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbS return TSDB_CODE_SUCCESS; } -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf) { - SCreateDbReq* pCreateMsg = calloc(1, sizeof(SCreateDbReq)); - if (setDbOptions(pCreateMsg, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { - tfree(pCreateMsg); - terrno = TSDB_CODE_TSC_INVALID_OPERATION; +// can only perform the parameters based on the macro definitation +static int32_t doCheckDbOptions(SCreateDbReq* pCreate, SMsgBuf* pMsgBuf) { + char msg[512] = {0}; + if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->replications != -1 && + (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, + TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t blocks = pCreate->totalBlocks; + if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { + snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, + TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->quorum != -1 && + (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, + TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t val = pCreate->daysPerFile; + if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { + snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, + TSDB_MAX_DAYS_PER_FILE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->cacheBlockSize; + if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { + snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, + TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && + pCreate->precision != TSDB_TIME_PRECISION_NANO) { + snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, + TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->commitTime; + if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { + snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME, + TSDB_MAX_COMMIT_TIME); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->fsyncPeriod; + if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { + snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, + TSDB_MAX_FSYNC_PERIOD); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->compression != -1 && + (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, + TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->numOfVgroups; + if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) { + snprintf(msg, tListLen(msg), "invalid number of vgroups for DB:%d valid range: [%d, %d]", val, + TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); + } + + val = pCreate->maxRows; + if (val < TSDB_MIN_MAX_ROW_FBLOCK || val > TSDB_MAX_MAX_ROW_FBLOCK) { + snprintf(msg, tListLen(msg), "invalid number of max rows in file block for DB:%d valid range: [%d, %d]", val, + TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); + } + + val = pCreate->minRows; + if (val < TSDB_MIN_MIN_ROW_FBLOCK || val > TSDB_MAX_MIN_ROW_FBLOCK) { + snprintf(msg, tListLen(msg), "invalid number of min rows in file block for DB:%d valid range: [%d, %d]", val, + TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + } + + return TSDB_CODE_SUCCESS; +} + +char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t* outputLen, SParseContext* pCtx, SMsgBuf* pMsgBuf) { + SCreateDbReq createReq = {0}; + + if (setDbOptions(&createReq, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_OPERATION; return NULL; } @@ -259,11 +366,27 @@ SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx return NULL; } - tNameGetFullDbName(&name, pCreateMsg->db); - return pCreateMsg; + tNameGetFullDbName(&name, createReq.db); + + if (doCheckDbOptions(&createReq, pMsgBuf) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_OPERATION; + return NULL; + } + + int32_t tlen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = malloc(tlen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSCreateDbReq(pReq, tlen, &createReq); + *outputLen = tlen; + return pReq; } -char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { +char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* outputLen, SParseContext* pParseCtx, + SMsgBuf* pMsgBuf) { SMCreateStbReq createReq = {0}; createReq.igExists = pCreateTableSql->existCheck ? 1 : 0; createReq.pColumns = pCreateTableSql->colInfo.pColumns; @@ -290,11 +413,11 @@ char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseCo void* pBuf = pReq; tSerializeSMCreateStbReq(&pBuf, &createReq); - *len = tlen; + *outputLen = tlen; return pReq; } -char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { +char* buildDropStableReq(SSqlInfo* pInfo, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SToken* tableName = taosArrayGet(pInfo->pMiscInfo->a, 0); SName name = {0}; @@ -319,11 +442,11 @@ char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx void* pBuf = pReq; tSerializeSMDropStbReq(&pBuf, &dropReq); - *len = tlen; + *outputLen = tlen; return pReq; } -SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { +char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid host name (name too long, maximum length 128)"; const char* msg2 = "dnode name can not be string"; const char* msg3 = "port should be an integer that is less than 65535 and greater than 0"; @@ -355,33 +478,44 @@ SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs return NULL; } - SCreateDnodeReq *pCreate = (SCreateDnodeReq *) calloc(1, sizeof(SCreateDnodeReq)); - if (pCreate == NULL) { - buildInvalidOperationMsg(pMsgBuf, msg4); + SCreateDnodeReq createReq = {0}; + + strncpy(createReq.fqdn, id->z, id->n); + createReq.port = val; + + int32_t tlen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); + void* pReq = malloc(tlen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - strncpy(pCreate->fqdn, id->z, id->n); - pCreate->port = htonl(val); - - *len = sizeof(SCreateDnodeReq); - return pCreate; + tSerializeSCreateDnodeReq(pReq, tlen, &createReq); + *outputLen = tlen; + return pReq; } -SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { +char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf) { + SDropDnodeReq dropReq = {0}; + SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); char* end = NULL; - SDropDnodeReq * pDrop = (SDropDnodeReq *)calloc(1, sizeof(SDropDnodeReq)); - pDrop->dnodeId = strtoll(pzName->z, &end, 10); - pDrop->dnodeId = htonl(pDrop->dnodeId); - *len = sizeof(SDropDnodeReq); + dropReq.dnodeId = strtoll(pzName->z, &end, 10); if (end - pzName->z != pzName->n) { buildInvalidOperationMsg(pMsgBuf, "invalid dnode id"); - tfree(pDrop); return NULL; } - return pDrop; + int32_t tlen = tSerializeSDropDnodeReq(NULL, 0, &dropReq); + void* pReq = malloc(tlen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSDropDnodeReq(pReq, tlen, &dropReq); + *outputLen = tlen; + return pReq; } \ No newline at end of file diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index a56a6524fc..ffc9f4a3f6 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3647,7 +3647,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) //TODO remove it int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) { SArray* vgroupList = NULL; - int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); + int32_t code = catalogGetTableDistVgInfo(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 4b70d2c6c3..b4125b0a24 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -50,7 +50,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); - int32_t code = catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); + int32_t code = catalogGetDBVgInfo(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); if (code != TSDB_CODE_SUCCESS) { terrno = code; return code; @@ -114,105 +114,10 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out } *pEpSet = pCtx->mgmtEpSet; - *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf); + *output = buildShowMsg(pShowInfo, outputLen, pCtx, pMsgBuf); if (*output == NULL) { return terrno; } - - *outputLen = sizeof(SShowReq) /* + htons(pShowMsg->payloadLen)*/; - } - - return TSDB_CODE_SUCCESS; -} - -// can only perform the parameters based on the macro definitation -static int32_t doCheckDbOptions(SCreateDbReq* pCreate, SMsgBuf* pMsgBuf) { - char msg[512] = {0}; - - if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->replications != -1 && - (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - int32_t blocks = ntohl(pCreate->totalBlocks); - if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, - TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->quorum != -1 && - (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, - TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - int32_t val = htonl(pCreate->daysPerFile); - if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { - snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, - TSDB_MAX_DAYS_PER_FILE); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->cacheBlockSize); - if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { - snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, - TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && - pCreate->precision != TSDB_TIME_PRECISION_NANO) { - snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, - TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->commitTime); - if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { - snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME, - TSDB_MAX_COMMIT_TIME); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->fsyncPeriod); - if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { - snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, - TSDB_MAX_FSYNC_PERIOD); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->compression != -1 && - (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, - TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->numOfVgroups); - if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) { - snprintf(msg, tListLen(msg), "invalid number of vgroups for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); - } - - val = htonl(pCreate->maxRows); - if (val < TSDB_MIN_MAX_ROW_FBLOCK || val > TSDB_MAX_MAX_ROW_FBLOCK) { - snprintf(msg, tListLen(msg), "invalid number of max rows in file block for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); - } - - val = htonl(pCreate->minRows); - if (val < TSDB_MIN_MIN_ROW_FBLOCK || val > TSDB_MAX_MIN_ROW_FBLOCK) { - snprintf(msg, tListLen(msg), "invalid number of min rows in file block for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); } return TSDB_CODE_SUCCESS; @@ -852,11 +757,15 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - SUseDbReq* pUseDbMsg = (SUseDbReq*)calloc(1, sizeof(SUseDbReq)); - tNameExtractFullName(&n, pUseDbMsg->db); + SUseDbReq usedbReq = {0}; + tNameExtractFullName(&n, usedbReq.db); - pDcl->pMsg = (char*)pUseDbMsg; - pDcl->msgLen = sizeof(SUseDbReq); + int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); + void* pBuf = malloc(bufLen); + tSerializeSUseDbReq(pBuf, bufLen, &usedbReq); + + pDcl->pMsg = pBuf; + pDcl->msgLen = bufLen; pDcl->msgType = TDMT_MND_USE_DB; break; } @@ -880,14 +789,11 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - SCreateDbReq* pCreateMsg = buildCreateDbMsg(pCreateDB, pCtx, pMsgBuf); - if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_OPERATION; - goto _error; - } + int32_t bufLen = 0; + char* pBuf = buildCreateDbMsg(pCreateDB, &bufLen, pCtx, pMsgBuf); - pDcl->pMsg = (char*)pCreateMsg; - pDcl->msgLen = sizeof(SCreateDbReq); + pDcl->pMsg = pBuf; + pDcl->msgLen = bufLen; pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_DB) ? TDMT_MND_CREATE_DB : TDMT_MND_ALTER_DB; break; } @@ -905,15 +811,18 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - SDropDbReq* pDropDbMsg = (SDropDbReq*)calloc(1, sizeof(SDropDbReq)); - - code = tNameExtractFullName(&name, pDropDbMsg->db); - pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + SDropDbReq dropdbReq = {0}; + code = tNameExtractFullName(&name, dropdbReq.db); + dropdbReq.ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_DB_NAME_T); + int32_t bufLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pBuf = malloc(bufLen); + tSerializeSDropDbReq(pBuf, bufLen, &dropdbReq); + pDcl->msgType = TDMT_MND_DROP_DB; - pDcl->msgLen = sizeof(SDropDbReq); - pDcl->pMsg = (char*)pDropDbMsg; + pDcl->msgLen = bufLen; + pDcl->pMsg = pBuf; break; } diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/new_sql.c index 4d03de14d4..b7a82e5d97 100644 --- a/source/libs/parser/src/new_sql.c +++ b/source/libs/parser/src/new_sql.c @@ -109,21 +109,21 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 124 +#define YYNOCODE 125 #define YYACTIONTYPE unsigned short int #define NewParseTOKENTYPE SToken typedef union { int yyinit; NewParseTOKENTYPE yy0; - EOrder yy10; - EFillMode yy14; - SNode* yy168; - ENullOrder yy177; - SNodeList* yy192; - bool yy209; - EOperatorType yy228; - EJoinType yy229; - SToken yy241; + EOperatorType yy40; + EFillMode yy44; + SToken yy79; + ENullOrder yy107; + EJoinType yy162; + SNodeList* yy174; + EOrder yy188; + SNode* yy212; + bool yy237; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -138,17 +138,17 @@ typedef union { #define NewParseCTX_PARAM #define NewParseCTX_FETCH #define NewParseCTX_STORE -#define YYNSTATE 144 -#define YYNRULE 135 +#define YYNSTATE 147 +#define YYNRULE 138 #define YYNTOKEN 72 -#define YY_MAX_SHIFT 143 -#define YY_MIN_SHIFTREDUCE 240 -#define YY_MAX_SHIFTREDUCE 374 -#define YY_ERROR_ACTION 375 -#define YY_ACCEPT_ACTION 376 -#define YY_NO_ACTION 377 -#define YY_MIN_REDUCE 378 -#define YY_MAX_REDUCE 512 +#define YY_MAX_SHIFT 146 +#define YY_MIN_SHIFTREDUCE 243 +#define YY_MAX_SHIFTREDUCE 380 +#define YY_ERROR_ACTION 381 +#define YY_ACCEPT_ACTION 382 +#define YY_NO_ACTION 383 +#define YY_MIN_REDUCE 384 +#define YY_MAX_REDUCE 521 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -215,200 +215,208 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (692) +#define YY_ACTTAB_COUNT (737) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 387, 385, 89, 22, 64, 388, 385, 462, 29, 27, - /* 10 */ 25, 24, 23, 395, 385, 138, 410, 138, 410, 139, - /* 20 */ 71, 109, 78, 396, 262, 399, 21, 84, 93, 143, - /* 30 */ 280, 281, 282, 283, 284, 285, 286, 288, 289, 290, - /* 40 */ 29, 27, 25, 24, 23, 395, 385, 138, 410, 138, - /* 50 */ 410, 139, 318, 113, 81, 396, 97, 399, 21, 84, - /* 60 */ 93, 51, 280, 281, 282, 283, 284, 285, 286, 288, - /* 70 */ 289, 290, 128, 395, 385, 114, 447, 138, 410, 139, - /* 80 */ 125, 9, 38, 396, 51, 399, 435, 255, 28, 26, - /* 90 */ 83, 431, 491, 444, 317, 242, 243, 244, 245, 140, - /* 100 */ 248, 491, 98, 1, 343, 490, 67, 10, 106, 489, - /* 110 */ 326, 395, 385, 60, 50, 138, 410, 139, 489, 123, - /* 120 */ 39, 396, 253, 399, 435, 51, 125, 9, 91, 431, - /* 130 */ 105, 341, 342, 344, 345, 395, 385, 73, 466, 138, - /* 140 */ 410, 139, 4, 314, 39, 396, 411, 399, 435, 51, - /* 150 */ 447, 447, 91, 431, 242, 243, 244, 245, 140, 248, - /* 160 */ 395, 385, 487, 8, 138, 410, 139, 443, 442, 39, - /* 170 */ 396, 40, 399, 435, 340, 18, 107, 91, 431, 7, - /* 180 */ 6, 29, 27, 25, 24, 23, 135, 451, 128, 395, - /* 190 */ 385, 7, 6, 138, 410, 139, 28, 26, 77, 396, - /* 200 */ 473, 399, 298, 242, 243, 244, 245, 140, 248, 104, - /* 210 */ 98, 1, 28, 26, 392, 10, 390, 491, 52, 242, - /* 220 */ 243, 244, 245, 140, 248, 248, 98, 5, 102, 54, - /* 230 */ 50, 136, 57, 19, 489, 395, 385, 131, 103, 138, - /* 240 */ 410, 139, 472, 287, 39, 396, 291, 399, 435, 51, - /* 250 */ 373, 374, 434, 431, 25, 24, 23, 29, 27, 25, - /* 260 */ 24, 23, 30, 395, 385, 292, 56, 138, 410, 139, - /* 270 */ 90, 256, 39, 396, 132, 399, 435, 28, 26, 319, - /* 280 */ 118, 431, 3, 453, 242, 243, 244, 245, 140, 248, - /* 290 */ 70, 98, 5, 117, 395, 385, 130, 128, 126, 410, - /* 300 */ 139, 122, 43, 44, 396, 20, 399, 119, 59, 277, - /* 310 */ 41, 29, 27, 25, 24, 23, 101, 133, 47, 65, - /* 320 */ 440, 121, 30, 120, 69, 259, 491, 115, 110, 108, - /* 330 */ 99, 459, 29, 27, 25, 24, 23, 2, 314, 50, - /* 340 */ 28, 26, 127, 489, 61, 16, 417, 242, 243, 244, - /* 350 */ 245, 140, 248, 42, 98, 5, 28, 26, 29, 27, - /* 360 */ 25, 24, 23, 242, 243, 244, 245, 140, 248, 252, - /* 370 */ 98, 1, 395, 385, 262, 255, 138, 410, 139, 31, - /* 380 */ 448, 39, 396, 256, 399, 435, 28, 26, 62, 463, - /* 390 */ 432, 370, 371, 242, 243, 244, 245, 140, 248, 94, - /* 400 */ 98, 5, 506, 137, 488, 395, 385, 253, 134, 138, - /* 410 */ 410, 139, 13, 72, 45, 396, 30, 399, 395, 385, - /* 420 */ 14, 53, 126, 410, 139, 337, 55, 44, 396, 34, - /* 430 */ 399, 395, 385, 339, 46, 138, 410, 139, 58, 35, - /* 440 */ 81, 396, 100, 399, 111, 376, 141, 333, 332, 112, - /* 450 */ 390, 129, 504, 395, 385, 460, 6, 138, 410, 139, - /* 460 */ 36, 15, 81, 396, 116, 399, 312, 278, 395, 385, - /* 470 */ 311, 33, 138, 410, 139, 491, 66, 81, 396, 92, - /* 480 */ 399, 395, 385, 32, 389, 138, 410, 139, 50, 49, - /* 490 */ 45, 396, 489, 399, 395, 385, 260, 364, 138, 410, - /* 500 */ 139, 17, 11, 75, 396, 37, 399, 395, 385, 359, - /* 510 */ 358, 138, 410, 139, 95, 363, 79, 396, 362, 399, - /* 520 */ 29, 27, 25, 24, 23, 96, 395, 385, 505, 12, - /* 530 */ 138, 410, 139, 74, 379, 76, 396, 246, 399, 395, - /* 540 */ 385, 378, 142, 138, 410, 139, 377, 377, 80, 396, - /* 550 */ 377, 399, 395, 385, 377, 377, 138, 410, 139, 377, - /* 560 */ 377, 407, 396, 377, 399, 395, 385, 377, 377, 138, - /* 570 */ 410, 139, 377, 377, 406, 396, 377, 399, 395, 385, - /* 580 */ 377, 377, 138, 410, 139, 377, 377, 405, 396, 377, - /* 590 */ 399, 395, 385, 377, 377, 138, 410, 139, 377, 377, - /* 600 */ 87, 396, 377, 399, 377, 395, 385, 377, 377, 138, - /* 610 */ 410, 139, 377, 377, 86, 396, 377, 399, 395, 385, - /* 620 */ 377, 377, 138, 410, 139, 377, 377, 88, 396, 377, - /* 630 */ 399, 395, 385, 377, 377, 138, 410, 139, 377, 377, - /* 640 */ 85, 396, 377, 399, 395, 385, 377, 377, 138, 410, - /* 650 */ 139, 377, 377, 82, 396, 377, 399, 122, 43, 377, - /* 660 */ 377, 377, 122, 43, 377, 377, 41, 377, 377, 122, - /* 670 */ 43, 41, 377, 377, 124, 63, 440, 441, 41, 445, - /* 680 */ 48, 440, 441, 377, 445, 377, 377, 68, 440, 441, - /* 690 */ 377, 445, + /* 0 */ 393, 391, 92, 23, 66, 394, 391, 469, 30, 28, + /* 10 */ 26, 25, 24, 401, 391, 141, 416, 141, 416, 142, + /* 20 */ 72, 112, 46, 402, 265, 405, 22, 87, 96, 117, + /* 30 */ 283, 284, 285, 286, 287, 288, 289, 291, 292, 293, + /* 40 */ 30, 28, 26, 25, 24, 401, 391, 141, 416, 141, + /* 50 */ 416, 142, 146, 116, 46, 402, 54, 405, 22, 87, + /* 60 */ 96, 514, 283, 284, 285, 286, 287, 288, 289, 291, + /* 70 */ 292, 293, 131, 401, 391, 128, 9, 141, 416, 142, + /* 80 */ 4, 318, 39, 402, 54, 405, 441, 109, 29, 27, + /* 90 */ 86, 437, 132, 513, 63, 245, 246, 247, 248, 143, + /* 100 */ 251, 500, 101, 1, 118, 113, 111, 10, 245, 246, + /* 110 */ 247, 248, 143, 251, 500, 53, 453, 401, 391, 498, + /* 120 */ 258, 141, 416, 142, 126, 54, 40, 402, 499, 405, + /* 130 */ 441, 453, 498, 450, 94, 437, 453, 401, 391, 69, + /* 140 */ 330, 141, 416, 142, 473, 417, 40, 402, 449, 405, + /* 150 */ 441, 134, 256, 448, 94, 437, 41, 401, 391, 344, + /* 160 */ 8, 141, 416, 142, 496, 74, 40, 402, 347, 405, + /* 170 */ 441, 458, 110, 318, 94, 437, 131, 401, 391, 107, + /* 180 */ 322, 141, 416, 142, 457, 57, 78, 402, 60, 405, + /* 190 */ 128, 9, 29, 27, 108, 345, 346, 348, 349, 245, + /* 200 */ 246, 247, 248, 143, 251, 500, 101, 1, 19, 31, + /* 210 */ 133, 10, 295, 54, 30, 28, 26, 25, 24, 53, + /* 220 */ 29, 27, 321, 498, 26, 25, 24, 245, 246, 247, + /* 230 */ 248, 143, 251, 480, 101, 5, 401, 391, 7, 6, + /* 240 */ 141, 416, 142, 7, 6, 40, 402, 251, 405, 441, + /* 250 */ 122, 138, 280, 440, 437, 55, 398, 54, 396, 401, + /* 260 */ 391, 301, 105, 141, 416, 142, 379, 380, 40, 402, + /* 270 */ 106, 405, 441, 29, 27, 323, 121, 437, 479, 59, + /* 280 */ 245, 246, 247, 248, 143, 251, 93, 101, 5, 30, + /* 290 */ 28, 26, 25, 24, 401, 391, 139, 131, 129, 416, + /* 300 */ 142, 125, 44, 45, 402, 265, 405, 3, 460, 135, + /* 310 */ 42, 120, 20, 30, 28, 26, 25, 24, 62, 67, + /* 320 */ 446, 124, 290, 123, 70, 294, 500, 259, 104, 21, + /* 330 */ 102, 466, 48, 64, 2, 30, 28, 26, 25, 24, + /* 340 */ 53, 16, 31, 423, 498, 262, 71, 318, 255, 29, + /* 350 */ 27, 130, 136, 43, 258, 454, 245, 246, 247, 248, + /* 360 */ 143, 251, 32, 101, 5, 29, 27, 30, 28, 26, + /* 370 */ 25, 24, 245, 246, 247, 248, 143, 251, 65, 101, + /* 380 */ 1, 401, 391, 259, 470, 141, 416, 142, 97, 140, + /* 390 */ 40, 402, 515, 405, 441, 29, 27, 497, 137, 438, + /* 400 */ 73, 256, 245, 246, 247, 248, 143, 251, 56, 101, + /* 410 */ 5, 13, 30, 28, 26, 25, 24, 31, 14, 401, + /* 420 */ 391, 341, 58, 141, 416, 142, 376, 377, 84, 402, + /* 430 */ 100, 405, 401, 391, 35, 343, 129, 416, 142, 47, + /* 440 */ 61, 45, 402, 337, 405, 36, 114, 401, 391, 382, + /* 450 */ 144, 141, 416, 142, 336, 37, 84, 402, 103, 405, + /* 460 */ 115, 396, 401, 391, 18, 315, 141, 416, 142, 467, + /* 470 */ 6, 80, 402, 15, 405, 281, 314, 401, 391, 500, + /* 480 */ 68, 141, 416, 142, 33, 34, 84, 402, 95, 405, + /* 490 */ 395, 52, 17, 53, 263, 401, 391, 498, 370, 141, + /* 500 */ 416, 142, 11, 119, 79, 402, 38, 405, 401, 391, + /* 510 */ 365, 75, 141, 416, 142, 364, 98, 81, 402, 369, + /* 520 */ 405, 368, 401, 391, 385, 99, 141, 416, 142, 249, + /* 530 */ 12, 76, 402, 384, 405, 145, 383, 401, 391, 383, + /* 540 */ 383, 141, 416, 142, 383, 383, 82, 402, 383, 405, + /* 550 */ 383, 383, 401, 391, 383, 383, 141, 416, 142, 383, + /* 560 */ 383, 77, 402, 383, 405, 383, 383, 383, 383, 383, + /* 570 */ 401, 391, 383, 383, 141, 416, 142, 383, 383, 83, + /* 580 */ 402, 383, 405, 401, 391, 383, 383, 141, 416, 142, + /* 590 */ 383, 383, 413, 402, 383, 405, 383, 401, 391, 383, + /* 600 */ 383, 141, 416, 142, 383, 383, 412, 402, 383, 405, + /* 610 */ 383, 383, 401, 391, 383, 383, 141, 416, 142, 383, + /* 620 */ 383, 411, 402, 383, 405, 383, 383, 401, 391, 383, + /* 630 */ 383, 141, 416, 142, 383, 383, 90, 402, 383, 405, + /* 640 */ 383, 383, 383, 383, 383, 401, 391, 383, 383, 141, + /* 650 */ 416, 142, 383, 383, 89, 402, 383, 405, 401, 391, + /* 660 */ 383, 383, 141, 416, 142, 383, 383, 91, 402, 383, + /* 670 */ 405, 383, 401, 391, 383, 383, 141, 416, 142, 383, + /* 680 */ 383, 88, 402, 383, 405, 125, 44, 401, 391, 383, + /* 690 */ 383, 141, 416, 142, 42, 383, 85, 402, 383, 405, + /* 700 */ 125, 44, 127, 49, 446, 447, 383, 451, 383, 42, + /* 710 */ 383, 383, 383, 383, 125, 44, 383, 383, 50, 446, + /* 720 */ 447, 383, 451, 42, 383, 383, 383, 383, 383, 383, + /* 730 */ 383, 383, 51, 446, 447, 383, 451, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 74, 75, 76, 88, 89, 74, 75, 82, 8, 9, /* 10 */ 10, 11, 12, 74, 75, 78, 79, 78, 79, 80, - /* 20 */ 122, 84, 83, 84, 24, 86, 26, 27, 28, 13, + /* 20 */ 123, 84, 83, 84, 24, 86, 26, 27, 28, 22, /* 30 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, /* 40 */ 8, 9, 10, 11, 12, 74, 75, 78, 79, 78, - /* 50 */ 79, 80, 4, 84, 83, 84, 85, 86, 26, 27, - /* 60 */ 28, 45, 30, 31, 32, 33, 34, 35, 36, 37, - /* 70 */ 38, 39, 73, 74, 75, 22, 81, 78, 79, 80, - /* 80 */ 22, 23, 83, 84, 45, 86, 87, 22, 8, 9, - /* 90 */ 91, 92, 102, 98, 46, 15, 16, 17, 18, 19, - /* 100 */ 20, 102, 22, 23, 29, 115, 41, 27, 114, 119, - /* 110 */ 10, 74, 75, 107, 115, 78, 79, 80, 119, 100, - /* 120 */ 83, 84, 22, 86, 87, 45, 22, 23, 91, 92, - /* 130 */ 55, 56, 57, 58, 59, 74, 75, 116, 101, 78, - /* 140 */ 79, 80, 43, 44, 83, 84, 79, 86, 87, 45, - /* 150 */ 81, 81, 91, 92, 15, 16, 17, 18, 19, 20, - /* 160 */ 74, 75, 101, 103, 78, 79, 80, 98, 98, 83, - /* 170 */ 84, 21, 86, 87, 24, 2, 54, 91, 92, 1, - /* 180 */ 2, 8, 9, 10, 11, 12, 21, 101, 73, 74, - /* 190 */ 75, 1, 2, 78, 79, 80, 8, 9, 83, 84, - /* 200 */ 113, 86, 24, 15, 16, 17, 18, 19, 20, 53, - /* 210 */ 22, 23, 8, 9, 23, 27, 25, 102, 112, 15, - /* 220 */ 16, 17, 18, 19, 20, 20, 22, 23, 75, 21, - /* 230 */ 115, 66, 24, 26, 119, 74, 75, 3, 75, 78, - /* 240 */ 79, 80, 113, 36, 83, 84, 39, 86, 87, 45, - /* 250 */ 70, 71, 91, 92, 10, 11, 12, 8, 9, 10, - /* 260 */ 11, 12, 21, 74, 75, 24, 112, 78, 79, 80, - /* 270 */ 75, 22, 83, 84, 21, 86, 87, 8, 9, 10, - /* 280 */ 91, 92, 61, 109, 15, 16, 17, 18, 19, 20, - /* 290 */ 41, 22, 23, 60, 74, 75, 62, 73, 78, 79, - /* 300 */ 80, 77, 78, 83, 84, 2, 86, 27, 108, 29, - /* 310 */ 86, 8, 9, 10, 11, 12, 48, 64, 106, 95, - /* 320 */ 96, 97, 21, 99, 104, 24, 102, 50, 51, 52, - /* 330 */ 110, 111, 8, 9, 10, 11, 12, 47, 44, 115, - /* 340 */ 8, 9, 10, 119, 105, 23, 90, 15, 16, 17, - /* 350 */ 18, 19, 20, 78, 22, 23, 8, 9, 8, 9, - /* 360 */ 10, 11, 12, 15, 16, 17, 18, 19, 20, 22, - /* 370 */ 22, 23, 74, 75, 24, 22, 78, 79, 80, 40, - /* 380 */ 81, 83, 84, 22, 86, 87, 8, 9, 93, 82, - /* 390 */ 92, 67, 68, 15, 16, 17, 18, 19, 20, 69, - /* 400 */ 22, 23, 123, 65, 118, 74, 75, 22, 63, 78, - /* 410 */ 79, 80, 21, 117, 83, 84, 21, 86, 74, 75, - /* 420 */ 49, 24, 78, 79, 80, 24, 23, 83, 84, 21, - /* 430 */ 86, 74, 75, 24, 23, 78, 79, 80, 23, 23, - /* 440 */ 83, 84, 85, 86, 15, 72, 73, 24, 24, 21, - /* 450 */ 25, 120, 121, 74, 75, 111, 2, 78, 79, 80, - /* 460 */ 23, 49, 83, 84, 85, 86, 24, 29, 74, 75, - /* 470 */ 24, 21, 78, 79, 80, 102, 25, 83, 84, 85, - /* 480 */ 86, 74, 75, 42, 25, 78, 79, 80, 115, 25, - /* 490 */ 83, 84, 119, 86, 74, 75, 24, 24, 78, 79, - /* 500 */ 80, 21, 49, 83, 84, 4, 86, 74, 75, 15, - /* 510 */ 15, 78, 79, 80, 15, 15, 83, 84, 15, 86, - /* 520 */ 8, 9, 10, 11, 12, 15, 74, 75, 121, 23, - /* 530 */ 78, 79, 80, 25, 0, 83, 84, 17, 86, 74, - /* 540 */ 75, 0, 14, 78, 79, 80, 124, 124, 83, 84, - /* 550 */ 124, 86, 74, 75, 124, 124, 78, 79, 80, 124, - /* 560 */ 124, 83, 84, 124, 86, 74, 75, 124, 124, 78, - /* 570 */ 79, 80, 124, 124, 83, 84, 124, 86, 74, 75, - /* 580 */ 124, 124, 78, 79, 80, 124, 124, 83, 84, 124, - /* 590 */ 86, 74, 75, 124, 124, 78, 79, 80, 124, 124, - /* 600 */ 83, 84, 124, 86, 124, 74, 75, 124, 124, 78, - /* 610 */ 79, 80, 124, 124, 83, 84, 124, 86, 74, 75, - /* 620 */ 124, 124, 78, 79, 80, 124, 124, 83, 84, 124, - /* 630 */ 86, 74, 75, 124, 124, 78, 79, 80, 124, 124, - /* 640 */ 83, 84, 124, 86, 74, 75, 124, 124, 78, 79, - /* 650 */ 80, 124, 124, 83, 84, 124, 86, 77, 78, 124, - /* 660 */ 124, 124, 77, 78, 124, 124, 86, 124, 124, 77, - /* 670 */ 78, 86, 124, 124, 94, 95, 96, 97, 86, 99, - /* 680 */ 95, 96, 97, 124, 99, 124, 124, 95, 96, 97, - /* 690 */ 124, 99, + /* 50 */ 79, 80, 13, 84, 83, 84, 45, 86, 26, 27, + /* 60 */ 28, 122, 30, 31, 32, 33, 34, 35, 36, 37, + /* 70 */ 38, 39, 73, 74, 75, 22, 23, 78, 79, 80, + /* 80 */ 43, 44, 83, 84, 45, 86, 87, 114, 8, 9, + /* 90 */ 91, 92, 121, 122, 107, 15, 16, 17, 18, 19, + /* 100 */ 20, 102, 22, 23, 50, 51, 52, 27, 15, 16, + /* 110 */ 17, 18, 19, 20, 102, 116, 81, 74, 75, 120, + /* 120 */ 22, 78, 79, 80, 100, 45, 83, 84, 116, 86, + /* 130 */ 87, 81, 120, 98, 91, 92, 81, 74, 75, 41, + /* 140 */ 10, 78, 79, 80, 101, 79, 83, 84, 98, 86, + /* 150 */ 87, 3, 22, 98, 91, 92, 21, 74, 75, 24, + /* 160 */ 103, 78, 79, 80, 101, 117, 83, 84, 29, 86, + /* 170 */ 87, 42, 54, 44, 91, 92, 73, 74, 75, 53, + /* 180 */ 4, 78, 79, 80, 101, 21, 83, 84, 24, 86, + /* 190 */ 22, 23, 8, 9, 55, 56, 57, 58, 59, 15, + /* 200 */ 16, 17, 18, 19, 20, 102, 22, 23, 2, 21, + /* 210 */ 62, 27, 24, 45, 8, 9, 10, 11, 12, 116, + /* 220 */ 8, 9, 46, 120, 10, 11, 12, 15, 16, 17, + /* 230 */ 18, 19, 20, 113, 22, 23, 74, 75, 1, 2, + /* 240 */ 78, 79, 80, 1, 2, 83, 84, 20, 86, 87, + /* 250 */ 27, 21, 29, 91, 92, 112, 23, 45, 25, 74, + /* 260 */ 75, 24, 75, 78, 79, 80, 70, 71, 83, 84, + /* 270 */ 75, 86, 87, 8, 9, 10, 91, 92, 113, 112, + /* 280 */ 15, 16, 17, 18, 19, 20, 75, 22, 23, 8, + /* 290 */ 9, 10, 11, 12, 74, 75, 66, 73, 78, 79, + /* 300 */ 80, 77, 78, 83, 84, 24, 86, 61, 109, 21, + /* 310 */ 86, 60, 26, 8, 9, 10, 11, 12, 108, 95, + /* 320 */ 96, 97, 36, 99, 104, 39, 102, 22, 48, 2, + /* 330 */ 110, 111, 106, 105, 47, 8, 9, 10, 11, 12, + /* 340 */ 116, 23, 21, 90, 120, 24, 41, 44, 22, 8, + /* 350 */ 9, 10, 64, 78, 22, 81, 15, 16, 17, 18, + /* 360 */ 19, 20, 40, 22, 23, 8, 9, 8, 9, 10, + /* 370 */ 11, 12, 15, 16, 17, 18, 19, 20, 93, 22, + /* 380 */ 23, 74, 75, 22, 82, 78, 79, 80, 69, 65, + /* 390 */ 83, 84, 124, 86, 87, 8, 9, 119, 63, 92, + /* 400 */ 118, 22, 15, 16, 17, 18, 19, 20, 24, 22, + /* 410 */ 23, 21, 8, 9, 10, 11, 12, 21, 49, 74, + /* 420 */ 75, 24, 23, 78, 79, 80, 67, 68, 83, 84, + /* 430 */ 85, 86, 74, 75, 21, 24, 78, 79, 80, 23, + /* 440 */ 23, 83, 84, 24, 86, 23, 15, 74, 75, 72, + /* 450 */ 73, 78, 79, 80, 24, 23, 83, 84, 85, 86, + /* 460 */ 21, 25, 74, 75, 21, 24, 78, 79, 80, 111, + /* 470 */ 2, 83, 84, 49, 86, 29, 24, 74, 75, 102, + /* 480 */ 25, 78, 79, 80, 42, 21, 83, 84, 85, 86, + /* 490 */ 25, 25, 21, 116, 24, 74, 75, 120, 24, 78, + /* 500 */ 79, 80, 49, 115, 83, 84, 4, 86, 74, 75, + /* 510 */ 15, 25, 78, 79, 80, 15, 15, 83, 84, 15, + /* 520 */ 86, 15, 74, 75, 0, 15, 78, 79, 80, 17, + /* 530 */ 23, 83, 84, 0, 86, 14, 125, 74, 75, 125, + /* 540 */ 125, 78, 79, 80, 125, 125, 83, 84, 125, 86, + /* 550 */ 125, 125, 74, 75, 125, 125, 78, 79, 80, 125, + /* 560 */ 125, 83, 84, 125, 86, 125, 125, 125, 125, 125, + /* 570 */ 74, 75, 125, 125, 78, 79, 80, 125, 125, 83, + /* 580 */ 84, 125, 86, 74, 75, 125, 125, 78, 79, 80, + /* 590 */ 125, 125, 83, 84, 125, 86, 125, 74, 75, 125, + /* 600 */ 125, 78, 79, 80, 125, 125, 83, 84, 125, 86, + /* 610 */ 125, 125, 74, 75, 125, 125, 78, 79, 80, 125, + /* 620 */ 125, 83, 84, 125, 86, 125, 125, 74, 75, 125, + /* 630 */ 125, 78, 79, 80, 125, 125, 83, 84, 125, 86, + /* 640 */ 125, 125, 125, 125, 125, 74, 75, 125, 125, 78, + /* 650 */ 79, 80, 125, 125, 83, 84, 125, 86, 74, 75, + /* 660 */ 125, 125, 78, 79, 80, 125, 125, 83, 84, 125, + /* 670 */ 86, 125, 74, 75, 125, 125, 78, 79, 80, 125, + /* 680 */ 125, 83, 84, 125, 86, 77, 78, 74, 75, 125, + /* 690 */ 125, 78, 79, 80, 86, 125, 83, 84, 125, 86, + /* 700 */ 77, 78, 94, 95, 96, 97, 125, 99, 125, 86, + /* 710 */ 125, 125, 125, 125, 77, 78, 125, 125, 95, 96, + /* 720 */ 97, 125, 99, 86, 125, 125, 125, 125, 125, 125, + /* 730 */ 125, 125, 95, 96, 97, 125, 99, }; -#define YY_SHIFT_COUNT (143) +#define YY_SHIFT_COUNT (146) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (541) +#define YY_SHIFT_MAX (533) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 16, 80, 188, 188, 188, 204, 188, 188, 269, 104, - /* 10 */ 348, 378, 332, 378, 378, 378, 378, 378, 378, 378, - /* 20 */ 378, 378, 378, 378, 378, 378, 378, 378, 378, 378, - /* 30 */ 378, 58, 58, 58, 139, 53, 53, 39, 0, 32, - /* 40 */ 139, 65, 65, 65, 249, 324, 75, 277, 99, 100, - /* 50 */ 234, 48, 122, 156, 205, 205, 122, 156, 205, 221, - /* 60 */ 233, 268, 290, 294, 322, 294, 347, 353, 294, 339, - /* 70 */ 361, 330, 338, 345, 385, 173, 303, 350, 512, 512, - /* 80 */ 512, 512, 512, 178, 207, 244, 244, 244, 244, 150, - /* 90 */ 208, 190, 241, 280, 180, 253, 165, 301, 191, 391, - /* 100 */ 395, 371, 397, 401, 403, 408, 409, 411, 415, 423, - /* 110 */ 416, 424, 429, 428, 425, 437, 395, 412, 454, 438, - /* 120 */ 442, 446, 451, 441, 450, 459, 464, 472, 473, 480, - /* 130 */ 453, 501, 494, 495, 499, 500, 503, 510, 508, 506, - /* 140 */ 520, 534, 541, 528, + /* 0 */ 39, 80, 184, 184, 184, 212, 184, 184, 265, 168, + /* 10 */ 357, 387, 341, 387, 387, 387, 387, 387, 387, 387, + /* 20 */ 387, 387, 387, 387, 387, 387, 387, 387, 387, 387, + /* 30 */ 387, 387, 53, 53, 53, 93, 7, 7, 11, 0, + /* 40 */ 32, 93, 98, 98, 98, 305, 359, 139, 54, 129, + /* 50 */ 37, 129, 130, 148, 176, 118, 126, 227, 227, 118, + /* 60 */ 126, 227, 246, 251, 280, 287, 318, 303, 326, 332, + /* 70 */ 322, 361, 319, 324, 335, 379, 206, 327, 281, 404, + /* 80 */ 404, 404, 404, 404, 404, 404, 237, 286, 214, 214, + /* 90 */ 214, 214, 135, 164, 242, 188, 223, 196, 288, 230, + /* 100 */ 321, 233, 390, 396, 369, 384, 397, 399, 413, 411, + /* 110 */ 416, 417, 419, 422, 430, 431, 439, 436, 432, 443, + /* 120 */ 424, 468, 446, 441, 452, 455, 442, 464, 465, 466, + /* 130 */ 470, 474, 471, 453, 502, 495, 500, 501, 504, 506, + /* 140 */ 510, 486, 507, 512, 524, 533, 521, }; -#define YY_REDUCE_COUNT (74) -#define YY_REDUCE_MIN (-102) -#define YY_REDUCE_MAX (592) +#define YY_REDUCE_COUNT (75) +#define YY_REDUCE_MIN (-103) +#define YY_REDUCE_MAX (637) static const short yy_reduce_ofst[] = { - /* 0 */ 373, -1, 37, 61, 86, 115, 161, 189, 220, 224, - /* 10 */ 298, 331, -29, 344, 357, 379, 394, 407, -61, 420, - /* 20 */ 433, 452, 465, 478, 491, 504, 517, 531, 544, 557, - /* 30 */ 570, 580, 585, 592, -74, -63, -31, -10, -85, -85, - /* 40 */ -69, -5, 69, 70, -75, -102, -6, 6, 19, 67, - /* 50 */ 21, 60, 87, 106, 153, 163, 129, 154, 195, 174, - /* 60 */ 200, 212, 239, 19, 256, 19, 275, 299, 19, 295, - /* 70 */ 307, 279, 286, 296, 67, + /* 0 */ 377, -1, 43, 63, 83, 103, 162, 185, 220, 224, + /* 10 */ 307, -29, 345, 358, 373, 388, 403, -61, 421, 434, + /* 20 */ 448, 463, 478, 496, 509, 523, 538, 553, 571, 584, + /* 30 */ 598, 613, 608, 623, 637, -74, -63, -31, 12, -85, + /* 40 */ -85, -69, 35, 50, 55, -75, -103, -27, -13, 24, + /* 50 */ 24, 24, 66, 48, 57, 120, 143, 187, 195, 165, + /* 60 */ 167, 211, 199, 210, 226, 228, 253, 24, 275, 274, + /* 70 */ 285, 302, 268, 278, 282, 66, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 10 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 20 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 30 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 40 */ 375, 446, 446, 446, 461, 507, 375, 469, 375, 375, - /* 50 */ 492, 454, 476, 474, 375, 375, 476, 474, 375, 486, - /* 60 */ 484, 467, 465, 438, 375, 375, 375, 375, 439, 375, - /* 70 */ 375, 510, 498, 494, 375, 375, 375, 375, 414, 413, - /* 80 */ 412, 408, 409, 375, 375, 403, 404, 402, 401, 375, - /* 90 */ 375, 503, 375, 375, 375, 495, 499, 375, 391, 458, - /* 100 */ 468, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 110 */ 375, 375, 375, 375, 391, 375, 485, 375, 433, 375, - /* 120 */ 445, 441, 375, 375, 437, 390, 375, 375, 375, 493, - /* 130 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 140 */ 375, 375, 375, 375, + /* 0 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, + /* 10 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, + /* 20 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, + /* 30 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, + /* 40 */ 381, 381, 452, 452, 452, 468, 516, 381, 476, 444, + /* 50 */ 458, 445, 381, 501, 461, 483, 481, 381, 381, 483, + /* 60 */ 481, 381, 495, 491, 474, 472, 381, 458, 381, 381, + /* 70 */ 381, 381, 519, 507, 503, 381, 381, 381, 381, 494, + /* 80 */ 493, 420, 419, 418, 414, 415, 381, 381, 409, 410, + /* 90 */ 408, 407, 381, 381, 512, 381, 381, 381, 504, 508, + /* 100 */ 381, 397, 465, 475, 381, 381, 381, 381, 381, 381, + /* 110 */ 381, 381, 381, 381, 381, 381, 381, 397, 381, 492, + /* 120 */ 381, 439, 381, 451, 447, 381, 381, 443, 396, 381, + /* 130 */ 381, 381, 502, 381, 381, 381, 381, 381, 381, 381, + /* 140 */ 381, 381, 381, 381, 381, 381, 381, }; /********** End of lemon-generated parsing tables *****************************/ @@ -630,15 +638,16 @@ static const char *const yyTokenName[] = { /* 112 */ "sliding_opt", /* 113 */ "fill_opt", /* 114 */ "fill_mode", - /* 115 */ "query_expression_body", - /* 116 */ "order_by_clause_opt", - /* 117 */ "slimit_clause_opt", - /* 118 */ "limit_clause_opt", - /* 119 */ "query_primary", - /* 120 */ "sort_specification_list", - /* 121 */ "sort_specification", - /* 122 */ "ordering_specification_opt", - /* 123 */ "null_ordering_opt", + /* 115 */ "group_by_list", + /* 116 */ "query_expression_body", + /* 117 */ "order_by_clause_opt", + /* 118 */ "slimit_clause_opt", + /* 119 */ "limit_clause_opt", + /* 120 */ "query_primary", + /* 121 */ "sort_specification_list", + /* 122 */ "sort_specification", + /* 123 */ "ordering_specification_opt", + /* 124 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -720,67 +729,70 @@ static const char *const yyRuleName[] = { /* 71 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", /* 72 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", /* 73 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 74 */ "join_type ::= INNER", - /* 75 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 76 */ "set_quantifier_opt ::=", - /* 77 */ "set_quantifier_opt ::= DISTINCT", - /* 78 */ "set_quantifier_opt ::= ALL", - /* 79 */ "select_list ::= NK_STAR", - /* 80 */ "select_list ::= select_sublist", - /* 81 */ "select_sublist ::= select_item", - /* 82 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 83 */ "select_item ::= expression", - /* 84 */ "select_item ::= expression column_alias", - /* 85 */ "select_item ::= expression AS column_alias", - /* 86 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 87 */ "where_clause_opt ::=", - /* 88 */ "where_clause_opt ::= WHERE search_condition", - /* 89 */ "partition_by_clause_opt ::=", - /* 90 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 91 */ "twindow_clause_opt ::=", - /* 92 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", - /* 93 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", - /* 94 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 95 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 96 */ "sliding_opt ::=", - /* 97 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 98 */ "fill_opt ::=", - /* 99 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 100 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 101 */ "fill_mode ::= NONE", - /* 102 */ "fill_mode ::= PREV", - /* 103 */ "fill_mode ::= NULL", - /* 104 */ "fill_mode ::= LINEAR", - /* 105 */ "fill_mode ::= NEXT", - /* 106 */ "group_by_clause_opt ::=", - /* 107 */ "group_by_clause_opt ::= GROUP BY expression_list", - /* 108 */ "having_clause_opt ::=", - /* 109 */ "having_clause_opt ::= HAVING search_condition", - /* 110 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 111 */ "query_expression_body ::= query_primary", - /* 112 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 113 */ "query_primary ::= query_specification", - /* 114 */ "order_by_clause_opt ::=", - /* 115 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 116 */ "slimit_clause_opt ::=", - /* 117 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 118 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 119 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 120 */ "limit_clause_opt ::=", - /* 121 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 122 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 123 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 124 */ "subquery ::= NK_LP query_expression NK_RP", - /* 125 */ "search_condition ::= boolean_value_expression", - /* 126 */ "sort_specification_list ::= sort_specification", - /* 127 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 128 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 129 */ "ordering_specification_opt ::=", - /* 130 */ "ordering_specification_opt ::= ASC", - /* 131 */ "ordering_specification_opt ::= DESC", - /* 132 */ "null_ordering_opt ::=", - /* 133 */ "null_ordering_opt ::= NULLS FIRST", - /* 134 */ "null_ordering_opt ::= NULLS LAST", + /* 74 */ "join_type ::=", + /* 75 */ "join_type ::= INNER", + /* 76 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 77 */ "set_quantifier_opt ::=", + /* 78 */ "set_quantifier_opt ::= DISTINCT", + /* 79 */ "set_quantifier_opt ::= ALL", + /* 80 */ "select_list ::= NK_STAR", + /* 81 */ "select_list ::= select_sublist", + /* 82 */ "select_sublist ::= select_item", + /* 83 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 84 */ "select_item ::= expression", + /* 85 */ "select_item ::= expression column_alias", + /* 86 */ "select_item ::= expression AS column_alias", + /* 87 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 88 */ "where_clause_opt ::=", + /* 89 */ "where_clause_opt ::= WHERE search_condition", + /* 90 */ "partition_by_clause_opt ::=", + /* 91 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 92 */ "twindow_clause_opt ::=", + /* 93 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 94 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 95 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 96 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 97 */ "sliding_opt ::=", + /* 98 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 99 */ "fill_opt ::=", + /* 100 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 101 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 102 */ "fill_mode ::= NONE", + /* 103 */ "fill_mode ::= PREV", + /* 104 */ "fill_mode ::= NULL", + /* 105 */ "fill_mode ::= LINEAR", + /* 106 */ "fill_mode ::= NEXT", + /* 107 */ "group_by_clause_opt ::=", + /* 108 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 109 */ "group_by_list ::= expression", + /* 110 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 111 */ "having_clause_opt ::=", + /* 112 */ "having_clause_opt ::= HAVING search_condition", + /* 113 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 114 */ "query_expression_body ::= query_primary", + /* 115 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 116 */ "query_primary ::= query_specification", + /* 117 */ "order_by_clause_opt ::=", + /* 118 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 119 */ "slimit_clause_opt ::=", + /* 120 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 121 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 122 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 123 */ "limit_clause_opt ::=", + /* 124 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 125 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 126 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 127 */ "subquery ::= NK_LP query_expression NK_RP", + /* 128 */ "search_condition ::= boolean_value_expression", + /* 129 */ "sort_specification_list ::= sort_specification", + /* 130 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 131 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 132 */ "ordering_specification_opt ::=", + /* 133 */ "ordering_specification_opt ::= ASC", + /* 134 */ "ordering_specification_opt ::= DESC", + /* 135 */ "null_ordering_opt ::=", + /* 136 */ "null_ordering_opt ::= NULLS FIRST", + /* 137 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -932,13 +944,13 @@ static void yy_destructor( case 111: /* select_item */ case 112: /* sliding_opt */ case 113: /* fill_opt */ - case 115: /* query_expression_body */ - case 117: /* slimit_clause_opt */ - case 118: /* limit_clause_opt */ - case 119: /* query_primary */ - case 121: /* sort_specification */ + case 116: /* query_expression_body */ + case 118: /* slimit_clause_opt */ + case 119: /* limit_clause_opt */ + case 120: /* query_primary */ + case 122: /* sort_specification */ { - PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy168)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy212)); } break; case 76: /* literal_list */ @@ -947,10 +959,11 @@ static void yy_destructor( case 106: /* partition_by_clause_opt */ case 108: /* group_by_clause_opt */ case 110: /* select_sublist */ - case 116: /* order_by_clause_opt */ - case 120: /* sort_specification_list */ + case 115: /* group_by_list */ + case 117: /* order_by_clause_opt */ + case 121: /* sort_specification_list */ { - PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy192)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy174)); } break; case 77: /* db_name */ @@ -985,12 +998,12 @@ static void yy_destructor( PARSER_DESTRUCTOR_TRACE; } break; - case 122: /* ordering_specification_opt */ + case 123: /* ordering_specification_opt */ { PARSER_DESTRUCTOR_TRACE; } break; - case 123: /* null_ordering_opt */ + case 124: /* null_ordering_opt */ { PARSER_DESTRUCTOR_TRACE; } @@ -1363,67 +1376,70 @@ static const struct { { 99, -3 }, /* (71) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ { 99, -3 }, /* (72) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ { 97, -6 }, /* (73) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 100, -1 }, /* (74) join_type ::= INNER */ - { 102, -9 }, /* (75) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 103, 0 }, /* (76) set_quantifier_opt ::= */ - { 103, -1 }, /* (77) set_quantifier_opt ::= DISTINCT */ - { 103, -1 }, /* (78) set_quantifier_opt ::= ALL */ - { 104, -1 }, /* (79) select_list ::= NK_STAR */ - { 104, -1 }, /* (80) select_list ::= select_sublist */ - { 110, -1 }, /* (81) select_sublist ::= select_item */ - { 110, -3 }, /* (82) select_sublist ::= select_sublist NK_COMMA select_item */ - { 111, -1 }, /* (83) select_item ::= expression */ - { 111, -2 }, /* (84) select_item ::= expression column_alias */ - { 111, -3 }, /* (85) select_item ::= expression AS column_alias */ - { 111, -3 }, /* (86) select_item ::= table_name NK_DOT NK_STAR */ - { 105, 0 }, /* (87) where_clause_opt ::= */ - { 105, -2 }, /* (88) where_clause_opt ::= WHERE search_condition */ - { 106, 0 }, /* (89) partition_by_clause_opt ::= */ - { 106, -3 }, /* (90) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 107, 0 }, /* (91) twindow_clause_opt ::= */ - { 107, -6 }, /* (92) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ - { 107, -4 }, /* (93) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ - { 107, -6 }, /* (94) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 107, -8 }, /* (95) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 112, 0 }, /* (96) sliding_opt ::= */ - { 112, -4 }, /* (97) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 113, 0 }, /* (98) fill_opt ::= */ - { 113, -4 }, /* (99) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 113, -6 }, /* (100) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 114, -1 }, /* (101) fill_mode ::= NONE */ - { 114, -1 }, /* (102) fill_mode ::= PREV */ - { 114, -1 }, /* (103) fill_mode ::= NULL */ - { 114, -1 }, /* (104) fill_mode ::= LINEAR */ - { 114, -1 }, /* (105) fill_mode ::= NEXT */ - { 108, 0 }, /* (106) group_by_clause_opt ::= */ - { 108, -3 }, /* (107) group_by_clause_opt ::= GROUP BY expression_list */ - { 109, 0 }, /* (108) having_clause_opt ::= */ - { 109, -2 }, /* (109) having_clause_opt ::= HAVING search_condition */ - { 73, -4 }, /* (110) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 115, -1 }, /* (111) query_expression_body ::= query_primary */ - { 115, -4 }, /* (112) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 119, -1 }, /* (113) query_primary ::= query_specification */ - { 116, 0 }, /* (114) order_by_clause_opt ::= */ - { 116, -3 }, /* (115) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 117, 0 }, /* (116) slimit_clause_opt ::= */ - { 117, -2 }, /* (117) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 117, -4 }, /* (118) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 117, -4 }, /* (119) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 118, 0 }, /* (120) limit_clause_opt ::= */ - { 118, -2 }, /* (121) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 118, -4 }, /* (122) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 118, -4 }, /* (123) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 86, -3 }, /* (124) subquery ::= NK_LP query_expression NK_RP */ - { 101, -1 }, /* (125) search_condition ::= boolean_value_expression */ - { 120, -1 }, /* (126) sort_specification_list ::= sort_specification */ - { 120, -3 }, /* (127) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 121, -3 }, /* (128) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 122, 0 }, /* (129) ordering_specification_opt ::= */ - { 122, -1 }, /* (130) ordering_specification_opt ::= ASC */ - { 122, -1 }, /* (131) ordering_specification_opt ::= DESC */ - { 123, 0 }, /* (132) null_ordering_opt ::= */ - { 123, -2 }, /* (133) null_ordering_opt ::= NULLS FIRST */ - { 123, -2 }, /* (134) null_ordering_opt ::= NULLS LAST */ + { 100, 0 }, /* (74) join_type ::= */ + { 100, -1 }, /* (75) join_type ::= INNER */ + { 102, -9 }, /* (76) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 103, 0 }, /* (77) set_quantifier_opt ::= */ + { 103, -1 }, /* (78) set_quantifier_opt ::= DISTINCT */ + { 103, -1 }, /* (79) set_quantifier_opt ::= ALL */ + { 104, -1 }, /* (80) select_list ::= NK_STAR */ + { 104, -1 }, /* (81) select_list ::= select_sublist */ + { 110, -1 }, /* (82) select_sublist ::= select_item */ + { 110, -3 }, /* (83) select_sublist ::= select_sublist NK_COMMA select_item */ + { 111, -1 }, /* (84) select_item ::= expression */ + { 111, -2 }, /* (85) select_item ::= expression column_alias */ + { 111, -3 }, /* (86) select_item ::= expression AS column_alias */ + { 111, -3 }, /* (87) select_item ::= table_name NK_DOT NK_STAR */ + { 105, 0 }, /* (88) where_clause_opt ::= */ + { 105, -2 }, /* (89) where_clause_opt ::= WHERE search_condition */ + { 106, 0 }, /* (90) partition_by_clause_opt ::= */ + { 106, -3 }, /* (91) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 107, 0 }, /* (92) twindow_clause_opt ::= */ + { 107, -6 }, /* (93) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 107, -4 }, /* (94) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 107, -6 }, /* (95) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 107, -8 }, /* (96) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 112, 0 }, /* (97) sliding_opt ::= */ + { 112, -4 }, /* (98) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 113, 0 }, /* (99) fill_opt ::= */ + { 113, -4 }, /* (100) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 113, -6 }, /* (101) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 114, -1 }, /* (102) fill_mode ::= NONE */ + { 114, -1 }, /* (103) fill_mode ::= PREV */ + { 114, -1 }, /* (104) fill_mode ::= NULL */ + { 114, -1 }, /* (105) fill_mode ::= LINEAR */ + { 114, -1 }, /* (106) fill_mode ::= NEXT */ + { 108, 0 }, /* (107) group_by_clause_opt ::= */ + { 108, -3 }, /* (108) group_by_clause_opt ::= GROUP BY group_by_list */ + { 115, -1 }, /* (109) group_by_list ::= expression */ + { 115, -3 }, /* (110) group_by_list ::= group_by_list NK_COMMA expression */ + { 109, 0 }, /* (111) having_clause_opt ::= */ + { 109, -2 }, /* (112) having_clause_opt ::= HAVING search_condition */ + { 73, -4 }, /* (113) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 116, -1 }, /* (114) query_expression_body ::= query_primary */ + { 116, -4 }, /* (115) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 120, -1 }, /* (116) query_primary ::= query_specification */ + { 117, 0 }, /* (117) order_by_clause_opt ::= */ + { 117, -3 }, /* (118) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 118, 0 }, /* (119) slimit_clause_opt ::= */ + { 118, -2 }, /* (120) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 118, -4 }, /* (121) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 118, -4 }, /* (122) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 119, 0 }, /* (123) limit_clause_opt ::= */ + { 119, -2 }, /* (124) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 119, -4 }, /* (125) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 119, -4 }, /* (126) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 86, -3 }, /* (127) subquery ::= NK_LP query_expression NK_RP */ + { 101, -1 }, /* (128) search_condition ::= boolean_value_expression */ + { 121, -1 }, /* (129) sort_specification_list ::= sort_specification */ + { 121, -3 }, /* (130) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 122, -3 }, /* (131) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 123, 0 }, /* (132) ordering_specification_opt ::= */ + { 123, -1 }, /* (133) ordering_specification_opt ::= ASC */ + { 123, -1 }, /* (134) ordering_specification_opt ::= DESC */ + { 124, 0 }, /* (135) null_ordering_opt ::= */ + { 124, -2 }, /* (136) null_ordering_opt ::= NULLS FIRST */ + { 124, -2 }, /* (137) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1514,27 +1530,27 @@ static YYACTIONTYPE yy_reduce( { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } break; case 1: /* cmd ::= query_expression */ -{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy168; } +{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy212; } break; case 2: /* literal ::= NK_INTEGER */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 3: /* literal ::= NK_FLOAT */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 4: /* literal ::= NK_STRING */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 5: /* literal ::= NK_BOOL */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 6: /* literal ::= TIMESTAMP NK_STRING */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 7: /* literal ::= duration_literal */ case 17: /* expression ::= literal */ yytestcase(yyruleno==17); @@ -1546,25 +1562,25 @@ static YYACTIONTYPE yy_reduce( case 62: /* table_reference ::= table_primary */ yytestcase(yyruleno==62); case 63: /* table_reference ::= joined_table */ yytestcase(yyruleno==63); case 67: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==67); - case 111: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==111); - case 113: /* query_primary ::= query_specification */ yytestcase(yyruleno==113); - case 125: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==125); -{ PARSER_TRACE; yylhsminor.yy168 = yymsp[0].minor.yy168; } - yymsp[0].minor.yy168 = yylhsminor.yy168; + case 114: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==114); + case 116: /* query_primary ::= query_specification */ yytestcase(yyruleno==116); + case 128: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==128); +{ PARSER_TRACE; yylhsminor.yy212 = yymsp[0].minor.yy212; } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 8: /* duration_literal ::= NK_VARIABLE */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 9: /* literal_list ::= literal */ case 30: /* expression_list ::= expression */ yytestcase(yyruleno==30); -{ PARSER_TRACE; yylhsminor.yy192 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[0].minor.yy192 = yylhsminor.yy192; +{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; case 10: /* literal_list ::= literal_list NK_COMMA literal */ case 31: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==31); -{ PARSER_TRACE; yylhsminor.yy192 = addNodeToList(pCxt, yymsp[-2].minor.yy192, releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[-2].minor.yy192 = yylhsminor.yy192; +{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[-2].minor.yy174 = yylhsminor.yy174; break; case 11: /* db_name ::= NK_ID */ case 12: /* table_name ::= NK_ID */ yytestcase(yyruleno==12); @@ -1572,369 +1588,380 @@ static YYACTIONTYPE yy_reduce( case 14: /* function_name ::= NK_ID */ yytestcase(yyruleno==14); case 15: /* table_alias ::= NK_ID */ yytestcase(yyruleno==15); case 16: /* column_alias ::= NK_ID */ yytestcase(yyruleno==16); -{ PARSER_TRACE; yylhsminor.yy241 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy241 = yylhsminor.yy241; +{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy79 = yylhsminor.yy79; break; case 19: /* expression ::= function_name NK_LP expression_list NK_RP */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy241, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy241, yymsp[-1].minor.yy192)); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy79, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy79, yymsp[-1].minor.yy174)); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; case 20: /* expression ::= function_name NK_LP NK_STAR NK_RP */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy241, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy241, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy79, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy79, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; case 22: /* expression ::= NK_LP expression NK_RP */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168)); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212)); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 23: /* expression ::= NK_PLUS expression */ { PARSER_TRACE; - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 24: /* expression ::= NK_MINUS expression */ { PARSER_TRACE; - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy168), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy212), NULL)); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 25: /* expression ::= expression NK_PLUS expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 26: /* expression ::= expression NK_MINUS expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 27: /* expression ::= expression NK_STAR expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 28: /* expression ::= expression NK_SLASH expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 29: /* expression ::= expression NK_REM expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 32: /* column_reference ::= column_name */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy241, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy241)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy79, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy79)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 33: /* column_reference ::= table_name NK_DOT column_name */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241, createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241)); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy79, createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy79)); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 34: /* predicate ::= expression compare_op expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, yymsp[-1].minor.yy228, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, yymsp[-1].minor.yy40, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 35: /* predicate ::= expression BETWEEN expression AND expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy168), releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[-4].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy212), releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[-4].minor.yy212 = yylhsminor.yy212; break; case 36: /* predicate ::= expression NOT BETWEEN expression AND expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[-5].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[-5].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[-5].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[-5].minor.yy212 = yylhsminor.yy212; break; case 37: /* predicate ::= expression IS NULL */ -{ PARSER_TRACE; yylhsminor.yy168 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), true); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), true); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 38: /* predicate ::= expression IS NOT NULL */ -{ PARSER_TRACE; yylhsminor.yy168 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy168), false); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy212), false); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; case 39: /* predicate ::= expression in_op in_predicate_value */ -{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, yymsp[-1].minor.yy228, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), yymsp[0].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, yymsp[-1].minor.yy40, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 40: /* compare_op ::= NK_LT */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LOWER_THAN; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_THAN; } break; case 41: /* compare_op ::= NK_GT */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_GREATER_THAN; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_THAN; } break; case 42: /* compare_op ::= NK_LE */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LOWER_EQUAL; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_EQUAL; } break; case 43: /* compare_op ::= NK_GE */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_GREATER_EQUAL; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_EQUAL; } break; case 44: /* compare_op ::= NK_NE */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_NOT_EQUAL; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NOT_EQUAL; } break; case 45: /* compare_op ::= NK_EQ */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_EQUAL; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_EQUAL; } break; case 46: /* compare_op ::= LIKE */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LIKE; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LIKE; } break; case 47: /* compare_op ::= NOT LIKE */ -{ PARSER_TRACE; yymsp[-1].minor.yy228 = OP_TYPE_NOT_LIKE; } +{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_LIKE; } break; case 48: /* compare_op ::= MATCH */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_MATCH; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_MATCH; } break; case 49: /* compare_op ::= NMATCH */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_NMATCH; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NMATCH; } break; case 50: /* in_op ::= IN */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_IN; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_IN; } break; case 51: /* in_op ::= NOT IN */ -{ PARSER_TRACE; yymsp[-1].minor.yy228 = OP_TYPE_NOT_IN; } +{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_IN; } break; case 52: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ PARSER_TRACE; yymsp[-2].minor.yy168 = createNodeListNode(pCxt, yymsp[-1].minor.yy192); } +{ PARSER_TRACE; yymsp[-2].minor.yy212 = createNodeListNode(pCxt, yymsp[-1].minor.yy174); } break; case 54: /* boolean_value_expression ::= NOT boolean_primary */ -{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy168, NULL); } +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy212, NULL); } break; case 55: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 56: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 58: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ case 71: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ yytestcase(yyruleno==71); case 72: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==72); -{ PARSER_TRACE; yymsp[-2].minor.yy168 = yymsp[-1].minor.yy168; } +{ PARSER_TRACE; yymsp[-2].minor.yy212 = yymsp[-1].minor.yy212; } break; case 59: /* from_clause ::= FROM table_reference_list */ - case 88: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==88); - case 109: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==109); -{ PARSER_TRACE; yymsp[-1].minor.yy168 = yymsp[0].minor.yy168; } + case 89: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==89); + case 112: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==112); +{ PARSER_TRACE; yymsp[-1].minor.yy212 = yymsp[0].minor.yy212; } break; case 61: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy168, yymsp[0].minor.yy168, NULL); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy212, yymsp[0].minor.yy212, NULL); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 64: /* table_primary ::= table_name alias_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 65: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, &yymsp[-3].minor.yy241, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, &yymsp[-3].minor.yy79, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; case 66: /* table_primary ::= subquery alias_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168), &yymsp[0].minor.yy241); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212), &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 68: /* alias_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy241 = nil_token; } +{ PARSER_TRACE; yymsp[1].minor.yy79 = nil_token; } break; case 69: /* alias_opt ::= table_alias */ -{ PARSER_TRACE; yylhsminor.yy241 = yymsp[0].minor.yy241; } - yymsp[0].minor.yy241 = yylhsminor.yy241; +{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy79; } + yymsp[0].minor.yy79 = yylhsminor.yy79; break; case 70: /* alias_opt ::= AS table_alias */ -{ PARSER_TRACE; yymsp[-1].minor.yy241 = yymsp[0].minor.yy241; } +{ PARSER_TRACE; yymsp[-1].minor.yy79 = yymsp[0].minor.yy79; } break; case 73: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, yymsp[-4].minor.yy229, yymsp[-5].minor.yy168, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-5].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, yymsp[-4].minor.yy162, yymsp[-5].minor.yy212, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-5].minor.yy212 = yylhsminor.yy212; break; - case 74: /* join_type ::= INNER */ -{ PARSER_TRACE; yymsp[0].minor.yy229 = JOIN_TYPE_INNER; } + case 74: /* join_type ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy162 = JOIN_TYPE_INNER; } break; - case 75: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 75: /* join_type ::= INNER */ +{ PARSER_TRACE; yymsp[0].minor.yy162 = JOIN_TYPE_INNER; } + break; + case 76: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { PARSER_TRACE; - yymsp[-8].minor.yy168 = createSelectStmt(pCxt, yymsp[-7].minor.yy209, yymsp[-6].minor.yy192, yymsp[-5].minor.yy168); - yymsp[-8].minor.yy168 = addWhereClause(pCxt, yymsp[-8].minor.yy168, yymsp[-4].minor.yy168); - yymsp[-8].minor.yy168 = addPartitionByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-3].minor.yy192); - yymsp[-8].minor.yy168 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy168, yymsp[-2].minor.yy168); - yymsp[-8].minor.yy168 = addGroupByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-1].minor.yy192); - yymsp[-8].minor.yy168 = addHavingClause(pCxt, yymsp[-8].minor.yy168, yymsp[0].minor.yy168); + yymsp[-8].minor.yy212 = createSelectStmt(pCxt, yymsp[-7].minor.yy237, yymsp[-6].minor.yy174, yymsp[-5].minor.yy212); + yymsp[-8].minor.yy212 = addWhereClause(pCxt, yymsp[-8].minor.yy212, yymsp[-4].minor.yy212); + yymsp[-8].minor.yy212 = addPartitionByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-3].minor.yy174); + yymsp[-8].minor.yy212 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy212, yymsp[-2].minor.yy212); + yymsp[-8].minor.yy212 = addGroupByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-1].minor.yy174); + yymsp[-8].minor.yy212 = addHavingClause(pCxt, yymsp[-8].minor.yy212, yymsp[0].minor.yy212); } break; - case 76: /* set_quantifier_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy209 = false; } + case 77: /* set_quantifier_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy237 = false; } break; - case 77: /* set_quantifier_opt ::= DISTINCT */ -{ PARSER_TRACE; yymsp[0].minor.yy209 = true; } + case 78: /* set_quantifier_opt ::= DISTINCT */ +{ PARSER_TRACE; yymsp[0].minor.yy237 = true; } break; - case 78: /* set_quantifier_opt ::= ALL */ -{ PARSER_TRACE; yymsp[0].minor.yy209 = false; } + case 79: /* set_quantifier_opt ::= ALL */ +{ PARSER_TRACE; yymsp[0].minor.yy237 = false; } break; - case 79: /* select_list ::= NK_STAR */ -{ PARSER_TRACE; yymsp[0].minor.yy192 = NULL; } + case 80: /* select_list ::= NK_STAR */ +{ PARSER_TRACE; yymsp[0].minor.yy174 = NULL; } break; - case 80: /* select_list ::= select_sublist */ -{ PARSER_TRACE; yylhsminor.yy192 = yymsp[0].minor.yy192; } - yymsp[0].minor.yy192 = yylhsminor.yy192; + case 81: /* select_list ::= select_sublist */ +{ PARSER_TRACE; yylhsminor.yy174 = yymsp[0].minor.yy174; } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; - case 81: /* select_sublist ::= select_item */ - case 126: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==126); -{ PARSER_TRACE; yylhsminor.yy192 = createNodeList(pCxt, yymsp[0].minor.yy168); } - yymsp[0].minor.yy192 = yylhsminor.yy192; + case 82: /* select_sublist ::= select_item */ + case 129: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==129); +{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, yymsp[0].minor.yy212); } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; - case 82: /* select_sublist ::= select_sublist NK_COMMA select_item */ - case 127: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==127); -{ PARSER_TRACE; yylhsminor.yy192 = addNodeToList(pCxt, yymsp[-2].minor.yy192, yymsp[0].minor.yy168); } - yymsp[-2].minor.yy192 = yylhsminor.yy192; + case 83: /* select_sublist ::= select_sublist NK_COMMA select_item */ + case 130: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==130); +{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy174 = yylhsminor.yy174; break; - case 83: /* select_item ::= expression */ + case 84: /* select_item ::= expression */ { PARSER_TRACE; - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy168), &t); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212), &t); } - yymsp[0].minor.yy168 = yylhsminor.yy168; + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 84: /* select_item ::= expression column_alias */ -{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168), &yymsp[0].minor.yy241); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; + case 85: /* select_item ::= expression column_alias */ +{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212), &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; - case 85: /* select_item ::= expression AS column_alias */ -{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), &yymsp[0].minor.yy241); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 86: /* select_item ::= expression AS column_alias */ +{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), &yymsp[0].minor.yy79); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 86: /* select_item ::= table_name NK_DOT NK_STAR */ -{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 87: /* select_item ::= table_name NK_DOT NK_STAR */ +{ PARSER_TRACE; yylhsminor.yy212 = createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 87: /* where_clause_opt ::= */ - case 91: /* twindow_clause_opt ::= */ yytestcase(yyruleno==91); - case 96: /* sliding_opt ::= */ yytestcase(yyruleno==96); - case 98: /* fill_opt ::= */ yytestcase(yyruleno==98); - case 108: /* having_clause_opt ::= */ yytestcase(yyruleno==108); - case 116: /* slimit_clause_opt ::= */ yytestcase(yyruleno==116); - case 120: /* limit_clause_opt ::= */ yytestcase(yyruleno==120); -{ PARSER_TRACE; yymsp[1].minor.yy168 = NULL; } + case 88: /* where_clause_opt ::= */ + case 92: /* twindow_clause_opt ::= */ yytestcase(yyruleno==92); + case 97: /* sliding_opt ::= */ yytestcase(yyruleno==97); + case 99: /* fill_opt ::= */ yytestcase(yyruleno==99); + case 111: /* having_clause_opt ::= */ yytestcase(yyruleno==111); + case 119: /* slimit_clause_opt ::= */ yytestcase(yyruleno==119); + case 123: /* limit_clause_opt ::= */ yytestcase(yyruleno==123); +{ PARSER_TRACE; yymsp[1].minor.yy212 = NULL; } break; - case 89: /* partition_by_clause_opt ::= */ - case 106: /* group_by_clause_opt ::= */ yytestcase(yyruleno==106); - case 114: /* order_by_clause_opt ::= */ yytestcase(yyruleno==114); -{ PARSER_TRACE; yymsp[1].minor.yy192 = NULL; } + case 90: /* partition_by_clause_opt ::= */ + case 107: /* group_by_clause_opt ::= */ yytestcase(yyruleno==107); + case 117: /* order_by_clause_opt ::= */ yytestcase(yyruleno==117); +{ PARSER_TRACE; yymsp[1].minor.yy174 = NULL; } break; - case 90: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 107: /* group_by_clause_opt ::= GROUP BY expression_list */ yytestcase(yyruleno==107); - case 115: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==115); -{ PARSER_TRACE; yymsp[-2].minor.yy192 = yymsp[0].minor.yy192; } + case 91: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 108: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==108); + case 118: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==118); +{ PARSER_TRACE; yymsp[-2].minor.yy174 = yymsp[0].minor.yy174; } break; - case 92: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy168 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy168), &yymsp[-1].minor.yy0); } + case 93: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy212), &yymsp[-1].minor.yy0); } break; - case 93: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168)); } + case 94: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212)); } break; - case 94: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ PARSER_TRACE; yymsp[-5].minor.yy168 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy168, NULL, yymsp[-1].minor.yy168, yymsp[0].minor.yy168); } + case 95: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy212, NULL, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } break; - case 95: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ PARSER_TRACE; yymsp[-7].minor.yy168 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy168, yymsp[-3].minor.yy168, yymsp[-1].minor.yy168, yymsp[0].minor.yy168); } + case 96: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-7].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy212, yymsp[-3].minor.yy212, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } break; - case 97: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ PARSER_TRACE; yymsp[-3].minor.yy168 = yymsp[-1].minor.yy168; } + case 98: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = yymsp[-1].minor.yy212; } break; - case 99: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createFillNode(pCxt, yymsp[-1].minor.yy14, NULL); } + case 100: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createFillNode(pCxt, yymsp[-1].minor.yy44, NULL); } break; - case 100: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy168 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy192)); } + case 101: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy174)); } break; - case 101: /* fill_mode ::= NONE */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NONE; } + case 102: /* fill_mode ::= NONE */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NONE; } break; - case 102: /* fill_mode ::= PREV */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_PREV; } + case 103: /* fill_mode ::= PREV */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_PREV; } break; - case 103: /* fill_mode ::= NULL */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NULL; } + case 104: /* fill_mode ::= NULL */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NULL; } break; - case 104: /* fill_mode ::= LINEAR */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_LINEAR; } + case 105: /* fill_mode ::= LINEAR */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_LINEAR; } break; - case 105: /* fill_mode ::= NEXT */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NEXT; } + case 106: /* fill_mode ::= NEXT */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NEXT; } break; - case 110: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 109: /* group_by_list ::= expression */ +{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } + yymsp[0].minor.yy174 = yylhsminor.yy174; + break; + case 110: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } + yymsp[-2].minor.yy174 = yylhsminor.yy174; + break; + case 113: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { PARSER_TRACE; - yylhsminor.yy168 = addOrderByClause(pCxt, yymsp[-3].minor.yy168, yymsp[-2].minor.yy192); - yylhsminor.yy168 = addSlimitClause(pCxt, yylhsminor.yy168, yymsp[-1].minor.yy168); - yylhsminor.yy168 = addLimitClause(pCxt, yylhsminor.yy168, yymsp[0].minor.yy168); + yylhsminor.yy212 = addOrderByClause(pCxt, yymsp[-3].minor.yy212, yymsp[-2].minor.yy174); + yylhsminor.yy212 = addSlimitClause(pCxt, yylhsminor.yy212, yymsp[-1].minor.yy212); + yylhsminor.yy212 = addLimitClause(pCxt, yylhsminor.yy212, yymsp[0].minor.yy212); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 112: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ PARSER_TRACE; yylhsminor.yy168 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; + case 115: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ PARSER_TRACE; yylhsminor.yy212 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 117: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 121: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==121); -{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 120: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 124: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==124); +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 118: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 122: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==122); -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 121: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 125: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==125); +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 119: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 123: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==123); -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 122: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 126: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==126); +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 124: /* subquery ::= NK_LP query_expression NK_RP */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 127: /* subquery ::= NK_LP query_expression NK_RP */ +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 128: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), yymsp[-1].minor.yy10, yymsp[0].minor.yy177); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 131: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), yymsp[-1].minor.yy188, yymsp[0].minor.yy107); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 129: /* ordering_specification_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy10 = ORDER_ASC; } + case 132: /* ordering_specification_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy188 = ORDER_ASC; } break; - case 130: /* ordering_specification_opt ::= ASC */ -{ PARSER_TRACE; yymsp[0].minor.yy10 = ORDER_ASC; } + case 133: /* ordering_specification_opt ::= ASC */ +{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_ASC; } break; - case 131: /* ordering_specification_opt ::= DESC */ -{ PARSER_TRACE; yymsp[0].minor.yy10 = ORDER_DESC; } + case 134: /* ordering_specification_opt ::= DESC */ +{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_DESC; } break; - case 132: /* null_ordering_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy177 = NULL_ORDER_DEFAULT; } + case 135: /* null_ordering_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy107 = NULL_ORDER_DEFAULT; } break; - case 133: /* null_ordering_opt ::= NULLS FIRST */ -{ PARSER_TRACE; yymsp[-1].minor.yy177 = NULL_ORDER_FIRST; } + case 136: /* null_ordering_opt ::= NULLS FIRST */ +{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_FIRST; } break; - case 134: /* null_ordering_opt ::= NULLS LAST */ -{ PARSER_TRACE; yymsp[-1].minor.yy177 = NULL_ORDER_LAST; } + case 137: /* null_ordering_opt ::= NULLS LAST */ +{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index dede151004..84486079c3 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -116,10 +116,6 @@ static uint32_t toNewTokenId(uint32_t tokenId) { return NEW_TK_FROM; case TK_JOIN: return NEW_TK_JOIN; - // case TK_ON: - // return NEW_TK_ON; - // case TK_INNER: - // return NEW_TK_INNER; // case TK_PARTITION: // return NEW_TK_PARTITION; case TK_SESSION: @@ -163,6 +159,8 @@ static uint32_t toNewTokenId(uint32_t tokenId) { case TK_OFFSET: return NEW_TK_OFFSET; case TK_SPACE: + case NEW_TK_ON: + case NEW_TK_INNER: break; default: printf("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!tokenId = %d\n", tokenId); @@ -248,6 +246,10 @@ typedef enum ESqlClause { } ESqlClause; static bool afterGroupBy(ESqlClause clause) { + return clause > SQL_CLAUSE_GROUP_BY; +} + +static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } @@ -259,7 +261,7 @@ typedef struct STranslateContext { SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode* int32_t currLevel; ESqlClause currClause; - void* pExt; + SSelectStmt* pCurrStmt; } STranslateContext; static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); @@ -284,6 +286,12 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "There mustn't be aggregation"; case TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT: return "ORDER BY item must be the number of a SELECT-list expression"; + case TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION: + return "Not a GROUP BY expression"; + case TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION: + return "Not SELECTed expression"; + case TSDB_CODE_PAR_NOT_SINGLE_GROUP: + return "Not a single-group group function"; default: return "Unknown error"; } @@ -329,7 +337,7 @@ static bool belongTable(const char* currentDb, const SColumnNode* pCol, const ST if ('\0' != pCol->dbName[0]) { cmp = strcmp(pCol->dbName, pTable->dbName); } else { - cmp = strcmp(currentDb, pTable->dbName); + cmp = (QUERY_NODE_REAL_TABLE == nodeType(pTable) ? strcmp(currentDb, pTable->dbName) : 0); } if (0 == cmp) { cmp = strcmp(pCol->tableAlias, pTable->tableAlias); @@ -415,23 +423,29 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { return found; } -static bool translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* pCol) { +static EDealRes translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* pCol) { SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); size_t nums = taosArrayGetSize(pTables); + bool foundTable = false; for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); if (belongTable(pCxt->pParseCxt->db, pCol, pTable)) { + foundTable = true; if (findAndSetColumn(pCol, pTable)) { break; } generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); - return false; + return DEAL_RES_ERROR; } } - return true; + if (!foundTable) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_TABLE_NOT_EXIST, pCol->tableAlias); + return DEAL_RES_ERROR; + } + return DEAL_RES_CONTINUE; } -static bool translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* pCol) { +static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* pCol) { SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); size_t nums = taosArrayGetSize(pTables); bool found = false; @@ -440,20 +454,20 @@ static bool translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* p if (findAndSetColumn(pCol, pTable)) { if (found) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName); - return false; + return DEAL_RES_ERROR; } found = true; } } if (!found) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); - return false; + return DEAL_RES_ERROR; } - return true; + return DEAL_RES_CONTINUE; } static bool translateColumnUseAlias(STranslateContext* pCxt, SColumnNode* pCol) { - SNodeList* pProjectionList = pCxt->pExt; + SNodeList* pProjectionList = pCxt->pCurrStmt->pProjectionList; SNode* pNode; FOREACH(pNode, pProjectionList) { SExprNode* pExpr = (SExprNode*)pNode; @@ -465,10 +479,10 @@ static bool translateColumnUseAlias(STranslateContext* pCxt, SColumnNode* pCol) return false; } -static bool translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { +static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { // count(*)/first(*)/last(*) if (0 == strcmp(pCol->colName, "*")) { - return true; + return DEAL_RES_CONTINUE; } if ('\0' != pCol->tableAlias[0]) { return translateColumnWithPrefix(pCxt, pCol); @@ -477,7 +491,7 @@ static bool translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { if (SQL_CLAUSE_ORDER_BY == pCxt->currClause) { found = translateColumnUseAlias(pCxt, pCol); } - return found ? true : translateColumnWithoutPrefix(pCxt, pCol); + return found ? DEAL_RES_CONTINUE : translateColumnWithoutPrefix(pCxt, pCol); } static int32_t trimStringCopy(const char* src, int32_t len, char* dst) { @@ -500,12 +514,12 @@ static int32_t trimStringCopy(const char* src, int32_t len, char* dst) { return j; } -static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) { +static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { if (pVal->isDuration) { char unit = 0; if (parseAbsoluteDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); - return false; + return DEAL_RES_ERROR; } } else { switch (pVal->node.resType.type) { @@ -519,7 +533,7 @@ static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; - pVal->datum.i = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); break; } case TSDB_DATA_TYPE_UTINYINT: @@ -549,10 +563,10 @@ static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) { int32_t n = strlen(pVal->literal); char* tmp = calloc(1, n); int32_t len = trimStringCopy(pVal->literal, n, tmp); - if (taosParseTime(tmp, &pVal->datum.u, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { + if (taosParseTime(tmp, &pVal->datum.i, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { tfree(tmp); generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); - return false; + return DEAL_RES_ERROR; } tfree(tmp); break; @@ -565,55 +579,56 @@ static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) { break; } } - return true; + return DEAL_RES_CONTINUE; } -static bool translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { +static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; if (nodesIsArithmeticOp(pOp)) { if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - return false; + return DEAL_RES_ERROR; } pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; - return true; } else if (nodesIsComparisonOp(pOp)) { if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - return false; + return DEAL_RES_ERROR; } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - return true; } else { // todo json operator - return true; } - return true; + return DEAL_RES_CONTINUE; } -static bool translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { +static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pCxt->fmgt, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); - return false; + return DEAL_RES_ERROR; } int32_t code = fmGetFuncResultType(pFunc); if (TSDB_CODE_SUCCESS != code) { generateSyntaxErrMsg(pCxt, code, pFunc->functionName); - return false; + return DEAL_RES_ERROR; } - if (fmIsAggFunc(pFunc->funcId) && afterGroupBy(pCxt->currClause)) { + if (fmIsAggFunc(pFunc->funcId) && beforeHaving(pCxt->currClause)) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION); - return false; + return DEAL_RES_ERROR; } - return true; + return DEAL_RES_CONTINUE; } -static bool doTranslateExpr(SNode* pNode, void* pContext) { +static EDealRes translateExprSubquery(STranslateContext* pCxt, SNode* pNode) { + return (TSDB_CODE_SUCCESS == translateSubquery(pCxt, pNode) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); +} + +static EDealRes doTranslateExpr(SNode* pNode, void* pContext) { STranslateContext* pCxt = (STranslateContext*)pContext; switch (nodeType(pNode)) { case QUERY_NODE_COLUMN: @@ -625,11 +640,11 @@ static bool doTranslateExpr(SNode* pNode, void* pContext) { case QUERY_NODE_FUNCTION: return translateFunction(pCxt, (SFunctionNode*)pNode); case QUERY_NODE_TEMP_TABLE: - return translateSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); + return translateExprSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); default: break; } - return true; + return DEAL_RES_CONTINUE; } static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) { @@ -642,6 +657,103 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { return pCxt->errCode; } +static bool isAliasColumn(SColumnNode* pCol) { + return ('\0' == pCol->tableAlias[0]); +} + +static bool isDistinctOrderBy(STranslateContext* pCxt) { + return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); +} + +static SNodeList* getGroupByList(STranslateContext* pCxt) { + if (isDistinctOrderBy(pCxt)) { + return pCxt->pCurrStmt->pProjectionList; + } + return pCxt->pCurrStmt->pGroupByList; +} + +static SNode* getGroupByNode(SNode* pNode) { + if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) { + return nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0); + } + return pNode; +} + +static int32_t getGroupByErrorCode(STranslateContext* pCxt) { + if (isDistinctOrderBy(pCxt)) { + return TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION; + } + return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; +} + +static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { + STranslateContext* pCxt = (STranslateContext*)pContext; + if (!nodesIsExprNode(pNode) || (QUERY_NODE_COLUMN == nodeType(pNode) && isAliasColumn((SColumnNode*)pNode))) { + return DEAL_RES_CONTINUE; + } + if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && !isDistinctOrderBy(pCxt)) { + return DEAL_RES_IGNORE_CHILD; + } + SNode* pGroupNode; + FOREACH(pGroupNode, getGroupByList(pCxt)) { + if (nodesEqualNode(getGroupByNode(pGroupNode), pNode)) { + return DEAL_RES_IGNORE_CHILD; + } + } + if (QUERY_NODE_COLUMN == nodeType(pNode) || + (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && isDistinctOrderBy(pCxt))) { + generateSyntaxErrMsg(pCxt, getGroupByErrorCode(pCxt)); + return DEAL_RES_ERROR; + } + return DEAL_RES_CONTINUE; +} + +static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { + nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt); + return pCxt->errCode; +} + +static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList) { + if (NULL == getGroupByList(pCxt)) { + return TSDB_CODE_SUCCESS; + } + nodesWalkList(pList, doCheckExprForGroupBy, pCxt); + return pCxt->errCode; +} + +typedef struct CheckAggColCoexistCxt { + STranslateContext* pTranslateCxt; + bool existAggFunc; + bool existCol; +} CheckAggColCoexistCxt; + +static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { + CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; + if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + pCxt->existAggFunc = true; + return DEAL_RES_IGNORE_CHILD; + } + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + pCxt->existCol = true; + } + return DEAL_RES_CONTINUE; +} + +static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL != pSelect->pGroupByList) { + return TSDB_CODE_SUCCESS; + } + CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false }; + nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + if (!pSelect->isDistinct) { + nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + } + if (cxt.existAggFunc && cxt.existCol) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_NOT_SINGLE_GROUP); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pTable)) { @@ -732,12 +844,13 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec *pOther = false; SNode* pNode; FOREACH(pNode, pOrderByList) { - if (QUERY_NODE_VALUE == nodeType(pNode)) { - SValueNode* pVal = (SValueNode*)pNode; - if (translateValue(pCxt, pVal)) { + SNode* pExpr = ((SOrderByExprNode*)pNode)->pExpr; + if (QUERY_NODE_VALUE == nodeType(pExpr)) { + SValueNode* pVal = (SValueNode*)pExpr; + if (!translateValue(pCxt, pVal)) { return false; } - int32_t pos = getPositionValue((SValueNode*)pNode); + int32_t pos = getPositionValue((SValueNode*)pExpr); if (pos < 0) { ERASE_NODE(pOrderByList); nodesDestroyNode(pNode); @@ -747,9 +860,9 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec return false; } else { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - setColumnInfoByExpr(NULL, (SExprNode*)nodesListGetNode(pProjectionList, pos), pCol); - REPLACE_NODE(pCol); - nodesDestroyNode(pNode); + setColumnInfoByExpr(NULL, (SExprNode*)nodesListGetNode(pProjectionList, pos - 1), pCol); + ((SOrderByExprNode*)pNode)->pExpr = (SNode*)pCol; + nodesDestroyNode(pExpr); } } else { *pOther = true; @@ -758,17 +871,20 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec return true; } -static int32_t translateOrderBy(STranslateContext* pCxt, SNodeList* pProjectionList, SNodeList* pOrderByList) { +static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { bool other; - if (!translateOrderByPosition(pCxt, pProjectionList, pOrderByList, &other)) { + if (!translateOrderByPosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other)) { return pCxt->errCode; } if (!other) { return TSDB_CODE_SUCCESS; } pCxt->currClause = SQL_CLAUSE_ORDER_BY; - pCxt->pExt = pProjectionList; - return translateExprList(pCxt, pOrderByList); + int32_t code = translateExprList(pCxt, pSelect->pOrderByList); + if (TSDB_CODE_SUCCESS == code) { + code = checkExprListForGroupBy(pCxt, pSelect->pOrderByList); + } + return code; } static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -778,12 +894,22 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect pCxt->currClause = SQL_CLAUSE_SELECT; code = translateExprList(pCxt, pSelect->pProjectionList); } + if (TSDB_CODE_SUCCESS == code) { + code = checkExprListForGroupBy(pCxt, pSelect->pProjectionList); + } return code; } -static int32_t translateHaving(STranslateContext* pCxt, SNode* pHaving) { +static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL == pSelect->pGroupByList && NULL != pSelect->pHaving) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); + } pCxt->currClause = SQL_CLAUSE_HAVING; - return translateExpr(pCxt, pHaving); + int32_t code = translateExpr(pCxt, pSelect->pHaving); + if (TSDB_CODE_SUCCESS == code) { + code = checkExprForGroupBy(pCxt, pSelect->pHaving); + } + return code; } static int32_t translateGroupBy(STranslateContext* pCxt, SNodeList* pGroupByList) { @@ -818,8 +944,8 @@ static int32_t translateFrom(STranslateContext* pCxt, SNode* pTable) { // } SSelectStmt; static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { - int32_t code = TSDB_CODE_SUCCESS; - code = translateFrom(pCxt, pSelect->pFromTable); + pCxt->pCurrStmt = pSelect; + int32_t code = translateFrom(pCxt, pSelect->pFromTable); if (TSDB_CODE_SUCCESS == code) { code = translateWhere(pCxt, pSelect->pWhere); } @@ -833,15 +959,17 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { code = translateGroupBy(pCxt, pSelect->pGroupByList); } if (TSDB_CODE_SUCCESS == code) { - code = translateHaving(pCxt, pSelect->pHaving); + code = translateHaving(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = translateSelectList(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { - code = translateOrderBy(pCxt, pSelect->pProjectionList, pSelect->pOrderByList); + code = translateOrderBy(pCxt, pSelect); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkAggColCoexist(pCxt, pSelect); } - // printf("%s:%d code = %d\n", __FUNCTION__, __LINE__, code); return code; } @@ -860,11 +988,11 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { ++(pCxt->currLevel); ESqlClause currClause = pCxt->currClause; - void* pExt = pCxt->pExt; + SSelectStmt* pCurrStmt = pCxt->pCurrStmt; int32_t code = translateQuery(pCxt, pNode); --(pCxt->currLevel); pCxt->currClause = currClause; - pCxt->pExt = pExt; + pCxt->pCurrStmt = pCurrStmt; return code; } diff --git a/source/libs/parser/src/ttokenizer.c b/source/libs/parser/src/ttokenizer.c index 8e5b22aaa3..d5d6b11833 100644 --- a/source/libs/parser/src/ttokenizer.c +++ b/source/libs/parser/src/ttokenizer.c @@ -227,7 +227,9 @@ static SKeyword keywordTable[] = { {"OUTPUTTYPE", TK_OUTPUTTYPE}, {"AGGREGATE", TK_AGGREGATE}, {"BUFSIZE", TK_BUFSIZE}, - {"PORT", TK_PORT}, + {"PORT", TK_PORT}, + {"INNER", NEW_TK_INNER}, + {"ON", NEW_TK_ON}, }; static const char isIdChar[] = { diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index be239bccfb..2616c6c251 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -53,8 +53,8 @@ protected: code = doTranslate(&cxt_, &query_); // cout << "doTranslate return " << code << endl; if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - return (TSDB_CODE_SUCCESS != translateCode); + cout << "sql:[" << cxt_.pSql << "] code:" << code << ", " << translateCode << ", msg:" << errMagBuf_ << endl; + return (code == translateCode); } if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) { cout << "input sql : [" << cxt_.pSql << "]" << endl; @@ -71,6 +71,24 @@ protected: private: static const int max_err_len = 1024; + void exprNodeToStr(const SNode* node, string& str, bool isProject) { + switch (nodeType(node)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_OPERATOR: + case QUERY_NODE_FUNCTION: { + SExprNode* pExpr = (SExprNode*)node; + str.append(" [" + dataTypeToStr(pExpr->resType) + "]"); + if (isProject) { + str.append(" AS " + string(pExpr->aliasName)); + } + break; + } + default: + break; + } + } + string dataTypeToStr(const SDataType& dt) { switch (dt.type) { case TSDB_DATA_TYPE_NULL: @@ -119,7 +137,7 @@ private: return "Unknown Data Type " + to_string(dt.type); } - void valueNodeToStr(const SValueNode* pVal, string& str, bool isProject) { + void valueNodeToStr(const SValueNode* pVal, string& str) { switch (pVal->node.resType.type) { case TSDB_DATA_TYPE_NULL: str.append("null"); @@ -160,10 +178,41 @@ private: default: break; } - str.append(" [" + dataTypeToStr(pVal->node.resType) + "]"); - if (isProject) { - str.append(" AS " + string(pVal->node.aliasName)); + } + + void columnNodeToStr(const SColumnNode* pCol, string& str) { + if ('\0' != pCol->dbName[0]) { + str.append(pCol->dbName); + str.append("."); } + if ('\0' != pCol->tableAlias[0]) { + str.append(pCol->tableAlias); + str.append("."); + } + str.append(pCol->colName); + } + + void operatorToStr(const SOperatorNode* pOp, string& str) { + nodeToStr(pOp->pLeft, str, false); + str.append(opTypeToStr(pOp->opType)); + nodeToStr(pOp->pRight, str, false); + } + + void functionToStr(const SFunctionNode* pFunc, string& str) { + str.append(pFunc->functionName); + str.append("("); + nodeListToStr(pFunc->pParameterList, "", str, false, ", "); + str.append(")"); + } + + void groupingSetToStr(SGroupingSetNode* pGroup, string& str) { + nodeToStr(nodesListGetNode(pGroup->pParameterList, 0), str, false); + } + + void orderByExprToStr(SOrderByExprNode* pOrderBy, string& str) { + nodeToStr(pOrderBy->pExpr, str, false); + str.append((ORDER_ASC == pOrderBy->order ? " ASC" : " DESC")); + str.append((NULL_ORDER_FIRST == pOrderBy->nullOrder ? " NULLS FIRST" : " NULLS LAST")); } void nodeToStr(const SNode* node, string& str, bool isProject) { @@ -173,48 +222,41 @@ private: switch (nodeType(node)) { case QUERY_NODE_COLUMN: { - SColumnNode* pCol = (SColumnNode*)node; - if ('\0' != pCol->dbName[0]) { - str.append(pCol->dbName); - str.append("."); - } - if ('\0' != pCol->tableAlias[0]) { - str.append(pCol->tableAlias); - str.append("."); - } - str.append(pCol->colName); - str.append(" [" + dataTypeToStr(pCol->node.resType) + "]"); - if (isProject) { - str.append(" AS " + string(pCol->node.aliasName)); - } + columnNodeToStr((SColumnNode*)node, str); break; } case QUERY_NODE_VALUE: { - valueNodeToStr((SValueNode*)node, str, isProject); + valueNodeToStr((SValueNode*)node, str); break; } case QUERY_NODE_OPERATOR: { - SOperatorNode* pOp = (SOperatorNode*)node; - nodeToStr(pOp->pLeft, str, false); - str.append(opTypeToStr(pOp->opType)); - nodeToStr(pOp->pRight, str, false); - str.append(" [" + dataTypeToStr(pOp->node.resType) + "]"); - if (isProject) { - str.append(" AS " + string(pOp->node.aliasName)); - } + operatorToStr((SOperatorNode*)node, str); + break; + } + case QUERY_NODE_FUNCTION: { + functionToStr((SFunctionNode*)node, str); + break; + } + case QUERY_NODE_GROUPING_SET: { + groupingSetToStr((SGroupingSetNode*)node, str); + break; + } + case QUERY_NODE_ORDER_BY_EXPR: { + orderByExprToStr((SOrderByExprNode*)node, str); break; } default: break; } + exprNodeToStr(node, str, isProject); } - void nodeListToStr(const SNodeList* nodelist, const string& prefix, string& str, bool isProject = false) { + void nodeListToStr(const SNodeList* nodelist, const string& prefix, string& str, bool isProject = false, const string& sep = string("\n")) { SNode* node = nullptr; FOREACH(node, nodelist) { str.append(prefix); nodeToStr(node, str, isProject); - str.append("\n"); + str.append(sep); } } @@ -265,8 +307,20 @@ private: } str.append("\n"); nodeListToStr(select->pProjectionList, prefix + "\t", str, true); - str.append("\n" + prefix + "FROM\n"); + str.append(prefix + "FROM\n"); tableToStr(select->pFromTable, prefix + "\t", str); + if (nullptr != select->pWhere) { + str.append("\n" + prefix + "WHERE\n\t"); + nodeToStr(select->pWhere, str, false); + } + if (nullptr != select->pGroupByList) { + str.append("\n" + prefix + "GROUP BY\n"); + nodeListToStr(select->pGroupByList, prefix + "\t", str, true); + } + if (nullptr != select->pOrderByList) { + str.append(prefix + "ORDER BY\n"); + nodeListToStr(select->pOrderByList, prefix + "\t", str, true); + } } void selectToSql(const SNode* node, string& sql) { @@ -332,15 +386,23 @@ private: case OP_TYPE_SUB: return " - "; case OP_TYPE_MULTI: + return " * "; case OP_TYPE_DIV: + return " / "; case OP_TYPE_MOD: + return " % "; case OP_TYPE_GREATER_THAN: + return " > "; case OP_TYPE_GREATER_EQUAL: + return " >= "; case OP_TYPE_LOWER_THAN: + return " < "; case OP_TYPE_LOWER_EQUAL: + return " <= "; case OP_TYPE_EQUAL: return " = "; case OP_TYPE_NOT_EQUAL: + return " != "; case OP_TYPE_IN: case OP_TYPE_NOT_IN: case OP_TYPE_LIKE: @@ -454,11 +516,41 @@ TEST_F(NewParserTest, selectExpression) { TEST_F(NewParserTest, selectClause) { setDatabase("root", "test"); + // GROUP BY clause + bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0"); + ASSERT_TRUE(run()); + + bind("SELECT count(*), c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2"); + ASSERT_TRUE(run()); + + bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 HAVING count(c1) > 10"); + ASSERT_TRUE(run()); + + bind("SELECT count(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2, c1"); + ASSERT_TRUE(run()); + + bind("SELECT count(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); + ASSERT_TRUE(run()); + + // ORDER BY clause bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY cnt"); ASSERT_TRUE(run()); bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY 1"); ASSERT_TRUE(run()); + + // DISTINCT clause + bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY c1"); + ASSERT_TRUE(run()); + + bind("SELECT DISTINCT c1 + 10, c2 FROM t1 WHERE c1 > 0 ORDER BY c1 + 10, c2"); + ASSERT_TRUE(run()); + + bind("SELECT DISTINCT c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 ORDER BY cc1, c2"); + ASSERT_TRUE(run()); + + bind("SELECT DISTINCT count(c2) FROM t1 WHERE c1 > 0 GROUP BY c1 ORDER BY count(c2)"); + ASSERT_TRUE(run()); } TEST_F(NewParserTest, selectSyntaxError) { @@ -480,15 +572,88 @@ TEST_F(NewParserTest, selectSyntaxError) { TEST_F(NewParserTest, selectSemanticError) { setDatabase("root", "test"); - bind("SELECT * FROM t10"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); - + // TSDB_CODE_PAR_INVALID_COLUMN bind("SELECT c1, c3 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN)); + bind("SELECT t1.c1, t1.c3 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN)); + + // TSDB_CODE_PAR_TABLE_NOT_EXIST + bind("SELECT * FROM t10"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); + + bind("SELECT * FROM test.t10"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); + + bind("SELECT t2.c1 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); + + // TSDB_CODE_PAR_AMBIGUOUS_COLUMN bind("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_AMBIGUOUS_COLUMN)); + + // TSDB_CODE_PAR_WRONG_VALUE_TYPE + bind("SELECT 10n FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE)); + + bind("SELECT TIMESTAMP '2010' FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE)); + + // TSDB_CODE_PAR_INVALID_FUNTION + bind("SELECT cnt(*) FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_FUNTION)); + + // TSDB_CODE_PAR_FUNTION_PARA_NUM + // TSDB_CODE_PAR_FUNTION_PARA_TYPE + + // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION + bind("SELECT c2 FROM t1 tt1 JOIN t1 tt2 ON count(*) > 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); bind("SELECT c2 FROM t1 where count(*) > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); + + bind("SELECT c2 FROM t1 GROUP BY count(*)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); + + // TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT + bind("SELECT c2 FROM t1 ORDER BY 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT)); + + bind("SELECT c2 FROM t1 ORDER BY 2"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT)); + + // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION + bind("SELECT count(*) cnt FROM t1 HAVING c1 > 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); + + bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c1 > 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); + + bind("SELECT count(*), c1 cnt FROM t1 GROUP BY c2 HAVING c2 > 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); + + bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c2 > 0 ORDER BY c1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); + + // TSDB_CODE_PAR_NOT_SINGLE_GROUP + bind("SELECT count(*), c1 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); + + bind("SELECT count(*) FROM t1 ORDER BY c1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); + + bind("SELECT c1 FROM t1 ORDER BY count(*)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); + + // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION + bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY ts"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); + + bind("SELECT DISTINCT c1 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); + + bind("SELECT DISTINCT c2 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); } diff --git a/source/nodes/test/nodesTest.cpp b/source/libs/planner/inc/plannerImpl.h similarity index 79% rename from source/nodes/test/nodesTest.cpp rename to source/libs/planner/inc/plannerImpl.h index 7df3cd8b4c..050329693c 100644 --- a/source/nodes/test/nodesTest.cpp +++ b/source/libs/planner/inc/plannerImpl.h @@ -13,13 +13,15 @@ * along with this program. If not, see . */ -#include +#ifndef _TD_PLANNER_IMPL_H_ +#define _TD_PLANNER_IMPL_H_ -TEST(NodesTest, traverseTest) { - // todo -} +#ifdef __cplusplus +extern "C" { +#endif -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); +#ifdef __cplusplus } +#endif + +#endif /*_TD_PLANNER_IMPL_H_*/ diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c new file mode 100644 index 0000000000..6acb927db9 --- /dev/null +++ b/source/libs/planner/src/plannerImpl.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +// int32_t getPlan(SNode* pRoot, SQueryPlanNode** pQueryPlan) { + +// } \ No newline at end of file diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 3e14bfca09..6564817301 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -55,106 +55,85 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } -int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { +int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { if (NULL == input || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } - SBuildUseDBInput* bInput = (SBuildUseDBInput *)input; + SBuildUseDBInput *bInput = input; - int32_t estimateSize = sizeof(SUseDbReq); - if (NULL == *msg || msgSize < estimateSize) { - tfree(*msg); - *msg = rpcMallocCont(estimateSize); - if (NULL == *msg) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } + SUseDbReq usedbReq = {0}; + strncpy(usedbReq.db, bInput->db, sizeof(usedbReq.db)); + usedbReq.db[sizeof(usedbReq.db) - 1] = 0; + usedbReq.vgVersion = bInput->vgVersion; - SUseDbReq *bMsg = (SUseDbReq *)*msg; + int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSUseDbReq(pBuf, bufLen, &usedbReq); - strncpy(bMsg->db, bInput->db, sizeof(bMsg->db)); - bMsg->db[sizeof(bMsg->db) - 1] = 0; + *msg = pBuf; + *msgLen = bufLen; - bMsg->vgVersion = bInput->vgVersion; - - *msgLen = (int32_t)sizeof(*bMsg); - - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } - -int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { +int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { if (NULL == output || NULL == msg || msgSize <= 0) { return TSDB_CODE_TSC_INVALID_INPUT; } - SUseDbRsp *pRsp = (SUseDbRsp *)msg; SUseDbOutput *pOut = (SUseDbOutput *)output; - int32_t code = 0; + int32_t code = 0; - if (msgSize <= sizeof(*pRsp)) { - qError("invalid use db rsp msg size, msgSize:%d", msgSize); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + SUseDbRsp usedbRsp = {0}; + if (tDeserializeSUseDbRsp(msg, msgSize, &usedbRsp) != 0) { + qError("invalid use db rsp msg, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; } - - pRsp->vgVersion = ntohl(pRsp->vgVersion); - pRsp->vgNum = ntohl(pRsp->vgNum); - pRsp->uid = be64toh(pRsp->uid); - if (pRsp->vgNum < 0) { - qError("invalid db[%s] vgroup number[%d]", pRsp->db, pRsp->vgNum); + if (usedbRsp.vgNum < 0) { + qError("invalid db[%s] vgroup number[%d]", usedbRsp.db, usedbRsp.vgNum); return TSDB_CODE_TSC_INVALID_VALUE; } - int32_t expectSize = pRsp->vgNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp); - if (msgSize != expectSize) { - qError("use db rsp size mis-match, msgSize:%d, expected:%d, vgnumber:%d", msgSize, expectSize, pRsp->vgNum); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - - pOut->dbVgroup = calloc(1, sizeof(SDBVgroupInfo)); + pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { - qError("calloc %d failed", (int32_t)sizeof(SDBVgroupInfo)); + qError("calloc %d failed", (int32_t)sizeof(SDBVgInfo)); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pOut->dbId = pRsp->uid; - pOut->dbVgroup->vgVersion = pRsp->vgVersion; - pOut->dbVgroup->hashMethod = pRsp->hashMethod; - pOut->dbVgroup->vgHash = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + pOut->dbId = usedbRsp.uid; + pOut->dbVgroup->vgVersion = usedbRsp.vgVersion; + pOut->dbVgroup->hashMethod = usedbRsp.hashMethod; + pOut->dbVgroup->vgHash = + taosHashInit(usedbRsp.vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == pOut->dbVgroup->vgHash) { - qError("taosHashInit %d failed", pRsp->vgNum); + qError("taosHashInit %d failed", usedbRsp.vgNum); tfree(pOut->dbVgroup); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - for (int32_t i = 0; i < pRsp->vgNum; ++i) { - pRsp->vgroupInfo[i].vgId = ntohl(pRsp->vgroupInfo[i].vgId); - pRsp->vgroupInfo[i].hashBegin = ntohl(pRsp->vgroupInfo[i].hashBegin); - pRsp->vgroupInfo[i].hashEnd = ntohl(pRsp->vgroupInfo[i].hashEnd); + for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { + SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); - for (int32_t n = 0; n < pRsp->vgroupInfo[i].epset.numOfEps; ++n) { - pRsp->vgroupInfo[i].epset.eps[n].port = ntohs(pRsp->vgroupInfo[i].epset.eps[n].port); - } - - if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { + if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { qError("taosHashPut failed"); goto _return; } } - memcpy(pOut->db, pRsp->db, sizeof(pOut->db)); + memcpy(pOut->db, usedbRsp.db, TSDB_DB_FNAME_LEN); return code; _return: + tFreeSUsedbRsp(&usedbRsp); if (pOut) { taosHashCleanup(pOut->dbVgroup->vgHash); tfree(pOut->dbVgroup); } - + return code; } diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 846f2d5099..6f8da57ee7 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -151,7 +151,8 @@ typedef struct { char version : 4; // RPC version char comp : 4; // compression algorithm, 0:no compression 1:lz4 char resflag : 2; // reserved bits - char spi : 3; // security parameter index + char spi : 1; // security parameter index + char secured : 2; char encrypt : 3; // encrypt algorithm, 0: no encryption uint32_t code; // del later @@ -170,6 +171,11 @@ typedef struct { uint8_t auth[TSDB_AUTH_LEN]; } STransDigestMsg; +typedef struct { + uint8_t user[TSDB_UNI_LEN]; + uint8_t secret[TSDB_PASSWORD_LEN]; +} STransUserMsg; + #pragma pack(pop) #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) @@ -227,7 +233,7 @@ typedef struct { uv_async_t* asyncs; } SAsyncPool; -SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb); +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb); void transDestroyAsyncPool(SAsyncPool* pool); int transSendAsync(SAsyncPool* pool, queue* mq); @@ -236,4 +242,8 @@ int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +// int transPackMsg(SRpcMsg *rpcMsg, bool sercured, bool auth, char **msg, int32_t *msgLen); + +// int transUnpackMsg(char *msg, SRpcMsg *pMsg, bool ); + #endif diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 5fc937bccd..c3d3cfa2ab 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -30,12 +30,21 @@ void* rpcOpen(const SRpcInit* pInit) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); } pRpc->cfp = pInit->cfp; - // pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - pRpc->numOfThreads = pInit->numOfThreads; + if (pInit->connType == TAOS_CONN_SERVER) { + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + } else { + pRpc->numOfThreads = pInit->numOfThreads; + } pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); pRpc->parent = pInit->parent; + if (pInit->user) { + memcpy(pRpc->user, pInit->user, strlen(pInit->user)); + } + if (pInit->secret) { + memcpy(pRpc->secret, pInit->secret, strlen(pInit->secret)); + } return pRpc; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 29770831fa..00d9174e76 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -38,6 +38,7 @@ typedef struct SCliConn { int32_t ref; // debug and log info struct sockaddr_in addr; + struct sockaddr_in locaddr; } SCliConn; typedef struct SCliMsg { @@ -130,14 +131,16 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; - tDebug("client conn %p %s received from %s:%d", conn, TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), - ntohs(conn->addr.sin_port)); + tDebug("client conn %p %s received from %s:%d, local info: %s:%d", conn, TMSG_INFO(pHead->msgType), + inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), inet_ntoa(conn->locaddr.sin_addr), + ntohs(conn->locaddr.sin_port)); if (conn->push != NULL && conn->notifyCount != 0) { (*conn->push->callback)(conn->push->arg, &rpcMsg); + conn->push = NULL; } else { if (pCtx->pSem == NULL) { - tTrace("client conn(sync) %p handle resp", conn); + tTrace("client conn%p handle resp", conn); (pRpc->cfp)(pRpc->parent, &rpcMsg, NULL); } else { tTrace("client conn(sync) %p handle resp", conn); @@ -146,6 +149,7 @@ static void clientHandleResp(SCliConn* conn) { } } conn->notifyCount += 1; + conn->secured = pHead->secured; // buf's mem alread translated to rpcMsg.pCont transClearBuffer(&conn->readBuf); @@ -154,12 +158,12 @@ static void clientHandleResp(SCliConn* conn) { SCliThrdObj* pThrd = conn->hostThrd; // user owns conn->persist = 1 - if (conn->push != NULL) { + if (conn->push == NULL) { addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); - } - destroyCmsg(pMsg); - conn->data = NULL; + destroyCmsg(conn->data); + conn->data = NULL; + } // start thread's timer of conn pool if not active if (!uv_is_active((uv_handle_t*)pThrd->timer) && pRpc->idleTime > 0) { uv_timer_start((uv_timer_t*)pThrd->timer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); @@ -182,21 +186,23 @@ static void clientHandleExcept(SCliConn* pConn) { if (pConn->push != NULL && pConn->notifyCount != 0) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); + pConn->push = NULL; } else { if (pCtx->pSem == NULL) { (pCtx->pTransInst->cfp)(pCtx->pTransInst->parent, &rpcMsg, NULL); } else { memcpy((char*)(pCtx->pRsp), (char*)(&rpcMsg), sizeof(rpcMsg)); - // SRpcMsg rpcMsg tsem_post(pCtx->pSem); } if (pConn->push != NULL) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); } + pConn->push = NULL; + } + if (pConn->push == NULL) { + destroyCmsg(pConn->data); + pConn->data = NULL; } - - destroyCmsg(pMsg); - pConn->data = NULL; // transDestroyConnCtx(pCtx); clientConnDestroy(pConn, true); pConn->notifyCount += 1; @@ -382,17 +388,40 @@ static void clientWriteCb(uv_write_t* req, int status) { static void clientWrite(SCliConn* pConn) { SCliMsg* pCliMsg = pConn->data; - SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); - STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + STransConnCtx* pCtx = pCliMsg->ctx; + SRpcInfo* pTransInst = pCtx->pTransInst; - int msgLen = transMsgLenFromCont(pMsg->contLen); + SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); + + STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + int msgLen = transMsgLenFromCont(pMsg->contLen); + + if (!pConn->secured) { + char* buf = calloc(1, msgLen + sizeof(STransUserMsg)); + memcpy(buf, (char*)pHead, msgLen); + + STransUserMsg* uMsg = (STransUserMsg*)(buf + msgLen); + memcpy(uMsg->user, pTransInst->user, tListLen(uMsg->user)); + memcpy(uMsg->secret, pTransInst->secret, tListLen(uMsg->secret)); + + // to avoid mem leak + destroyUserdata(pMsg); + + pMsg->pCont = (char*)buf + sizeof(STransMsgHead); + pMsg->contLen = msgLen + sizeof(STransUserMsg) - sizeof(STransMsgHead); + + pHead = (STransMsgHead*)buf; + pHead->secured = 1; + msgLen += sizeof(STransUserMsg); + } pHead->msgType = pMsg->msgType; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("client conn %p %s is send to %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("client conn %p %s is send to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType), + inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static void clientConnCb(uv_connect_t* req, int status) { @@ -407,6 +436,9 @@ static void clientConnCb(uv_connect_t* req, int status) { int addrlen = sizeof(pConn->addr); uv_tcp_getpeername((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->addr, &addrlen); + addrlen = sizeof(pConn->locaddr); + uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen); + tTrace("client conn %p create", pConn); assert(pConn->stream == req->handle); @@ -428,10 +460,6 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { tTrace("client msg tran time cost: %" PRIu64 "", el); et = taosGetTimestampUs(); - // if (pMsg->msg.handle != NULL) { - // // handle - //} - STransConnCtx* pCtx = pMsg->ctx; SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { @@ -557,7 +585,7 @@ static SCliThrdObj* createThrdObj() { pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, clientAsyncCb); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, clientAsyncCb); pThrd->timer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pThrd->timer); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index d0e504a0a1..7aa5aa16f1 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -211,7 +211,8 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { /* * formate of data buffer: * |<--------------------------data from socket------------------------------->| - * |<------STransMsgHead------->|<-------------------other data--------------->| + * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user + * info--->| */ static const int CAPACITY = 1024; @@ -239,6 +240,9 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { } return 0; } +int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {} + +int transUnpackMsg(STransMsgHead* msgHead) {} int transDestroyBuffer(SConnBuffer* buf) { if (buf->cap > 0) { tfree(buf->buf); @@ -246,9 +250,7 @@ int transDestroyBuffer(SConnBuffer* buf) { transClearBuffer(buf); } -SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb) { - static int sz = 10; - +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) { SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); pool->index = 0; pool->nAsync = sz; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index b8bbea92ce..7ddeb99c9d 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -31,9 +31,11 @@ typedef struct SSrvConn { void* pTransInst; // rpc init void* ahandle; // void* hostThrd; - void* pSrvMsg; + SArray* srvMsgs; + // void* pSrvMsg; struct sockaddr_in addr; + struct sockaddr_in locaddr; // SRpcMsg sendMsg; // del later @@ -94,6 +96,7 @@ static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); +static void uvStartSendRespInternal(SSrvMsg* smsg); static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); static void uvStartSendResp(SSrvMsg* msg); @@ -231,6 +234,11 @@ static void uvHandleReq(SSrvConn* pConn) { p->chandle = NULL; STransMsgHead* pHead = (STransMsgHead*)p->msg; + if (pHead->secured == 1) { + STransUserMsg* uMsg = (p->msg + p->msgLen - sizeof(STransUserMsg)); + memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); + memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); + } pConn->inType = pHead->msgType; assert(transIsReq(pHead->msgType)); @@ -258,8 +266,9 @@ static void uvHandleReq(SSrvConn* pConn) { transClearBuffer(&pConn->readBuf); pConn->ref++; - tDebug("server conn %p %s received from %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("server conn %p %s received from %s:%d, local info: %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), + inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth @@ -305,14 +314,19 @@ void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnWriteCb(uv_write_t* req, int status) { SSrvConn* conn = req->data; - - SSrvMsg* smsg = conn->pSrvMsg; - destroySmsg(smsg); - conn->pSrvMsg = NULL; - transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); + assert(taosArrayGetSize(conn->srvMsgs) >= 1); + SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); + taosArrayRemove(conn->srvMsgs, 0); + destroySmsg(msg); + + // send second data, just use for push + if (taosArrayGetSize(conn->srvMsgs) > 0) { + msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); + uvStartSendRespInternal(msg); + } } else { tError("server conn %p failed to write data, %s", conn, uv_err_name(status)); // @@ -331,41 +345,55 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { // impl later; tTrace("server conn %p prepare to send resp", smsg->pConn); - SRpcMsg* pMsg = &smsg->msg; + SSrvConn* pConn = smsg->pConn; + SRpcMsg* pMsg = &smsg->msg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + + pHead->secured = pMsg->code == 0 ? 1 : 0; // pHead->msgType = smsg->pConn->inType + 1; + pHead->code = htonl(pMsg->code); // add more info char* msg = (char*)pHead; int32_t len = transMsgLenFromCont(pMsg->contLen); if (transCompressMsg(msg, len, NULL)) { // impl later } - tDebug("server conn %p %s is sent to %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), + inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); pHead->msgLen = htonl(len); wb->base = msg; wb->len = len; } -static void uvStartSendResp(SSrvMsg* smsg) { - // impl + +static void uvStartSendRespInternal(SSrvMsg* smsg) { uv_buf_t wb; uvPrepareSendData(smsg, &wb); SSrvConn* pConn = smsg->pConn; uv_timer_stop(pConn->pTimer); - pConn->pSrvMsg = smsg; + // pConn->pSrvMsg = smsg; // conn->pWriter->data = smsg; uv_write(pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnWriteCb); - - // SRpcMsg* rpcMsg = smsg->msg; - +} +static void uvStartSendResp(SSrvMsg* smsg) { + // impl + SSrvConn* pConn = smsg->pConn; + if (taosArrayGetSize(pConn->srvMsgs) > 0) { + tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + taosArrayPush(pConn->srvMsgs, &smsg); + return; + } + taosArrayPush(pConn->srvMsgs, &smsg); + uvStartSendRespInternal(smsg); return; } static void destroySmsg(SSrvMsg* smsg) { @@ -487,13 +515,23 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); tTrace("server conn %p created, fd: %d", pConn, fd); + int addrlen = sizeof(pConn->addr); if (0 != uv_tcp_getpeername(pConn->pTcp, (struct sockaddr*)&pConn->addr, &addrlen)) { tError("server conn %p failed to get peer info", pConn); destroyConn(pConn, true); - } else { - uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + return; } + + addrlen = sizeof(pConn->locaddr); + if (0 != uv_tcp_getsockname(pConn->pTcp, (struct sockaddr*)&pConn->locaddr, &addrlen)) { + tError("server conn %p failed to get local info", pConn); + destroyConn(pConn, true); + return; + } + + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + } else { tDebug("failed to create new connection"); destroyConn(pConn, true); @@ -522,7 +560,7 @@ static bool addHandleToWorkloop(void* arg) { QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, uvWorkerAsyncCb); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; } @@ -562,6 +600,7 @@ void* workerThread(void* arg) { static SSrvConn* createConn() { SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // tTrace("conn %p created", pConn); ++pConn->ref; return pConn; @@ -576,8 +615,15 @@ static void destroyConn(SSrvConn* conn, bool clear) { return; } transDestroyBuffer(&conn->readBuf); - destroySmsg(conn->pSrvMsg); - conn->pSrvMsg = NULL; + + for (int i = 0; i < taosArrayGetSize(conn->srvMsgs); i++) { + SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, i); + destroySmsg(msg); + } + taosArrayDestroy(conn->srvMsgs); + + // destroySmsg(conn->pSrvMsg); + // conn->pSrvMsg = NULL; if (clear) { uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index 3c9c40f46a..b4f50219ff 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -3,12 +3,13 @@ add_executable(client "") add_executable(server "") add_executable(transUT "") add_executable(syncClient "") +add_executable(pushClient "") +add_executable(pushServer "") target_sources(transUT PRIVATE "transUT.cc" ) - target_sources(transportTest PRIVATE "transportTests.cc" @@ -26,6 +27,15 @@ target_sources (syncClient "syncClient.c" ) +target_sources(pushClient + PRIVATE + "pushClient.c" +) +target_sources(pushServer + PRIVATE + "pushServer.c" +) + target_include_directories(transportTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/transport" @@ -92,4 +102,29 @@ target_link_libraries (syncClient transport ) +target_include_directories(pushClient + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_link_libraries (pushClient + os + util + common + gtest_main + transport +) + +target_include_directories(pushServer + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_link_libraries (pushServer + os + util + common + gtest_main + transport +) diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index 2756eb4666..f1aadafacc 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -49,9 +49,8 @@ static int tcount = 0; typedef struct SPushArg { tsem_t sem; - } SPushArg; - +// ping int pushCallback(void *arg, SRpcMsg *msg) { SPushArg *push = arg; tsem_post(&push->sem); @@ -59,7 +58,8 @@ int pushCallback(void *arg, SRpcMsg *msg) { SRpcPush *createPushArg() { SRpcPush *push = calloc(1, sizeof(SRpcPush)); push->arg = calloc(1, sizeof(SPushArg)); - tsem_init(&push->arg->sem, 0, 0); + + tsem_init(&(((SPushArg *)push->arg)->sem), 0, 0); push->callback = pushCallback; return push; } @@ -83,14 +83,17 @@ static void *sendRequest(void *param) { rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; rpcMsg.push = push; - ; // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); int64_t start = taosGetTimestampUs(); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - tsem_wait(&pInfo->rspSem); + tsem_wait(&pInfo->rspSem); // ping->pong // tsem_wait(&pInfo->rspSem); - tsem_wait(&push->sem); + SPushArg *arg = push->arg; + /// e + tsem_wait(&arg->sem); // push callback + + // query_fetch(client->h) int64_t end = taosGetTimestampUs() - start; if (end <= 100) { u100++; diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c new file mode 100644 index 0000000000..0bcc47383b --- /dev/null +++ b/source/libs/transport/test/pushServer.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +//#define _DEFAULT_SOURCE +#include "os.h" +#include "rpcLog.h" +#include "tglobal.h" +#include "tqueue.h" +#include "trpc.h" + +int msgSize = 128; +int commit = 0; +int dataFd = -1; +STaosQueue *qhandle = NULL; +STaosQset * qset = NULL; + +void processShellMsg() { + static int num = 0; + STaosQall *qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + void * pvnode; + + qall = taosAllocateQall(); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(qset, qall, &pvnode, NULL); + tDebug("%d shell msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + + if (dataFd >= 0) { + if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + tInfo("failed to write data file, reason:%s", strerror(errno)); + } + } + } + + if (commit >= 2) { + num += numOfMsgs; + // if (taosFsync(dataFd) < 0) { + // tInfo("failed to flush data to file, reason:%s", strerror(errno)); + //} + + if (num % 10000 == 0) { + tInfo("%d request have been written into disk", num); + } + } + + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + rpcFreeCont(pRpcMsg->pCont); + + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + + void *handle = pRpcMsg->handle; + taosFreeQitem(pRpcMsg); + + { + // sleep(1); + SRpcMsg nRpcMsg = {0}; + nRpcMsg.pCont = rpcMallocCont(msgSize); + nRpcMsg.contLen = msgSize; + nRpcMsg.handle = handle; + nRpcMsg.code = TSDB_CODE_CTG_NOT_READY; + rpcSendResponse(&nRpcMsg); + } + } + } + + taosFreeQall(qall); +} + +int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { + // app shall retrieve the auth info based on meterID from DB or a data file + // demo code here only for simple demo + int ret = 0; + + if (strcmp(meterId, "michael") == 0) { + *spi = 1; + *encrypt = 0; + strcpy(secret, "mypassword"); + strcpy(ckey, "key"); + } else if (strcmp(meterId, "jeff") == 0) { + *spi = 0; + *encrypt = 0; + } else { + ret = -1; // user not there + } + + return ret; +} + +void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + taosWriteQitem(qhandle, pTemp); +} + +int main(int argc, char *argv[]) { + SRpcInit rpcInit; + char dataName[20] = "server.data"; + + taosBlockSIGPIPE(); + + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 7000; + rpcInit.label = "SER"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processRequestMsg; + rpcInit.sessions = 1000; + rpcInit.idleTime = 2 * 1500; + rpcInit.afp = retrieveAuthInfo; + + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { + rpcInit.localPort = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { + rpcInit.numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { + msgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { + rpcInit.sessions = atoi(argv[++i]); + } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { + tsCompressMsgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) { + commit = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { + rpcDebugFlag = atoi(argv[++i]); + dDebugFlag = rpcDebugFlag; + uDebugFlag = rpcDebugFlag; + } else { + printf("\nusage: %s [options] \n", argv[0]); + printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort); + printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); + printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions); + printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); + printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); + printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit); + printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); + printf(" [-h help]: print out this help\n\n"); + exit(0); + } + } + + tsAsyncLog = 0; + rpcInit.connType = TAOS_CONN_SERVER; + taosInitLog("server.log", 100000, 10); + + void *pRpc = rpcOpen(&rpcInit); + if (pRpc == NULL) { + tError("failed to start RPC server"); + return -1; + } + // sleep(5); + + tInfo("RPC server is running, ctrl-c to exit"); + + if (commit) { + dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); + if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + } + qhandle = taosOpenQueue(); + qset = taosOpenQset(); + taosAddIntoQset(qset, qhandle, NULL); + + processShellMsg(); + + if (dataFd >= 0) { + close(dataFd); + remove(dataName); + } + + return 0; +} diff --git a/source/nodes/src/nodesTraverseFuncs.c b/source/nodes/src/nodesTraverseFuncs.c deleted file mode 100644 index 2280d0b922..0000000000 --- a/source/nodes/src/nodesTraverseFuncs.c +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "nodes.h" - -typedef enum ETraversalOrder { - TRAVERSAL_PREORDER = 1, - TRAVERSAL_POSTORDER -} ETraversalOrder; - -static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext); - -static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { - if (NULL == pNode) { - return true; - } - - if (TRAVERSAL_PREORDER == order && !walker(pNode, pContext)) { - return false; - } - - bool res = true; - switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: - case QUERY_NODE_VALUE: - case QUERY_NODE_LIMIT: - // these node types with no subnodes - break; - case QUERY_NODE_OPERATOR: { - SOperatorNode* pOpNode = (SOperatorNode*)pNode; - res = walkNode(pOpNode->pLeft, order, walker, pContext); - if (res) { - res = walkNode(pOpNode->pRight, order, walker, pContext); - } - break; - } - case QUERY_NODE_LOGIC_CONDITION: - res = walkList(((SLogicConditionNode*)pNode)->pParameterList, order, walker, pContext); - break; - case QUERY_NODE_IS_NULL_CONDITION: - res = walkNode(((SIsNullCondNode*)pNode)->pExpr, order, walker, pContext); - break; - case QUERY_NODE_FUNCTION: - res = walkList(((SFunctionNode*)pNode)->pParameterList, order, walker, pContext); - break; - case QUERY_NODE_REAL_TABLE: - case QUERY_NODE_TEMP_TABLE: - break; // todo - case QUERY_NODE_JOIN_TABLE: { - SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; - res = walkNode(pJoinTableNode->pLeft, order, walker, pContext); - if (res) { - res = walkNode(pJoinTableNode->pRight, order, walker, pContext); - } - if (res) { - res = walkNode(pJoinTableNode->pOnCond, order, walker, pContext); - } - break; - } - case QUERY_NODE_GROUPING_SET: - res = walkList(((SGroupingSetNode*)pNode)->pParameterList, order, walker, pContext); - break; - case QUERY_NODE_ORDER_BY_EXPR: - res = walkNode(((SOrderByExprNode*)pNode)->pExpr, order, walker, pContext); - break; - case QUERY_NODE_STATE_WINDOW: - res = walkNode(((SStateWindowNode*)pNode)->pCol, order, walker, pContext); - break; - case QUERY_NODE_SESSION_WINDOW: - res = walkNode(((SSessionWindowNode*)pNode)->pCol, order, walker, pContext); - break; - case QUERY_NODE_INTERVAL_WINDOW: { - SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; - res = walkNode(pInterval->pInterval, order, walker, pContext); - if (res) { - res = walkNode(pInterval->pOffset, order, walker, pContext); - } - if (res) { - res = walkNode(pInterval->pSliding, order, walker, pContext); - } - if (res) { - res = walkNode(pInterval->pFill, order, walker, pContext); - } - break; - } - case QUERY_NODE_NODE_LIST: - res = walkList(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext); - break; - case QUERY_NODE_FILL: - res = walkNode(((SFillNode*)pNode)->pValues, order, walker, pContext); - break; - case QUERY_NODE_RAW_EXPR: - res = walkNode(((SRawExprNode*)pNode)->pNode, order, walker, pContext); - break; - default: - break; - } - - if (res && TRAVERSAL_POSTORDER == order) { - res = walker(pNode, pContext); - } - - return res; -} - -static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { - SNode* node; - FOREACH(node, pNodeList) { - if (!walkNode(node, order, walker, pContext)) { - return false; - } - } - return true; -} - -void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) { - (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); -} - -void nodesWalkList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext) { - (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); -} - -void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext) { - (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); -} - -void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext) { - (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext); -} - -bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext) { - -} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4cb1e392b9..70608e117e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -77,7 +77,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PTR, "Invalid pointer") TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization")