diff --git a/.gitmodules b/.gitmodules index a2266c46af..656478a47e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "src/connector/go"] path = src/connector/go url = git@github.com:taosdata/driver-go.git -[submodule "src/connector/grafanaplugin"] - path = src/connector/grafanaplugin - url = git@github.com:taosdata/grafanaplugin.git [submodule "src/connector/hivemq-tdengine-extension"] path = src/connector/hivemq-tdengine-extension url = git@github.com:taosdata/hivemq-tdengine-extension.git diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 48fb958ef0..4c126f78a5 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -111,15 +111,15 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_MAX, } EShowType; -#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 -#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 -#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3 +#define TSDB_ALTER_TABLE_ADD_TAG 1 +#define TSDB_ALTER_TABLE_DROP_TAG 2 +#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 #define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 #define TSDB_ALTER_TABLE_ADD_COLUMN 5 #define TSDB_ALTER_TABLE_DROP_COLUMN 6 -#define TSDB_ALTER_TABLE_CHANGE_COLUMN 7 -#define TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN 8 +#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7 +#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 #define TSDB_FILL_NONE 0 #define TSDB_FILL_NULL 1 @@ -147,10 +147,15 @@ typedef enum _mgmt_table { #define TSDB_COL_IS_NORMAL_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) #define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) + +#define TD_SUPER_TABLE TSDB_SUPER_TABLE +#define TD_CHILD_TABLE TSDB_CHILD_TABLE +#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE + typedef struct { int32_t vgId; - char* dbName; - char* tableFullName; + char* dbFName; + char* tbName; } SBuildTableMetaInput; typedef struct { @@ -158,6 +163,12 @@ typedef struct { int32_t vgVersion; } SBuildUseDBInput; +typedef struct SField { + char name[TSDB_COL_NAME_LEN]; + uint8_t type; + int32_t bytes; +} SField; + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -247,21 +258,32 @@ typedef struct SSchema { typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; - int32_t numOfTags; int32_t numOfColumns; - SSchema pSchema[]; + int32_t numOfTags; + SArray* pColumns; + SArray* pTags; } SMCreateStbReq; +int32_t tSerializeSMCreateStbReq(void** buf, SMCreateStbReq* pReq); +void* tDeserializeSMCreateStbReq(void* buf, SMCreateStbReq* pReq); + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; } SMDropStbReq; +int32_t tSerializeSMDropStbReq(void** buf, SMDropStbReq* pReq); +void* tDeserializeSMDropStbReq(void* buf, SMDropStbReq* pReq); + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t alterType; - SSchema schema; -} SMAlterStbReq; + int32_t numOfFields; + SArray* pFields; +} SMAltertbReq; + +int32_t tSerializeSMAlterStbReq(void** buf, SMAltertbReq* pReq); +void* tDeserializeSMAlterStbReq(void* buf, SMAltertbReq* pReq); typedef struct { int32_t pid; @@ -691,8 +713,8 @@ typedef struct { typedef struct { SMsgHead header; - char dbFname[TSDB_DB_FNAME_LEN]; - char tableFname[TSDB_TABLE_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; } STableInfoReq; typedef struct { @@ -717,9 +739,10 @@ typedef struct { } SVgroupsInfo; typedef struct { - char tbFname[TSDB_TABLE_FNAME_LEN]; // table full name - char stbFname[TSDB_TABLE_FNAME_LEN]; - char dbFname[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t dbId; int32_t numOfTags; int32_t numOfColumns; int8_t precision; @@ -1158,10 +1181,7 @@ typedef struct SVCreateTbReq { char* name; uint32_t ttl; uint32_t keep; -#define TD_SUPER_TABLE TSDB_SUPER_TABLE -#define TD_CHILD_TABLE TSDB_CHILD_TABLE -#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE - uint8_t type; + uint8_t type; union { struct { tb_uid_t suid; @@ -1179,42 +1199,44 @@ typedef struct SVCreateTbReq { SSchema* pSchema; } ntbCfg; }; -} SVCreateTbReq; +} SVCreateTbReq, SVUpdateTbReq; + +typedef struct { +} SVCreateTbRsp, SVUpdateTbRsp; + +int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); +void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); +int32_t tSerializeSVCreateTbRsp(void** buf, SVCreateTbRsp* pRsp); +void* tDeserializeSVCreateTbRsp(void* buf, SVCreateTbRsp* pRsp); typedef struct { uint64_t ver; // use a general definition SArray* pArray; } SVCreateTbBatchReq; -int tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); -void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); -int tSVCreateTbBatchReqSerialize(void** buf, SVCreateTbBatchReq* pReq); -void* tSVCreateTbBatchReqDeserialize(void* buf, SVCreateTbBatchReq* pReq); +typedef struct { +} SVCreateTbBatchRsp; + +int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); +void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); +int32_t tSerializeSVCreateTbBatchRsp(void** buf, SVCreateTbBatchRsp* pRsp); +void* tDeserializeSVCreateTbBatchRsp(void* buf, SVCreateTbBatchRsp* pRsp); typedef struct { - SMsgHead head; -} SVCreateTbRsp; - -typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - int8_t ignoreNotExists; -} SVAlterTbReq; - -typedef struct { - SMsgHead head; -} SVAlterTbRsp; - -typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - int64_t suid; + uint64_t ver; + char* name; + uint8_t type; + tb_uid_t suid; } SVDropTbReq; typedef struct { - SMsgHead head; } SVDropTbRsp; +int32_t tSerializeSVDropTbReq(void** buf, SVDropTbReq* pReq); +void* tDeserializeSVDropTbReq(void* buf, SVDropTbReq* pReq); +int32_t tSerializeSVDropTbRsp(void** buf, SVDropTbRsp* pRsp); +void* tDeserializeSVDropTbRsp(void* buf, SVDropTbRsp* pRsp); + typedef struct { SMsgHead head; int64_t uid; diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index cbc8c0e87c..98903f5617 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -207,36 +207,77 @@ #define TK_INTO 189 #define TK_VALUES 190 -#define NEW_TK_UNION 1 -#define NEW_TK_ALL 2 -#define NEW_TK_MINUS 3 -#define NEW_TK_EXCEPT 4 -#define NEW_TK_INTERSECT 5 -#define NEW_TK_NK_PLUS 6 -#define NEW_TK_NK_MINUS 7 -#define NEW_TK_NK_STAR 8 -#define NEW_TK_NK_SLASH 9 -#define NEW_TK_SHOW 10 -#define NEW_TK_DATABASES 11 -#define NEW_TK_NK_ID 12 -#define NEW_TK_NK_LP 13 -#define NEW_TK_NK_RP 14 -#define NEW_TK_NK_COMMA 15 -#define NEW_TK_NK_LITERAL 16 -#define NEW_TK_NK_DOT 17 -#define NEW_TK_SELECT 18 -#define NEW_TK_DISTINCT 19 -#define NEW_TK_AS 20 -#define NEW_TK_FROM 21 -#define NEW_TK_WITH 22 -#define NEW_TK_RECURSIVE 23 -#define NEW_TK_ORDER 24 -#define NEW_TK_BY 25 -#define NEW_TK_ASC 26 -#define NEW_TK_DESC 27 -#define NEW_TK_NULLS 28 -#define NEW_TK_FIRST 29 -#define NEW_TK_LAST 30 +#define NEW_TK_OR 1 +#define NEW_TK_AND 2 +#define NEW_TK_UNION 3 +#define NEW_TK_ALL 4 +#define NEW_TK_MINUS 5 +#define NEW_TK_EXCEPT 6 +#define NEW_TK_INTERSECT 7 +#define NEW_TK_NK_PLUS 8 +#define NEW_TK_NK_MINUS 9 +#define NEW_TK_NK_STAR 10 +#define NEW_TK_NK_SLASH 11 +#define NEW_TK_NK_REM 12 +#define NEW_TK_SHOW 13 +#define NEW_TK_DATABASES 14 +#define NEW_TK_NK_INTEGER 15 +#define NEW_TK_NK_FLOAT 16 +#define NEW_TK_NK_STRING 17 +#define NEW_TK_NK_BOOL 18 +#define NEW_TK_TIMESTAMP 19 +#define NEW_TK_NK_VARIABLE 20 +#define NEW_TK_NK_COMMA 21 +#define NEW_TK_NK_ID 22 +#define NEW_TK_NK_LP 23 +#define NEW_TK_NK_RP 24 +#define NEW_TK_NK_DOT 25 +#define NEW_TK_BETWEEN 26 +#define NEW_TK_NOT 27 +#define NEW_TK_IS 28 +#define NEW_TK_NULL 29 +#define NEW_TK_NK_LT 30 +#define NEW_TK_NK_GT 31 +#define NEW_TK_NK_LE 32 +#define NEW_TK_NK_GE 33 +#define NEW_TK_NK_NE 34 +#define NEW_TK_NK_EQ 35 +#define NEW_TK_LIKE 36 +#define NEW_TK_MATCH 37 +#define NEW_TK_NMATCH 38 +#define NEW_TK_IN 39 +#define NEW_TK_FROM 40 +#define NEW_TK_AS 41 +#define NEW_TK_JOIN 42 +#define NEW_TK_ON 43 +#define NEW_TK_INNER 44 +#define NEW_TK_SELECT 45 +#define NEW_TK_DISTINCT 46 +#define NEW_TK_WHERE 47 +#define NEW_TK_PARTITION 48 +#define NEW_TK_BY 49 +#define NEW_TK_SESSION 50 +#define NEW_TK_STATE_WINDOW 51 +#define NEW_TK_INTERVAL 52 +#define NEW_TK_SLIDING 53 +#define NEW_TK_FILL 54 +#define NEW_TK_VALUE 55 +#define NEW_TK_NONE 56 +#define NEW_TK_PREV 57 +#define NEW_TK_LINEAR 58 +#define NEW_TK_NEXT 59 +#define NEW_TK_GROUP 60 +#define NEW_TK_HAVING 61 +#define NEW_TK_ORDER 62 +#define NEW_TK_SLIMIT 63 +#define NEW_TK_SOFFSET 64 +#define NEW_TK_LIMIT 65 +#define NEW_TK_OFFSET 66 +#define NEW_TK_ASC 67 +#define NEW_TK_DESC 68 +#define NEW_TK_NULLS 69 +#define NEW_TK_FIRST 70 +#define NEW_TK_LAST 71 #define TK_SPACE 300 #define TK_COMMENT 301 @@ -247,6 +288,8 @@ #define TK_FILE 306 #define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query +#define TK_NIL 65535 + #endif diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 0b465b7b4e..ab1298785a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -32,6 +32,15 @@ extern "C" { struct SCatalog; +enum { + CTG_DBG_DB_NUM = 1, + CTG_DBG_META_NUM, + CTG_DBG_STB_NUM, + CTG_DBG_DB_RENT_NUM, + CTG_DBG_STB_RENT_NUM, +}; + + typedef struct SCatalogReq { SArray *pTableName; // element is SNAME SArray *pUdf; // udf name @@ -49,17 +58,19 @@ typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; uint32_t dbRentSec; - uint32_t stableRentSec; + uint32_t stbRentSec; } SCatalogCfg; typedef struct SSTableMetaVersion { + char dbFName[TSDB_DB_FNAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; uint64_t suid; int16_t sversion; int16_t tversion; } SSTableMetaVersion; typedef struct SDbVgVersion { - char dbName[TSDB_DB_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; int32_t vgVersion; } SDbVgVersion; @@ -97,9 +108,11 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, */ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList); -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); +int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgroupInfo* dbInfo); -int32_t catalogRemoveDBVgroup(struct SCatalog* pCatalog, SDbVgVersion* dbInfo); +int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId); + +int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid); /** * Get a table's meta data. @@ -123,6 +136,8 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, cons */ int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg); + /** * Force renew a table's local cached meta data. diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index d245d04166..292b4ab2c9 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -37,12 +37,6 @@ typedef struct SQueryNode { #define queryNodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type) -typedef struct SField { - char name[TSDB_COL_NAME_LEN]; - uint8_t type; - int32_t bytes; -} SField; - typedef struct SFieldInfo { int16_t numOfOutput; // number of column in result SField *final; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c7ca2d9e1d..549f36a898 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -81,16 +81,15 @@ typedef struct STableMeta { } STableMeta; typedef struct SDBVgroupInfo { - SRWLatch lock; - uint64_t dbId; int32_t vgVersion; int8_t hashMethod; - SHashObj *vgInfo; //key:vgId, value:SVgroupInfo + SHashObj *vgHash; //key:vgId, value:SVgroupInfo } SDBVgroupInfo; typedef struct SUseDbOutput { - char db[TSDB_DB_FNAME_LEN]; - SDBVgroupInfo dbVgroup; + char db[TSDB_DB_FNAME_LEN]; + uint64_t dbId; + SDBVgroupInfo *dbVgroup; } SUseDbOutput; enum { @@ -103,8 +102,10 @@ enum { typedef struct STableMetaOutput { int32_t metaType; - char ctbFname[TSDB_TABLE_FNAME_LEN]; - char tbFname[TSDB_TABLE_FNAME_LEN]; + uint64_t dbId; + char dbFName[TSDB_DB_FNAME_LEN]; + char ctbName[TSDB_TABLE_NAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; SCTableMeta ctbMeta; STableMeta *tbMeta; } STableMetaOutput; @@ -159,6 +160,8 @@ void initQueryModuleMsgHandle(); const SSchema* tGetTbnameColumnSchema(); bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); +int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta **pMeta); + extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index e297059bc8..cdc8a6f1ac 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -22,6 +22,19 @@ extern "C" { #include "tdef.h" +#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) +#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) + +#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) + +#define FOREACH(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) + +#define FORBOTH(node1, list1, node2, list2) \ + for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ + (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ + cell1 = cell1->pNext, cell2 = cell2->pNext) + typedef enum ENodeType { QUERY_NODE_COLUMN = 1, QUERY_NODE_VALUE, @@ -38,6 +51,11 @@ typedef enum ENodeType { QUERY_NODE_STATE_WINDOW, QUERY_NODE_SESSION_WINDOW, QUERY_NODE_INTERVAL_WINDOW, + QUERY_NODE_NODE_LIST, + QUERY_NODE_FILL, + + // only for parser + QUERY_NODE_RAW_EXPR, QUERY_NODE_SET_OPERATOR, QUERY_NODE_SELECT_STMT, @@ -52,9 +70,6 @@ typedef struct SNode { ENodeType type; } SNode; -#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) -#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) - typedef struct SListCell { SNode* pNode; struct SListCell* pNext; @@ -62,18 +77,16 @@ typedef struct SListCell { typedef struct SNodeList { int16_t length; - SListCell* pHeader; + SListCell* pHead; + SListCell* pTail; } SNodeList; -#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) - -#define FOREACH(node, list) \ - for (SListCell* cell = (NULL != (list) ? (list)->pHeader : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) - -#define FORBOTH(node1, list1, node2, list2) \ - for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHeader : NULL), *cell2 = (NULL != (list2) ? (list2)->pHeader : NULL); \ - (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ - cell1 = cell1->pNext, cell2 = cell2->pNext) +typedef struct SRawExprNode { + ENodeType nodeType; + char* p; + uint32_t n; + SNode* pNode; +} SRawExprNode; typedef struct SDataType { uint8_t type; @@ -86,6 +99,7 @@ typedef struct SExprNode { ENodeType nodeType; SDataType resType; char aliasName[TSDB_COL_NAME_LEN]; + SNodeList* pAssociationList; } SExprNode; typedef enum EColumnType { @@ -99,12 +113,22 @@ typedef struct SColumnNode { EColumnType colType; // column or tag char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; + char tableAlias[TSDB_TABLE_NAME_LEN]; char colName[TSDB_COL_NAME_LEN]; + SNode* pProjectRef; } SColumnNode; typedef struct SValueNode { - SExprNode type; // QUERY_NODE_VALUE + SExprNode node; // QUERY_NODE_VALUE char* literal; + bool isDuration; + union { + bool b; + int64_t i; + uint64_t u; + double d; + char* p; + } datum; } SValueNode; typedef enum EOperatorType { @@ -135,7 +159,7 @@ typedef enum EOperatorType { } EOperatorType; typedef struct SOperatorNode { - SExprNode type; // QUERY_NODE_OPERATOR + SExprNode node; // QUERY_NODE_OPERATOR EOperatorType opType; SNode* pLeft; SNode* pRight; @@ -156,25 +180,33 @@ typedef struct SLogicConditionNode { typedef struct SIsNullCondNode { ENodeType type; // QUERY_NODE_IS_NULL_CONDITION SNode* pExpr; - bool isNot; + bool isNull; } SIsNullCondNode; +typedef struct SNodeListNode { + ENodeType type; // QUERY_NODE_NODE_LIST + SNodeList* pNodeList; +} SNodeListNode; + typedef struct SFunctionNode { - SExprNode type; // QUERY_NODE_FUNCTION + SExprNode node; // QUERY_NODE_FUNCTION char functionName[TSDB_FUNC_NAME_LEN]; int32_t funcId; - SNodeList* pParameterList; // SNode + SNodeList* pParameterList; } SFunctionNode; typedef struct STableNode { ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; - char tableAliasName[TSDB_COL_NAME_LEN]; + char tableAlias[TSDB_TABLE_NAME_LEN]; } STableNode; +struct STableMeta; + typedef struct SRealTableNode { STableNode table; // QUERY_NODE_REAL_TABLE - char dbName[TSDB_DB_NAME_LEN]; + struct STableMeta* pMeta; } SRealTableNode; typedef struct STempTableNode { @@ -241,24 +273,40 @@ typedef struct SSessionWindowNode { typedef struct SIntervalWindowNode { ENodeType type; // QUERY_NODE_INTERVAL_WINDOW - int64_t interval; - int64_t sliding; - int64_t offset; + SNode* pInterval; // SValueNode + SNode* pOffset; // SValueNode + SNode* pSliding; // SValueNode + SNode* pFill; } SIntervalWindowNode; +typedef enum EFillMode { + FILL_MODE_NONE = 1, + FILL_MODE_VALUE, + FILL_MODE_PREV, + FILL_MODE_NULL, + FILL_MODE_LINEAR, + FILL_MODE_NEXT +} EFillMode; + +typedef struct SFillNode { + ENodeType type; // QUERY_NODE_FILL + EFillMode mode; + SNode* pValues; // SNodeListNode +} SFillNode; + typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; - bool isStar; SNodeList* pProjectionList; // SNode SNode* pFromTable; - SNode* pWhereCond; + SNode* pWhere; SNodeList* pPartitionByList; // SNode - SNode* pWindowClause; + SNode* pWindow; SNodeList* pGroupByList; // SGroupingSetNode + SNode* pHaving; SNodeList* pOrderByList; // SOrderByExprNode - SLimitNode limit; - SLimitNode slimit; + SNode* pLimit; + SNode* pSlimit; } SSelectStmt; typedef enum ESetOperatorType { @@ -270,12 +318,23 @@ typedef struct SSetOperator { ESetOperatorType opType; SNode* pLeft; SNode* pRight; + SNodeList* pOrderByList; // SOrderByExprNode + SNode* pLimit; } SSetOperator; +SNode* nodesMakeNode(ENodeType type); +void nodesDestroyNode(SNode* pNode); + +SNodeList* nodesMakeList(); +SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode); +void nodesDestroyList(SNodeList* pList); + typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); -bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); -bool nodesWalkNodeList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext); +void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); +void nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext); +void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext); +void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext); bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext); @@ -286,13 +345,13 @@ void nodesCloneNode(const SNode* pNode); int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); int32_t nodesStringToNode(const char* pStr, SNode** pNode); +bool nodesIsArithmeticOp(const SOperatorNode* pOp); +bool nodesIsComparisonOp(const SOperatorNode* pOp); +bool nodesIsJsonOp(const SOperatorNode* pOp); + bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); -SNode* nodesMakeNode(ENodeType type); -void nodesDestroyNode(SNode* pNode); -void nodesDestroyNodeList(SNodeList* pList); - #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 50af44a94c..88d6f77f72 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -218,18 +218,19 @@ int32_t* taosGetErrno(); // mnode-stable #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) #define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A1) -#define TSDB_CODE_MND_TOO_MANY_STBS TAOS_DEF_ERROR_CODE(0, 0x03A2) -#define TSDB_CODE_MND_INVALID_STB TAOS_DEF_ERROR_CODE(0, 0x03A3) -#define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x03A4) -#define TSDB_CODE_MND_STB_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03A5) -#define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x03A6) -#define TSDB_CODE_MND_TAG_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A7) -#define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A8) -#define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x03A9) -#define TSDB_CODE_MND_COLUMN_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AA) -#define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AB) -#define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x03AC) -#define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AD) +#define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03A2) +#define TSDB_CODE_MND_TOO_MANY_STBS TAOS_DEF_ERROR_CODE(0, 0x03A3) +#define TSDB_CODE_MND_INVALID_STB TAOS_DEF_ERROR_CODE(0, 0x03A4) +#define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x03A5) +#define TSDB_CODE_MND_INVALID_STB_ALTER_OPTION TAOS_DEF_ERROR_CODE(0, 0x03A6) +#define TSDB_CODE_MND_STB_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03A7) +#define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x03A8) +#define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x03A9) +#define TSDB_CODE_MND_TAG_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AA) +#define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AB) +#define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x03AC) +#define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AD) +#define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) // mnode-func #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) @@ -435,12 +436,21 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) //catalog is not ready #define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) //catalog memory error #define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) //catalog system error +#define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2405) //Database is dropped +#define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2406) //catalog is out of service //scheduler #define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) //scheduler status error #define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) //scheduler internal error - +//parser +#define TSDB_CODE_PAR_INVALID_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2601) //invalid column name +#define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) //table not exist +#define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) //ambiguous column +#define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604) //wrong value type +#define TSDB_CODE_PAR_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2605) //invalid number of arguments +#define TSDB_CODE_PAR_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2606) //inconsistent datatypes +#define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2607) //there mustn't be aggregation #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c977b4172c..d7574466cd 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -242,7 +242,7 @@ int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key); -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr); +void appHbMgrCleanup(void); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6c8878921b..3e1af765b0 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -41,19 +41,13 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%"PRIx64, rsp->db, rsp->vgVersion, rsp->uid); if (rsp->vgVersion < 0) { - SDbVgVersion dbInfo; - strcpy(dbInfo.dbName, rsp->db); - dbInfo.dbId = rsp->uid; - dbInfo.vgVersion = rsp->vgVersion; - - code = catalogRemoveDBVgroup(pCatalog, &dbInfo); + code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); } else { SDBVgroupInfo vgInfo = {0}; - vgInfo.dbId = rsp->uid; vgInfo.vgVersion = rsp->vgVersion; vgInfo.hashMethod = rsp->hashMethod; - vgInfo.vgInfo = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == vgInfo.vgInfo) { + vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == vgInfo.vgHash) { tscError("hash init[%d] failed", rsp->vgNum); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -67,16 +61,16 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog rsp->vgroupInfo[i].epset.eps[n].port = ntohs(rsp->vgroupInfo[i].epset.eps[n].port); } - if (0 != taosHashPut(vgInfo.vgInfo, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { + if (0 != taosHashPut(vgInfo.vgHash, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { tscError("hash push failed, errno:%d", errno); - taosHashCleanup(vgInfo.vgInfo); + taosHashCleanup(vgInfo.vgHash); return TSDB_CODE_TSC_OUT_OF_MEMORY; } } - code = catalogUpdateDBVgroup(pCatalog, rsp->db, &vgInfo); + code = catalogUpdateDBVgroup(pCatalog, rsp->db, rsp->uid, &vgInfo); if (code) { - taosHashCleanup(vgInfo.vgInfo); + taosHashCleanup(vgInfo.vgHash); } } @@ -90,6 +84,58 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog return TSDB_CODE_SUCCESS; } +static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t msgLen = 0; + int32_t code = 0; + int32_t schemaNum = 0; + + while (msgLen < valueLen) { + STableMetaRsp *rsp = (STableMetaRsp *)((char *)value + msgLen); + + rsp->numOfColumns = ntohl(rsp->numOfColumns); + rsp->suid = be64toh(rsp->suid); + + if (rsp->numOfColumns < 0) { + schemaNum = 0; + + tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); + + catalogRemoveSTableMeta(pCatalog, rsp->dbFName, rsp->stbName, rsp->suid); + } else { + tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); + + rsp->numOfTags = ntohl(rsp->numOfTags); + rsp->sversion = ntohl(rsp->sversion); + rsp->tversion = ntohl(rsp->tversion); + rsp->tuid = be64toh(rsp->tuid); + rsp->vgId = ntohl(rsp->vgId); + + SSchema* pSchema = rsp->pSchema; + + schemaNum = rsp->numOfColumns + rsp->numOfTags; + + for (int i = 0; i < schemaNum; ++i) { + pSchema->bytes = ntohl(pSchema->bytes); + pSchema->colId = ntohl(pSchema->colId); + + pSchema++; + } + + if (rsp->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchema[0].colId); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + catalogUpdateSTableMeta(pCatalog, rsp); + } + + msgLen += sizeof(STableMetaRsp) + schemaNum * sizeof(SSchema); + } + + return TSDB_CODE_SUCCESS; +} + + static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) { SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); if (NULL == info) { @@ -122,9 +168,24 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRs hbProcessDBInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_STBINFO: + case HEARTBEAT_KEY_STBINFO:{ + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb stb info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + int64_t *clusterId = (int64_t *)info->param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code)); + break; + } + + hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog); break; + } default: tscError("invalid hb key type:%d", kv->key); break; @@ -157,7 +218,7 @@ static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code tfree(param); if (rspNum) { - tscDebug("hb got %d rsp, %d empty rsp prior", rspNum, atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0)); + tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0)); } else { atomic_add_fetch_32(&emptyRspNum, 1); } @@ -204,6 +265,37 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SSTableMetaVersion *stbs = NULL; + uint32_t stbNum = 0; + int32_t code = 0; + + code = catalogGetExpiredSTables(pCatalog, &stbs, &stbNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (stbNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < stbNum; ++i) { + SSTableMetaVersion *stb = &stbs[i]; + stb->suid = htobe64(stb->suid); + stb->sversion = htons(stb->sversion); + stb->tversion = htons(stb->tversion); + } + + SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs}; + + tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen); + + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} + + int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req) { int64_t *clusterId = (int64_t *)param; struct SCatalog *pCatalog = NULL; @@ -219,6 +311,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req return code; } + code = hbGetExpiredStbInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + return TSDB_CODE_SUCCESS; } @@ -389,7 +486,6 @@ static void hbStopThread() { } SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { - return NULL; hbMgrInit(); SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { @@ -431,29 +527,22 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { return pAppHbMgr; } -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { - if (NULL == pAppHbMgr) { - return; - } - +void appHbMgrCleanup(void) { pthread_mutex_lock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); - if (pAppHbMgr == pTarget) { - taosHashCleanup(pTarget->activeInfo); - pTarget->activeInfo = NULL; - taosHashCleanup(pTarget->connInfo); - pTarget->connInfo = NULL; - } + taosHashCleanup(pTarget->activeInfo); + pTarget->activeInfo = NULL; + taosHashCleanup(pTarget->connInfo); + pTarget->connInfo = NULL; } pthread_mutex_unlock(&clientHbMgr.lock); } int hbMgrInit() { - return 0; // init once int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); if (old == 1) return 0; @@ -479,6 +568,7 @@ void hbMgrCleanUp() { if (old == 0) return; pthread_mutex_lock(&clientHbMgr.lock); + appHbMgrCleanup(); taosArrayDestroy(clientHbMgr.appHbMgrs); pthread_mutex_unlock(&clientHbMgr.lock); @@ -509,7 +599,6 @@ int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { - return 0; SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; SHbConnInfo info = {0}; @@ -539,9 +628,6 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) { return; } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); - if (atomic_load_32(&pAppHbMgr->connKeyCnt) <= 0) { - appHbMgrCleanup(pAppHbMgr); - } } int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e04a9cc81d..dfe7b12ce4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -110,7 +110,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - /*p->pAppHbMgr = appHbMgrInit(p, key);*/ + p->pAppHbMgr = appHbMgrInit(p, key); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -540,8 +540,12 @@ void* doFetchRow(SRequestObj* pRequest) { tsem_wait(&pRequest->body.rspSem); pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; - } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE && pResultInfo->pData != NULL) { - return NULL; + } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { + epSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + if (pResultInfo->completed) { + return NULL; + } } SMsgSendInfo* body = buildMsgInfoImpl(pRequest); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 6f74c29650..8ab1880069 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -76,7 +76,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->connType = HEARTBEAT_TYPE_QUERY; - /*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY);*/ + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY); // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, @@ -201,6 +201,7 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pRetrieve->numOfRows; pResInfo->pData = pRetrieve->data; + pResInfo->completed = pRetrieve->completed; pResInfo->current = 0; setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); @@ -301,15 +302,12 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { SDropDbRsp *rsp = (SDropDbRsp *)pMsg->pData; - SDbVgVersion dbVer = {0}; struct SCatalog *pCatalog = NULL; - - strncpy(dbVer.dbName, rsp->db, sizeof(dbVer.dbName)); - dbVer.dbId = be64toh(rsp->uid); + rsp->uid = be64toh(rsp->uid); catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - catalogRemoveDBVgroup(pCatalog, &dbVer); + catalogRemoveDB(pCatalog, rsp->db, rsp->uid); tsem_post(&pRequest->body.rspSem); return code; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 34b55fd812..f129508405 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -27,7 +27,7 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -int tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { +int32_t tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return -1; @@ -44,7 +44,7 @@ int tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { return 0; } -int tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { +int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { if (pIter->len == 0) { pIter->len += sizeof(SSubmitMsg); } else { @@ -63,7 +63,7 @@ int tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { return 0; } -int tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { +int32_t tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { if (pBlock->dataLen <= 0) return -1; pIter->totalLen = pBlock->dataLen; pIter->len = 0; @@ -85,14 +85,14 @@ SMemRow tGetSubmitBlkNext(SSubmitBlkIter *pIter) { } } -int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { - int tlen = 0; +int32_t tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { + int32_t tlen = 0; tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); int32_t kvNum = taosHashGetSize(pReq->info); tlen += taosEncodeFixedI32(buf, kvNum); - SKv *kv; - void* pIter = taosHashIterate(pReq->info, NULL); + SKv *kv; + void *pIter = taosHashIterate(pReq->info, NULL); while (pIter != NULL) { kv = pIter; tlen += taosEncodeSKv(buf, kv); @@ -111,7 +111,7 @@ void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) { if (pReq->info == NULL) { pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } - for(int i = 0; i < kvNum; i++) { + for (int32_t i = 0; i < kvNum; i++) { SKv kv; buf = taosDecodeSKv(buf, &kv); taosHashPut(pReq->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); @@ -120,54 +120,55 @@ void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) { return buf; } -int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp) { - int tlen = 0; +int32_t tSerializeSClientHbRsp(void **buf, const SClientHbRsp *pRsp) { + int32_t tlen = 0; int32_t kvNum = taosArrayGetSize(pRsp->info); tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey); tlen += taosEncodeFixedI32(buf, pRsp->status); tlen += taosEncodeFixedI32(buf, kvNum); - for (int i = 0; i < kvNum; i++) { + for (int32_t i = 0; i < kvNum; i++) { SKv *kv = (SKv *)taosArrayGet(pRsp->info, i); tlen += taosEncodeSKv(buf, kv); } return tlen; } -void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp) { + +void *tDeserializeSClientHbRsp(void *buf, SClientHbRsp *pRsp) { int32_t kvNum = 0; buf = taosDecodeSClientHbKey(buf, &pRsp->connKey); buf = taosDecodeFixedI32(buf, &pRsp->status); buf = taosDecodeFixedI32(buf, &kvNum); pRsp->info = taosArrayInit(kvNum, sizeof(SKv)); - for (int i = 0; i < kvNum; i++) { + for (int32_t i = 0; i < kvNum; i++) { SKv kv = {0}; buf = taosDecodeSKv(buf, &kv); taosArrayPush(pRsp->info, &kv); } - + return buf; } -int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pBatchReq) { - int tlen = 0; +int32_t tSerializeSClientHbBatchReq(void **buf, const SClientHbBatchReq *pBatchReq) { + int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pBatchReq->reqId); int32_t reqNum = taosArrayGetSize(pBatchReq->reqs); - tlen += taosEncodeFixedI32(buf, reqNum); - for (int i = 0; i < reqNum; i++) { - SClientHbReq* pReq = taosArrayGet(pBatchReq->reqs, i); + tlen += taosEncodeFixedI32(buf, reqNum); + for (int32_t i = 0; i < reqNum; i++) { + SClientHbReq *pReq = taosArrayGet(pBatchReq->reqs, i); tlen += tSerializeSClientHbReq(buf, pReq); } return tlen; } -void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pBatchReq) { +void *tDeserializeSClientHbBatchReq(void *buf, SClientHbBatchReq *pBatchReq) { buf = taosDecodeFixedI64(buf, &pBatchReq->reqId); if (pBatchReq->reqs == NULL) { pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq)); } - + int32_t reqNum; buf = taosDecodeFixedI32(buf, &reqNum); - for (int i = 0; i < reqNum; i++) { + for (int32_t i = 0; i < reqNum; i++) { SClientHbReq req = {0}; buf = tDeserializeSClientHbReq(buf, &req); taosArrayPush(pBatchReq->reqs, &req); @@ -175,31 +176,31 @@ void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pBatchReq) { return buf; } -int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp) { - int tlen = 0; +int32_t tSerializeSClientHbBatchRsp(void **buf, const SClientHbBatchRsp *pBatchRsp) { + int32_t tlen = 0; int32_t sz = taosArrayGetSize(pBatchRsp->rsps); tlen += taosEncodeFixedI32(buf, sz); - for (int i = 0; i < sz; i++) { - SClientHbRsp* pRsp = taosArrayGet(pBatchRsp->rsps, i); + for (int32_t i = 0; i < sz; i++) { + SClientHbRsp *pRsp = taosArrayGet(pBatchRsp->rsps, i); tlen += tSerializeSClientHbRsp(buf, pRsp); } return tlen; } -void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp) { +void *tDeserializeSClientHbBatchRsp(void *buf, SClientHbBatchRsp *pBatchRsp) { int32_t sz; buf = taosDecodeFixedI32(buf, &sz); pBatchRsp->rsps = taosArrayInit(sz, sizeof(SClientHbRsp)); - for (int i = 0; i < sz; i++) { + for (int32_t i = 0; i < sz; i++) { SClientHbRsp rsp = {0}; - buf = tDeserializeSClientHbRsp(buf, &rsp); + buf = tDeserializeSClientHbRsp(buf, &rsp); taosArrayPush(pBatchRsp->rsps, &rsp); } return buf; } -int tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { - int tlen = 0; +int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { + int32_t tlen = 0; tlen += taosEncodeFixedU64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); @@ -293,8 +294,8 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { return buf; } -int tSVCreateTbBatchReqSerialize(void **buf, SVCreateTbBatchReq *pReq) { - int tlen = 0; +int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { + int32_t tlen = 0; tlen += taosEncodeFixedU64(buf, pReq->ver); tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray)); @@ -306,7 +307,7 @@ int tSVCreateTbBatchReqSerialize(void **buf, SVCreateTbBatchReq *pReq) { return tlen; } -void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) { +void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { uint32_t nsize = 0; buf = taosDecodeFixedU64(buf, &pReq->ver); @@ -320,3 +321,139 @@ void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) { return buf; } + +int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedU8(buf, pReq->type); + return tlen; +} + +void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { + buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeString(buf, &pReq->name); + buf = taosDecodeFixedU8(buf, &pReq->type); + return buf; +} + +int32_t tSerializeSMCreateStbReq(void **buf, SMCreateStbReq *pReq) { + int32_t tlen = 0; + + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedI8(buf, pReq->igExists); + tlen += taosEncodeFixedI32(buf, pReq->numOfColumns); + tlen += taosEncodeFixedI32(buf, pReq->numOfTags); + + for (int32_t i = 0; i < pReq->numOfColumns; ++i) { + SField *pField = taosArrayGet(pReq->pColumns, i); + tlen += taosEncodeFixedI8(buf, pField->type); + tlen += taosEncodeFixedI32(buf, pField->bytes); + tlen += taosEncodeString(buf, pField->name); + } + + for (int32_t i = 0; i < pReq->numOfTags; ++i) { + SField *pField = taosArrayGet(pReq->pTags, i); + tlen += taosEncodeFixedI8(buf, pField->type); + tlen += taosEncodeFixedI32(buf, pField->bytes); + tlen += taosEncodeString(buf, pField->name); + } + + return tlen; +} + +void *tDeserializeSMCreateStbReq(void *buf, SMCreateStbReq *pReq) { + buf = taosDecodeStringTo(buf, pReq->name); + buf = taosDecodeFixedI8(buf, &pReq->igExists); + buf = taosDecodeFixedI32(buf, &pReq->numOfColumns); + buf = taosDecodeFixedI32(buf, &pReq->numOfTags); + + pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField)); + pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField)); + if (pReq->pColumns == NULL || pReq->pTags == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < pReq->numOfColumns; ++i) { + SField field = {0}; + buf = taosDecodeFixedI8(buf, &field.type); + buf = taosDecodeFixedI32(buf, &field.bytes); + buf = taosDecodeStringTo(buf, field.name); + if (taosArrayPush(pReq->pColumns, &field) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } + + for (int32_t i = 0; i < pReq->numOfTags; ++i) { + SField field = {0}; + buf = taosDecodeFixedI8(buf, &field.type); + buf = taosDecodeFixedI32(buf, &field.bytes); + buf = taosDecodeStringTo(buf, field.name); + if (taosArrayPush(pReq->pTags, &field) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } + + return buf; +} + +int32_t tSerializeSMDropStbReq(void **buf, SMDropStbReq *pReq) { + int32_t tlen = 0; + + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedI8(buf, pReq->igNotExists); + + return tlen; +} + +void *tDeserializeSMDropStbReq(void *buf, SMDropStbReq *pReq) { + buf = taosDecodeStringTo(buf, pReq->name); + buf = taosDecodeFixedI8(buf, &pReq->igNotExists); + + return buf; +} + +int32_t tSerializeSMAlterStbReq(void **buf, SMAltertbReq *pReq) { + int32_t tlen = 0; + + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedI8(buf, pReq->alterType); + tlen += taosEncodeFixedI32(buf, pReq->numOfFields); + + for (int32_t i = 0; i < pReq->numOfFields; ++i) { + SField *pField = taosArrayGet(pReq->pFields, i); + tlen += taosEncodeFixedU8(buf, pField->type); + tlen += taosEncodeFixedI32(buf, pField->bytes); + tlen += taosEncodeString(buf, pField->name); + } + + return tlen; +} + +void *tDeserializeSMAlterStbReq(void *buf, SMAltertbReq *pReq) { + buf = taosDecodeStringTo(buf, pReq->name); + buf = taosDecodeFixedI8(buf, &pReq->alterType); + buf = taosDecodeFixedI32(buf, &pReq->numOfFields); + + pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SField)); + if (pReq->pFields == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < pReq->numOfFields; ++i) { + SField field = {0}; + buf = taosDecodeFixedU8(buf, &field.type); + buf = taosDecodeFixedI32(buf, &field.bytes); + buf = taosDecodeStringTo(buf, field.name); + if (taosArrayPush(pReq->pFields, &field) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } + + return buf; +} diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index c6e6c0006d..dc8390fba3 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -125,7 +125,7 @@ const char* Testbase::GetMetaName(int32_t index) { int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; } -const char* Testbase::GetMetaTbName() { return pMeta->tbFname; } +const char* Testbase::GetMetaTbName() { return pMeta->tbName; } void Testbase::SendShowRetrieveReq() { int32_t contLen = sizeof(SRetrieveTableReq); @@ -144,7 +144,7 @@ void Testbase::SendShowRetrieveReq() { pos = 0; } -const char* Testbase::GetShowName() { return pMeta->tbFname; } +const char* Testbase::GetShowName() { return pMeta->tbName; } int8_t Testbase::GetShowInt8() { int8_t data = *((int8_t*)(pData + pos)); diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp index 17092ed737..380a837c58 100644 --- a/source/dnode/mgmt/impl/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -108,7 +108,7 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { } } -TEST_F(DndTestVnode, 02_ALTER_Vnode) { +TEST_F(DndTestVnode, 02_Alter_Vnode) { for (int i = 0; i < 3; ++i) { int32_t contLen = sizeof(SAlterVnodeReq); @@ -199,19 +199,16 @@ TEST_F(DndTestVnode, 03_Create_Stb) { req.stbCfg.nTagCols = 3; req.stbCfg.pTagSchema = &schemas[2]; - int32_t bsize = tSerializeSVCreateTbReq(NULL, &req); - void* buf = rpcMallocCont(sizeof(SMsgHead) + bsize); - SMsgHead* pMsgHead = (SMsgHead*)buf; + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); - pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); - pMsgHead->vgId = htonl(2); + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(2); - void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void* pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tSerializeSVCreateTbReq(&pBuf, &req); - int32_t contLen = sizeof(SMsgHead) + bsize; - - SRpcMsg* pRsp = test.SendReq(TDMT_VND_CREATE_STB, buf, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_VND_CREATE_STB, (void*)pHead, contLen); ASSERT_NE(pRsp, nullptr); if (i == 0) { ASSERT_EQ(pRsp->code, 0); @@ -222,36 +219,98 @@ TEST_F(DndTestVnode, 03_Create_Stb) { } } -TEST_F(DndTestVnode, 04_ALTER_Stb) { -#if 0 +TEST_F(DndTestVnode, 04_Alter_Stb) { + for (int i = 0; i < 1; ++i) { + SVCreateTbReq req = {0}; + req.ver = 0; + req.name = (char*)"stb1"; + req.ttl = 0; + req.keep = 0; + req.type = TD_SUPER_TABLE; + + SSchema schemas[5] = {0}; + { + SSchema* pSchema = &schemas[0]; + pSchema->bytes = htonl(8); + pSchema->type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema->name, "ts"); + } + + { + SSchema* pSchema = &schemas[1]; + pSchema->bytes = htonl(4); + pSchema->type = TSDB_DATA_TYPE_INT; + strcpy(pSchema->name, "col1"); + } + + { + SSchema* pSchema = &schemas[2]; + pSchema->bytes = htonl(2); + pSchema->type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema->name, "_tag1"); + } + + { + SSchema* pSchema = &schemas[3]; + pSchema->bytes = htonl(8); + pSchema->type = TSDB_DATA_TYPE_BIGINT; + strcpy(pSchema->name, "_tag2"); + } + + { + SSchema* pSchema = &schemas[4]; + pSchema->bytes = htonl(16); + pSchema->type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema->name, "_tag3"); + } + + req.stbCfg.suid = 9527; + req.stbCfg.nCols = 2; + req.stbCfg.pSchema = &schemas[0]; + req.stbCfg.nTagCols = 3; + req.stbCfg.pTagSchema = &schemas[2]; + + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(2); + + void* pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVCreateTbReq(&pBuf, &req); + + SRpcMsg* pRsp = test.SendReq(TDMT_VND_ALTER_STB, (void*)pHead, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } +} + +TEST_F(DndTestVnode, 05_DROP_Stb) { { for (int i = 0; i < 3; ++i) { - SRpcMsg* pRsp = test.SendReq(TDMT_VND_ALTER_STB, pReq, contLen); + SVDropTbReq req = {0}; + req.ver = 0; + req.name = (char*)"stb1"; + req.suid = 9599; + req.type = TD_SUPER_TABLE; + + int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(2); + + void* pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVDropTbReq(&pBuf, &req); + + SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, (void*)pHead, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } } -#endif } -TEST_F(DndTestVnode, 05_DROP_Stb) { -#if 0 - { - for (int i = 0; i < 3; ++i) { - SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - if (i == 0) { - ASSERT_EQ(pRsp->code, 0); - test.Restart(); - } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_TDB_INVALID_TABLE_ID); - } - } - } -#endif -} - -TEST_F(DndTestVnode, 06_DROP_Vnode) { +TEST_F(DndTestVnode, 06_Drop_Vnode) { for (int i = 0; i < 3; ++i) { int32_t contLen = sizeof(SDropVnodeReq); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9609deb235..415b7e0fe3 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -301,10 +301,12 @@ typedef struct { uint64_t uid; uint64_t dbUid; int32_t version; + int32_t nextColId; int32_t numOfColumns; int32_t numOfTags; + SSchema* pColumns; + SSchema* pTags; SRWLatch lock; - SSchema* pSchema; } SStbObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 48847dc6a3..0855b5bd4d 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -28,6 +28,9 @@ void mndCleanupStb(SMnode *pMnode); SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen); + + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 189cbfea6d..c754494c24 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -426,7 +426,7 @@ static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_BNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 5f80672369..a53298efb3 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -186,7 +186,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp cols++; pMeta->numOfColumns = htonl(cols); - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -196,7 +196,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp pShow->numOfRows = 1; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5e6f457a67..1bfe8c9d9a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -907,9 +907,9 @@ int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void * len = 0; - SDbObj *pDb = mndAcquireDb(pMnode, db->dbName); + SDbObj *pDb = mndAcquireDb(pMnode, db->dbFName); if (pDb == NULL) { - mInfo("db %s not exist", db->dbName); + mInfo("db %s not exist", db->dbFName); len = sizeof(SUseDbRsp); } else if (pDb->uid != db->dbId || db->vgVersion < pDb->vgVersion) { @@ -929,7 +929,7 @@ int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void * } pRsp = (SUseDbRsp *)((char *)buf + bufOffset); - memcpy(pRsp->db, db->dbName, TSDB_DB_FNAME_LEN); + memcpy(pRsp->db, db->dbFName, TSDB_DB_FNAME_LEN); if (pDb) { int32_t vgNum = 0; mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); @@ -1113,7 +1113,7 @@ static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMe pShow->numOfRows = sdbGetSize(pSdb, SDB_DB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9a96bdb0df..3b43cd8081 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -608,7 +608,7 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp pShow->numOfRows = TSDB_CONFIG_NUMBER; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -715,7 +715,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_DNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 0e086fdb22..b3fc0e1da0 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -482,7 +482,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index a969c0162f..dde48853ad 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -624,7 +624,7 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_MNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); mndUpdateMnodeRole(pMnode); return 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d379d294ab..d63ade4320 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -18,6 +18,7 @@ #include "mndProfile.h" //#include "mndConsumer.h" #include "mndDb.h" +#include "mndStb.h" #include "mndMnode.h" #include "mndShow.h" //#include "mndTopic.h" @@ -376,9 +377,16 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } break; } - case HEARTBEAT_KEY_STBINFO: - + case HEARTBEAT_KEY_STBINFO: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateStbInfo(pMnode, (SSTableMetaVersion *)kv->value, kv->valueLen/sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv); + } break; + } default: mError("invalid kv key:%d", kv->key); hbRsp.status = TSDB_CODE_MND_APP_ERROR; @@ -623,7 +631,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -792,7 +800,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = 1000000; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 6077a95a7b..e29ab5a5bd 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -426,7 +426,7 @@ static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_QNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 7688ea76ab..cbba659719 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -428,7 +428,7 @@ static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_SNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 45b8428663..6914923b66 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -84,12 +84,20 @@ static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT64(pRaw, dataPos, pStb->uid, STB_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pStb->dbUid, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->version, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, STB_ENCODE_OVER) - int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; - for (int32_t i = 0; i < totalCols; ++i) { - SSchema *pSchema = &pStb->pSchema[i]; + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pStb->pColumns[i]; + SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pStb->pTags[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) @@ -137,17 +145,26 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pStb->uid, STB_DECODE_OVER) SDB_GET_INT64(pRaw, dataPos, &pStb->dbUid, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->version, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, STB_DECODE_OVER) - int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; - pStb->pSchema = calloc(totalCols, sizeof(SSchema)); - if (pStb->pSchema == NULL) { + pStb->pColumns = calloc(pStb->numOfColumns, sizeof(SSchema)); + pStb->pTags = calloc(pStb->numOfTags, sizeof(SSchema)); + if (pStb->pColumns == NULL || pStb->pTags == NULL) { goto STB_DECODE_OVER; } - for (int32_t i = 0; i < totalCols; ++i) { - SSchema *pSchema = &pStb->pSchema[i]; + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pStb->pColumns[i]; + SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pStb->pTags[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) @@ -176,29 +193,47 @@ static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { mTrace("stb:%s, perform delete action, row:%p", pStb->name, pStb); + tfree(pStb->pColumns); + tfree(pStb->pTags); return 0; } static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { mTrace("stb:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); - atomic_exchange_32(&pOld->updateTime, pNew->updateTime); - atomic_exchange_32(&pOld->version, pNew->version); taosWLockLatch(&pOld->lock); - pOld->numOfColumns = pNew->numOfColumns; - pOld->numOfTags = pNew->numOfTags; - int32_t totalCols = pNew->numOfTags + pNew->numOfColumns; - int32_t totalSize = totalCols * sizeof(SSchema); - if (pOld->numOfTags + pOld->numOfColumns < totalCols) { - void *pSchema = malloc(totalSize); - if (pSchema != NULL) { - free(pOld->pSchema); - pOld->pSchema = pSchema; + if (pOld->numOfColumns < pNew->numOfColumns) { + void *pColumns = malloc(pNew->numOfColumns * sizeof(SSchema)); + if (pColumns != NULL) { + free(pOld->pColumns); + pOld->pColumns = pColumns; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); } } - memcpy(pOld->pSchema, pNew->pSchema, totalSize); + if (pOld->numOfTags < pNew->numOfTags) { + void *pTags = malloc(pNew->numOfTags * sizeof(SSchema)); + if (pTags != NULL) { + free(pOld->pTags); + pOld->pTags = pTags; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); + } + } + + pOld->updateTime = pNew->updateTime; + pOld->version = pNew->version; + pOld->nextColId = pNew->nextColId; + pOld->numOfColumns = pNew->numOfColumns; + pOld->numOfTags = pNew->numOfTags; + memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema)); + memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema)); taosWUnLockLatch(&pOld->lock); return 0; } @@ -227,71 +262,67 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { return mndAcquireDb(pMnode, db); } -static void *mndBuildCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { - SVCreateTbReq req; - void *buf; - int32_t bsize; - SMsgHead *pMsgHead; - - req.ver = 0; +static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { SName name = {0}; tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + SVCreateTbReq req = {0}; + req.ver = 0; req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; req.type = TD_SUPER_TABLE; req.stbCfg.suid = pStb->uid; req.stbCfg.nCols = pStb->numOfColumns; - req.stbCfg.pSchema = pStb->pSchema; + req.stbCfg.pSchema = pStb->pColumns; req.stbCfg.nTagCols = pStb->numOfTags; - req.stbCfg.pTagSchema = pStb->pSchema + pStb->numOfColumns; + req.stbCfg.pTagSchema = pStb->pTags; - bsize = tSerializeSVCreateTbReq(NULL, &req); - buf = malloc(sizeof(SMsgHead) + bsize); - if (buf == NULL) { + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pMsgHead = (SMsgHead *)buf; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); - pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); - pMsgHead->vgId = htonl(pVgroup->vgId); - - void *pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tSerializeSVCreateTbReq(&pBuf, &req); - *pContLen = sizeof(SMsgHead) + bsize; - return buf; + *pContLen = contLen; + return pHead; } -static SVDropTbReq *mndBuildDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { - int32_t contLen = sizeof(SVDropTbReq); +static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - SVDropTbReq *pDrop = calloc(1, contLen); - if (pDrop == NULL) { + SVDropTbReq req = {0}; + req.ver = 0; + req.name = (char *)tNameGetTableName(&name); + req.type = TD_SUPER_TABLE; + req.suid = pStb->uid; + + int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pDrop->head.contLen = htonl(contLen); - pDrop->head.vgId = htonl(pVgroup->vgId); - memcpy(pDrop->name, pStb->name, TSDB_TABLE_FNAME_LEN); - pDrop->suid = htobe64(pStb->uid); + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); - return pDrop; + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVDropTbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; } static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { - pCreate->numOfColumns = htonl(pCreate->numOfColumns); - pCreate->numOfTags = htonl(pCreate->numOfTags); - int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags; - for (int32_t i = 0; i < totalCols; ++i) { - SSchema *pSchema = &pCreate->pSchema[i]; - pSchema->bytes = htonl(pSchema->bytes); - } - if (pCreate->igExists < 0 || pCreate->igExists > 1) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -307,18 +338,39 @@ static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { return -1; } - int32_t maxColId = (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS); - for (int32_t i = 0; i < totalCols; ++i) { - SSchema *pSchema = &pCreate->pSchema[i]; - if (pSchema->type < 0) { + SField *pField = taosArrayGet(pCreate->pColumns, 0) ; + if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + + for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { + SField *pField = taosArrayGet(pCreate->pColumns, i); + if (pField->type < 0) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } - if (pSchema->bytes <= 0) { + if (pField->bytes <= 0) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } - if (pSchema->name[0] == 0) { + if (pField->name[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + } + + for (int32_t i = 0; i < pCreate->numOfTags; ++i) { + SField *pField = taosArrayGet(pCreate->pTags, i); + if (pField->type < 0) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + if (pField->bytes <= 0) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + if (pField->name[0] == 0) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -358,14 +410,14 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; - int32_t contLen; + int32_t contLen; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - void *pReq = mndBuildCreateStbReq(pMnode, pVgroup, pStb, &contLen); + void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen); if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); @@ -400,7 +452,8 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - SVDropTbReq *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb); + int32_t contLen = 0; + void *pReq = mndBuildVDropStbReq(pMnode, pVgroup, pStb, &contLen); if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); @@ -411,7 +464,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pReq; - action.contLen = sizeof(SVDropTbReq); + action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -427,27 +480,42 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { SStbObj stbObj = {0}; - tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); - tstrncpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN); + memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); + memcpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN); stbObj.createdTime = taosGetTimestampMs(); stbObj.updateTime = stbObj.createdTime; stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); stbObj.dbUid = pDb->uid; stbObj.version = 1; + stbObj.nextColId = 1; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; - int32_t totalCols = stbObj.numOfColumns + stbObj.numOfTags; - int32_t totalSize = totalCols * sizeof(SSchema); - stbObj.pSchema = malloc(totalSize); - if (stbObj.pSchema == NULL) { + stbObj.pColumns = malloc(stbObj.numOfColumns * sizeof(SSchema)); + stbObj.pTags = malloc(stbObj.numOfTags * sizeof(SSchema)); + if (stbObj.pColumns == NULL || stbObj.pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.pSchema, pCreate->pSchema, totalSize); - for (int32_t i = 0; i < totalCols; ++i) { - stbObj.pSchema[i].colId = i + 1; + for (int32_t i = 0; i < stbObj.numOfColumns; ++i) { + SField *pField = taosArrayGet(pCreate->pColumns, i); + SSchema *pSchema = &stbObj.pColumns[i]; + pSchema->type = pField->type; + pSchema->bytes = pField->bytes; + memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); + pSchema->colId = stbObj.nextColId; + stbObj.nextColId++; + } + + for (int32_t i = 0; i < stbObj.numOfTags; ++i) { + SField *pField = taosArrayGet(pCreate->pTags, i); + SSchema *pSchema = &stbObj.pTags[i]; + pSchema->type = pField->type; + pSchema->bytes = pField->bytes; + memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); + pSchema->colId = stbObj.nextColId; + stbObj.nextColId++; } int32_t code = -1; @@ -471,58 +539,60 @@ CREATE_STB_OVER: } static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMCreateStbReq *pCreate = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SStbObj *pTopicStb = NULL; + SStbObj *pStb = NULL; + SDbObj *pDb = NULL; + SMCreateStbReq createReq = {0}; - mDebug("stb:%s, start to create", pCreate->name); + if (tDeserializeSMCreateStbReq(pReq->rpcMsg.pCont, &createReq) == NULL) goto CREATE_STB_OVER; - if (mndCheckCreateStbReq(pCreate) != 0) { - mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } + mDebug("stb:%s, start to create", createReq.name); + if (mndCheckCreateStbReq(&createReq) != 0) goto CREATE_STB_OVER; - SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name); + pStb = mndAcquireStb(pMnode, createReq.name); if (pStb != NULL) { - mndReleaseStb(pMnode, pStb); - if (pCreate->igExists) { - mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); - return 0; + if (createReq.igExists) { + mDebug("stb:%s, already exist, ignore exist is set", createReq.name); + code = 0; + goto CREATE_STB_OVER; } else { terrno = TSDB_CODE_MND_STB_ALREADY_EXIST; - mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_STB_OVER; } } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { - mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_STB_OVER; } - // topic should have different name with stb - SStbObj *pTopic = mndAcquireStb(pMnode, pCreate->name); - if (pTopic != NULL) { - sdbRelease(pMnode->pSdb, pTopic); + pTopicStb = mndAcquireStb(pMnode, createReq.name); + if (pTopicStb != NULL) { terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC; - mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_STB_OVER; } - SDbObj *pDb = mndAcquireDbByStb(pMnode, pCreate->name); + pDb = mndAcquireDbByStb(pMnode, createReq.name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_STB_OVER; } - int32_t code = mndCreateStb(pMnode, pReq, pCreate, pDb); - mndReleaseDb(pMnode, pDb); + code = mndCreateStb(pMnode, pReq, &createReq, pDb); +CREATE_STB_OVER: if (code != 0) { - terrno = code; - mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + mError("stb:%s, failed to create since %s", createReq.name, terrstr()); + } else { + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + mndReleaseStb(pMnode, pStb); + mndReleaseStb(pMnode, pTopicStb); + mndReleaseDb(pMnode, pDb); + taosArrayClear(createReq.pColumns); + taosArrayClear(createReq.pTags); + + return code; } static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp) { @@ -530,63 +600,455 @@ static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp) { return 0; } -static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { - SSchema *pSchema = &pAlter->schema; - pSchema->colId = htonl(pSchema->colId); - pSchema->bytes = htonl(pSchema->bytes); +static int32_t mndCheckAlterStbReq(SMAltertbReq *pAlter) { + if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } - if (pSchema->type <= 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; - } - if (pSchema->colId < 0 || pSchema->colId >= (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS)) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; - } - if (pSchema->bytes <= 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; - } - if (pSchema->name[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + for (int32_t i = 0; i < pAlter->numOfFields; ++i) { + SField *pField = taosArrayGet(pAlter->pFields, i); + + if (pField->type <= 0) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + if (pField->bytes <= 0) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + if (pField->name[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } } return 0; } -static int32_t mndUpdateStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pOld, SStbObj *pNew) { return 0; } +static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) { + for (int32_t tag = 0; tag < pStb->numOfTags; tag++) { + if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) { + return tag; + } + } + + return -1; +} + +static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *colName) { + for (int32_t col = 0; col < pStb->numOfColumns; col++) { + if (strcasecmp(pStb->pColumns[col].name, colName) == 0) { + return col; + } + } + + return -1; +} + +static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { + pNew->pTags = calloc(pNew->numOfTags, sizeof(SSchema)); + pNew->pColumns = calloc(pNew->numOfColumns, sizeof(SSchema)); + if (pNew->pTags == NULL || pNew->pColumns == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + memcpy(pNew->pColumns, pOld->pColumns, sizeof(SSchema) * pOld->numOfColumns); + memcpy(pNew->pTags, pOld->pTags, sizeof(SSchema) * pOld->numOfTags); + return 0; +} + +static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ntags) { + if (pOld->numOfTags + ntags > TSDB_MAX_TAGS) { + terrno = TSDB_CODE_MND_TOO_MANY_TAGS; + return -1; + } + + if (pOld->numOfColumns + ntags + pOld->numOfTags > TSDB_MAX_COLUMNS) { + terrno = TSDB_CODE_MND_TOO_MANY_COLUMNS; + return -1; + } + + pNew->numOfTags = pNew->numOfTags + ntags; + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + + for (int32_t i = 0; i < ntags; i++) { + SField *pField = taosArrayGet(pFields, i); + if (mndFindSuperTableColumnIndex(pOld, pField->name) > 0) { + terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; + return -1; + } + + if (mndFindSuperTableTagIndex(pOld, pField->name) > 0) { + terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; + return -1; + } + + SSchema *pSchema = &pNew->pTags[pOld->numOfTags + i]; + pSchema->bytes = pField->bytes; + pSchema->type = pField->type; + memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); + pSchema->colId = pNew->nextColId; + pNew->nextColId++; + + mDebug("stb:%s, start to add tag %s", pNew->name, pSchema->name); + } + + pNew->version++; + return 0; +} + +static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const char *tagName) { + int32_t tag = mndFindSuperTableTagIndex(pOld, tagName); + if (tag < 0) { + terrno = TSDB_CODE_MND_TAG_NOT_EXIST; + return -1; + } + + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + + memmove(pNew->pTags + tag, pNew->pTags + tag + 1, sizeof(SSchema) * (pNew->numOfTags - tag - 1)); + pNew->numOfTags--; + + pNew->version++; + mDebug("stb:%s, start to drop tag %s", pNew->name, tagName); + return 0; +} + +static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pFields) { + if ((int32_t)taosArrayGetSize(pFields) != 2) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + + SField *pField0 = taosArrayGet(pFields, 0); + SField *pField1 = taosArrayGet(pFields, 1); + + const char *oldTagName = pField0->name; + const char *newTagName = pField1->name; + + int32_t tag = mndFindSuperTableTagIndex(pOld, oldTagName); + if (tag < 0) { + terrno = TSDB_CODE_MND_TAG_NOT_EXIST; + return -1; + } + + if (mndFindSuperTableTagIndex(pOld, newTagName) >= 0) { + terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; + return -1; + } + + if (mndFindSuperTableColumnIndex(pOld, newTagName) >= 0) { + terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; + return -1; + } + + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + + SSchema *pSchema = (SSchema *)(pNew->pTags + tag); + memcpy(pSchema->name, newTagName, TSDB_COL_NAME_LEN); + + pNew->version++; + mDebug("stb:%s, start to modify tag %s to %s", pNew->name, oldTagName, newTagName); + return 0; +} + +static int32_t mndAlterStbTagBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) { + int32_t tag = mndFindSuperTableTagIndex(pOld, pField->name); + if (tag < 0) { + terrno = TSDB_CODE_MND_TAG_NOT_EXIST; + return -1; + } + + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + + SSchema *pTag = pNew->pTags + tag; + + if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_NCHAR)) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + + if (pField->bytes <= pTag->bytes) { + terrno = TSDB_CODE_MND_INVALID_ROW_BYTES; + return -1; + } + + pTag->bytes = pField->bytes; + pNew->version++; + + mDebug("stb:%s, start to modify tag len %s to %d", pNew->name, pField->name, pField->bytes); + return 0; +} + +static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols) { + if (pOld->numOfColumns + ncols + pOld->numOfTags > TSDB_MAX_COLUMNS) { + terrno = TSDB_CODE_MND_TOO_MANY_COLUMNS; + return -1; + } + + pNew->numOfColumns = pNew->numOfColumns + ncols; + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + + for (int32_t i = 0; i < ncols; i++) { + SField *pField = taosArrayGet(pFields, i); + if (mndFindSuperTableColumnIndex(pOld, pField->name) > 0) { + terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; + return -1; + } + + if (mndFindSuperTableTagIndex(pOld, pField->name) > 0) { + terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; + return -1; + } + + SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i]; + pSchema->bytes = pField->bytes; + pSchema->type = pField->type; + memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); + pSchema->colId = pNew->nextColId; + pNew->nextColId++; + + mDebug("stb:%s, start to add column %s", pNew->name, pSchema->name); + } + + pNew->version++; + return 0; +} + +static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const char *colName) { + int32_t col = mndFindSuperTableColumnIndex(pOld, colName); + if (col < 0) { + terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; + return -1; + } + + if (col == 0) { + terrno = TSDB_CODE_MND_INVALID_STB_ALTER_OPTION; + return -1; + } + + if (pOld->numOfColumns == 2) { + terrno = TSDB_CODE_MND_INVALID_STB_ALTER_OPTION; + return -1; + } + + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + + memmove(pNew->pColumns + col, pNew->pColumns + col + 1, sizeof(SSchema) * (pNew->numOfColumns - col - 1)); + pNew->numOfColumns--; + + pNew->version++; + mDebug("stb:%s, start to drop col %s", pNew->name, colName); + return 0; +} + +static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) { + int32_t col = mndFindSuperTableColumnIndex(pOld, pField->name); + if (col < 0) { + terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; + return -1; + } + + uint32_t nLen = 0; + for (int32_t i = 0; i < pOld->numOfColumns; ++i) { + nLen += (pOld->pColumns[i].colId == col) ? pField->bytes : pOld->pColumns[i].bytes; + } + + if (nLen > TSDB_MAX_BYTES_PER_ROW) { + terrno = TSDB_CODE_MND_INVALID_ROW_BYTES; + return -1; + } + + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + + SSchema *pCol = pNew->pColumns + col; + if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR)) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + + if (pField->bytes <= pCol->bytes) { + terrno = TSDB_CODE_MND_INVALID_ROW_BYTES; + return -1; + } + + pCol->bytes = pField->bytes; + pNew->version++; + + mDebug("stb:%s, start to modify col len %s to %d", pNew->name, pField->name, pField->bytes); + return 0; +} + +static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_UPDATING) != 0) return -1; + + return 0; +} + +static int32_t mndSetAlterStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; + + void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_STB; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndAlterStb(SMnode *pMnode, SMnodeMsg *pReq, const SMAltertbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { + SStbObj stbObj = {0}; + taosRLockLatch(&pOld->lock); + memcpy(&stbObj, pOld, sizeof(SStbObj)); + stbObj.pColumns = NULL; + stbObj.pTags = NULL; + stbObj.updateTime = taosGetTimestampMs(); + taosRUnLockLatch(&pOld->lock); + + int32_t code = -1; + STrans *pTrans = NULL; + SField *pField0 = taosArrayGet(pAlter->pFields, 0); + + switch (pAlter->alterType) { + case TSDB_ALTER_TABLE_ADD_TAG: + code = mndAddSuperTableTag(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); + break; + case TSDB_ALTER_TABLE_DROP_TAG: + code = mndDropSuperTableTag(pOld, &stbObj, pField0->name); + break; + case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: + code = mndAlterStbTagName(pOld, &stbObj, pAlter->pFields); + break; + case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: + code = mndAlterStbTagBytes(pOld, &stbObj, pField0); + break; + case TSDB_ALTER_TABLE_ADD_COLUMN: + code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); + break; + case TSDB_ALTER_TABLE_DROP_COLUMN: + code = mndDropSuperTableColumn(pOld, &stbObj, pField0->name); + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + code = mndAlterStbColumnBytes(pOld, &stbObj, pField0); + break; + default: + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + break; + } + + if (code != 0) goto ALTER_STB_OVER; + + code = -1; + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pReq->rpcMsg); + if (pTrans == NULL) goto ALTER_STB_OVER; + + mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name); + + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto ALTER_STB_OVER; + if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto ALTER_STB_OVER; + if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto ALTER_STB_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto ALTER_STB_OVER; + + code = 0; + +ALTER_STB_OVER: + mndTransDrop(pTrans); + tfree(stbObj.pTags); + tfree(stbObj.pColumns); + return code; +} static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMAlterStbReq *pAlter = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SMAltertbReq alterReq = {0}; - mDebug("stb:%s, start to alter", pAlter->name); + if (tDeserializeSMAlterStbReq(pReq->rpcMsg.pCont, &alterReq) == NULL) goto ALTER_STB_OVER; - if (mndCheckAlterStbReq(pAlter) != 0) { - mError("stb:%s, failed to alter since %s", pAlter->name, terrstr()); - return -1; + mDebug("stb:%s, start to alter", alterReq.name); + if (mndCheckAlterStbReq(&alterReq) != 0) goto ALTER_STB_OVER; + + pDb = mndAcquireDbByStb(pMnode, alterReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_INVALID_DB; + goto ALTER_STB_OVER; } - SStbObj *pStb = mndAcquireStb(pMnode, pAlter->name); + pStb = mndAcquireStb(pMnode, alterReq.name); if (pStb == NULL) { terrno = TSDB_CODE_MND_STB_NOT_EXIST; - mError("stb:%s, failed to alter since %s", pAlter->name, terrstr()); - return -1; + goto ALTER_STB_OVER; } - SStbObj stbObj = {0}; - memcpy(&stbObj, pStb, sizeof(SStbObj)); - - int32_t code = mndUpdateStb(pMnode, pReq, pStb, &stbObj); - mndReleaseStb(pMnode, pStb); + code = mndAlterStb(pMnode, pReq, &alterReq, pDb, pStb); +ALTER_STB_OVER: if (code != 0) { - mError("stb:%s, failed to alter since %s", pAlter->name, tstrerror(code)); - return code; + mError("stb:%s, failed to alter since %s", alterReq.name, terrstr()); + } else { + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + mndReleaseStb(pMnode, pStb); + mndReleaseDb(pMnode, pDb); + taosArrayClear(alterReq.pFields); + + return code; } static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp) { @@ -603,15 +1065,6 @@ static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pS return 0; } -static int32_t mndSetDropStbUndoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { - SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; - - return 0; -} - static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); if (pCommitRaw == NULL) return -1; @@ -621,22 +1074,54 @@ static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj * return 0; } -static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } +static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; -static int32_t mndSetDropStbUndoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; -static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pStb) { + int32_t contLen = 0; + void *pReq = mndBuildVDropStbReq(pMnode, pVgroup, pStb, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_DROP_STB; + action.acceptableCode = TSDB_CODE_VND_TB_NOT_EXIST; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); - if (pTrans == NULL)goto DROP_STB_OVER; + if (pTrans == NULL) goto DROP_STB_OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbUndoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbRedoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbUndoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; + if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto DROP_STB_OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_STB_OVER; code = 0; @@ -647,28 +1132,39 @@ DROP_STB_OVER: } static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMDropStbReq *pDrop = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; - mDebug("stb:%s, start to drop", pDrop->name); + SMDropStbReq dropReq = {0}; + tDeserializeSMDropStbReq(pReq->rpcMsg.pCont, &dropReq); - SStbObj *pStb = mndAcquireStb(pMnode, pDrop->name); + mDebug("stb:%s, start to drop", dropReq.name); + + SStbObj *pStb = mndAcquireStb(pMnode, dropReq.name); if (pStb == NULL) { - if (pDrop->igNotExists) { - mDebug("stb:%s, not exist, ignore not exist is set", pDrop->name); + if (dropReq.igNotExists) { + mDebug("stb:%s, not exist, ignore not exist is set", dropReq.name); return 0; } else { terrno = TSDB_CODE_MND_STB_NOT_EXIST; - mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); + mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); return -1; } } - int32_t code = mndDropStb(pMnode, pReq, pStb); + SDbObj *pDb = mndAcquireDbByStb(pMnode, dropReq.name); + if (pDb == NULL) { + mndReleaseStb(pMnode, pStb); + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); + return -1; + } + + int32_t code = mndDropStb(pMnode, pReq, pDb, pStb); + mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); if (code != 0) { - mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); + mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); return -1; } @@ -684,20 +1180,23 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; STableInfoReq *pInfo = pReq->rpcMsg.pCont; - mDebug("stb:%s, start to retrieve meta", pInfo->tableFname); + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", pInfo->dbFName, pInfo->tbName); - SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->tableFname); + mDebug("stb:%s, start to retrieve meta", tbFName); + + SDbObj *pDb = mndAcquireDb(pMnode, pInfo->dbFName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stb:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr()); + mError("stb:%s, failed to retrieve meta since %s", tbFName, terrstr()); return -1; } - SStbObj *pStb = mndAcquireStb(pMnode, pInfo->tableFname); + SStbObj *pStb = mndAcquireStb(pMnode, tbFName); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); terrno = TSDB_CODE_MND_INVALID_STB; - mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + mError("stb:%s, failed to get meta since %s", tbFName, terrstr()); return -1; } @@ -711,11 +1210,14 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + mError("stb:%s, failed to get meta since %s", tbFName, terrstr()); return -1; } - memcpy(pMeta->tbFname, pStb->name, TSDB_TABLE_FNAME_LEN); + strcpy(pMeta->dbFName, pStb->db); + strcpy(pMeta->tbName, pInfo->tbName); + strcpy(pMeta->stbName, pInfo->tbName); + pMeta->dbId = htobe64(pDb->uid); pMeta->numOfTags = htonl(pStb->numOfTags); pMeta->numOfColumns = htonl(pStb->numOfColumns); pMeta->precision = pDb->cfg.precision; @@ -725,14 +1227,24 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { pMeta->suid = htobe64(pStb->uid); pMeta->tuid = htobe64(pStb->uid); - for (int32_t i = 0; i < totalCols; ++i) { + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pMeta->pSchema[i]; - SSchema *pSrcSchema = &pStb->pSchema[i]; + SSchema *pSrcSchema = &pStb->pColumns[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; pSchema->colId = htonl(pSrcSchema->colId); pSchema->bytes = htonl(pSrcSchema->bytes); } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pMeta->pSchema[i + pStb->numOfColumns]; + SSchema *pSrcSchema = &pStb->pTags[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); + } + taosRUnLockLatch(&pStb->lock); mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); @@ -740,7 +1252,119 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { pReq->pCont = pMeta; pReq->contLen = contLen; - mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pStb->numOfColumns, pStb->numOfTags); + mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", tbFName, pStb->numOfColumns, pStb->numOfTags); + return 0; +} + +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen) { + SSdb *pSdb = pMnode->pSdb; + int32_t bufSize = num * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); + void *buf = malloc(bufSize); + int32_t len = 0; + int32_t contLen = 0; + STableMetaRsp *pRsp = NULL; + + for (int32_t i = 0; i < num; ++i) { + SSTableMetaVersion *stb = &stbs[i]; + stb->suid = be64toh(stb->suid); + stb->sversion = ntohs(stb->sversion); + stb->tversion = ntohs(stb->tversion); + + if ((contLen + sizeof(STableMetaRsp)) > bufSize) { + bufSize = contLen + (num - i) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); + buf = realloc(buf, bufSize); + } + + pRsp = (STableMetaRsp *)((char *)buf + contLen); + + strcpy(pRsp->dbFName, stb->dbFName); + strcpy(pRsp->tbName, stb->stbName); + strcpy(pRsp->stbName, stb->stbName); + + mDebug("start to retrieve meta, db:%s, stb:%s", stb->dbFName, stb->stbName); + + SDbObj *pDb = mndAcquireDb(pMnode, stb->dbFName); + if (pDb == NULL) { + pRsp->numOfColumns = -1; + pRsp->suid = htobe64(stb->suid); + contLen += sizeof(STableMetaRsp); + mWarn("db:%s, failed to require db since %s", stb->dbFName, terrstr()); + continue; + } + + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", stb->dbFName, stb->stbName); + + SStbObj *pStb = mndAcquireStb(pMnode, tbFName); + if (pStb == NULL) { + mndReleaseDb(pMnode, pDb); + pRsp->numOfColumns = -1; + pRsp->suid = htobe64(stb->suid); + contLen += sizeof(STableMetaRsp); + mWarn("stb:%s, failed to get meta since %s", tbFName, terrstr()); + continue; + } + + taosRLockLatch(&pStb->lock); + + if (stb->suid == pStb->uid && stb->sversion == pStb->version) { + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); + continue; + } + + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + int32_t len = totalCols * sizeof(SSchema); + + contLen += sizeof(STableMetaRsp) + len; + + if (contLen > bufSize) { + bufSize = contLen + (num - i - 1) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); + buf = realloc(buf, bufSize); + } + + pRsp->numOfTags = htonl(pStb->numOfTags); + pRsp->numOfColumns = htonl(pStb->numOfColumns); + pRsp->precision = pDb->cfg.precision; + pRsp->tableType = TSDB_SUPER_TABLE; + pRsp->update = pDb->cfg.update; + pRsp->sversion = htonl(pStb->version); + pRsp->suid = htobe64(pStb->uid); + pRsp->tuid = htobe64(pStb->uid); + + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pRsp->pSchema[i]; + SSchema *pSrcSchema = &pStb->pColumns[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pRsp->pSchema[i + pStb->numOfColumns]; + SSchema *pSrcSchema = &pStb->pTags[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); + } + + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); + } + + if (contLen > 0) { + *rsp = buf; + *rspLen = contLen; + } else { + *rsp = NULL; + tfree(buf); + *rspLen = 0; + } + return 0; } @@ -816,7 +1440,7 @@ static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pM pShow->numOfRows = sdbGetSize(pSdb, SDB_STB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -856,7 +1480,7 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 if (pStb->dbUid != pDb->uid) { if (strncmp(pStb->db, pDb->name, tListLen(pStb->db)) == 0) { - mError("Inconsistent table data, name:%s, db:%s, dbUid:%"PRIu64, pStb->name, pDb->name, pDb->uid); + mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid); } sdbRelease(pSdb, pStb); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 33def69a68..ea3cc7c2c6 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -338,7 +338,7 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; STableInfoReq *pInfo = pMsg->rpcMsg.pCont; - mDebug("topic:%s, start to retrieve meta", pInfo->tableFname); + mDebug("topic:%s, start to retrieve meta", pInfo->tbName); #if 0 SDbObj *pDb = mndAcquireDbByTopic(pMnode, pInfo->tableFname); @@ -469,7 +469,7 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_TOPIC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c36e128893..3040cd8a28 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -474,7 +474,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p pShow->numOfRows = sdbGetSize(pSdb, SDB_USER); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 01d39ebdc2..33aa4ce07c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -525,7 +525,7 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -638,7 +638,7 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->replica = dnodeId; pShow->numOfRows = mndGetVnodesNum(pMnode, dnodeId); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index bfdc0f42b6..bc31630ffe 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -64,7 +64,7 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { test.SendShowMetaReq(TSDB_MGMT_TABLE_CONNS, ""); STableMetaRsp* pMeta = test.GetShowMeta(); - EXPECT_STREQ(pMeta->tbFname, "show connections"); + EXPECT_STREQ(pMeta->tbName, "show connections"); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->numOfColumns, 7); EXPECT_EQ(pMeta->precision, 0); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index beb52d68c1..c5ab44a014 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -21,112 +21,324 @@ class MndTestStb : public ::testing::Test { public: void SetUp() override {} void TearDown() override {} + + void* BuildCreateDbReq(const char* dbname, int32_t* pContLen); + void* BuildDropDbReq(const char* dbname, int32_t* pContLen); + void* BuildCreateStbReq(const char* stbname, int32_t* pContLen); + void* BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen); + void* BuildAlterStbDropTagReq(const char* stbname, const char* tagname, int32_t* pContLen); + void* BuildAlterStbUpdateTagNameReq(const char* stbname, const char* tagname, const char* newtagname, + int32_t* pContLen); + void* BuildAlterStbUpdateTagBytesReq(const char* stbname, const char* tagname, int32_t bytes, int32_t* pContLen); + void* BuildAlterStbAddColumnReq(const char* stbname, const char* colname, int32_t* pContLen); + void* BuildAlterStbDropColumnReq(const char* stbname, const char* colname, int32_t* pContLen); + void* BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, int32_t* pContLen); }; Testbase MndTestStb::test; -TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { +void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { + int32_t contLen = sizeof(SCreateDbReq); + + SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); + strcpy(pReq->db, dbname); + pReq->numOfVgroups = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; + + *pContLen = contLen; + return pReq; +} + +void* MndTestStb::BuildDropDbReq(const char* dbname, int32_t* pContLen) { + int32_t contLen = sizeof(SDropDbReq); + + SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); + strcpy(pReq->db, dbname); + + *pContLen = contLen; + return pReq; +} + +void* MndTestStb::BuildCreateStbReq(const char* stbname, int32_t* pContLen) { + SMCreateStbReq createReq = {0}; + createReq.numOfColumns = 2; + createReq.numOfTags = 3; + createReq.igExists = 0; + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + strcpy(createReq.name, stbname); + { - int32_t contLen = sizeof(SCreateDbReq); + SField field = {0}; + field.bytes = 8; + field.type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(field.name, "ts"); + taosArrayPush(createReq.pColumns, &field); + } - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + { + SField field = {0}; + field.bytes = 12; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, "col1"); + taosArrayPush(createReq.pColumns, &field); + } + { + SField field = {0}; + field.bytes = 2; + field.type = TSDB_DATA_TYPE_TINYINT; + strcpy(field.name, "tag1"); + taosArrayPush(createReq.pTags, &field); + } + + { + SField field = {0}; + field.bytes = 8; + field.type = TSDB_DATA_TYPE_BIGINT; + strcpy(field.name, "tag2"); + taosArrayPush(createReq.pTags, &field); + } + + { + SField field = {0}; + field.bytes = 16; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, "tag3"); + taosArrayPush(createReq.pTags, &field); + } + + int32_t tlen = tSerializeSMCreateStbReq(NULL, &createReq); + void* pHead = rpcMallocCont(tlen); + + void* pBuf = pHead; + tSerializeSMCreateStbReq(&pBuf, &createReq); + *pContLen = tlen; + return pHead; +} + +void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen) { + SMAltertbReq req = {0}; + strcpy(req.name, stbname); + req.numOfFields = 1; + req.pFields = taosArrayInit(1, sizeof(SField)); + req.alterType = TSDB_ALTER_TABLE_ADD_TAG; + + SField field = {0}; + field.bytes = 12; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, tagname); + taosArrayPush(req.pFields, &field); + + int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + void* pHead = rpcMallocCont(contLen); + void* pBuf = pHead; + tSerializeSMAlterStbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +void* MndTestStb::BuildAlterStbDropTagReq(const char* stbname, const char* tagname, int32_t* pContLen) { + SMAltertbReq req = {0}; + strcpy(req.name, stbname); + req.numOfFields = 1; + req.pFields = taosArrayInit(1, sizeof(SField)); + req.alterType = TSDB_ALTER_TABLE_DROP_TAG; + + SField field = {0}; + field.bytes = 12; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, tagname); + taosArrayPush(req.pFields, &field); + + int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + void* pHead = rpcMallocCont(contLen); + void* pBuf = pHead; + tSerializeSMAlterStbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +void* MndTestStb::BuildAlterStbUpdateTagNameReq(const char* stbname, const char* tagname, const char* newtagname, + int32_t* pContLen) { + SMAltertbReq req = {0}; + strcpy(req.name, stbname); + req.numOfFields = 2; + req.pFields = taosArrayInit(2, sizeof(SField)); + req.alterType = TSDB_ALTER_TABLE_UPDATE_TAG_NAME; + + SField field = {0}; + field.bytes = 12; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, tagname); + taosArrayPush(req.pFields, &field); + + SField field2 = {0}; + field2.bytes = 12; + field2.type = TSDB_DATA_TYPE_BINARY; + strcpy(field2.name, newtagname); + taosArrayPush(req.pFields, &field2); + + int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + void* pHead = rpcMallocCont(contLen); + void* pBuf = pHead; + tSerializeSMAlterStbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +void* MndTestStb::BuildAlterStbUpdateTagBytesReq(const char* stbname, const char* tagname, int32_t bytes, + int32_t* pContLen) { + SMAltertbReq req = {0}; + strcpy(req.name, stbname); + req.numOfFields = 1; + req.pFields = taosArrayInit(1, sizeof(SField)); + req.alterType = TSDB_ALTER_TABLE_UPDATE_TAG_BYTES; + + SField field = {0}; + field.bytes = bytes; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, tagname); + taosArrayPush(req.pFields, &field); + + int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + void* pHead = rpcMallocCont(contLen); + void* pBuf = pHead; + tSerializeSMAlterStbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +void* MndTestStb::BuildAlterStbAddColumnReq(const char* stbname, const char* colname, int32_t* pContLen) { + SMAltertbReq req = {0}; + strcpy(req.name, stbname); + req.numOfFields = 1; + req.pFields = taosArrayInit(1, sizeof(SField)); + req.alterType = TSDB_ALTER_TABLE_ADD_COLUMN; + + SField field = {0}; + field.bytes = 12; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, colname); + taosArrayPush(req.pFields, &field); + + int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + void* pHead = rpcMallocCont(contLen); + void* pBuf = pHead; + tSerializeSMAlterStbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* colname, int32_t* pContLen) { + SMAltertbReq req = {0}; + strcpy(req.name, stbname); + req.numOfFields = 1; + req.pFields = taosArrayInit(1, sizeof(SField)); + req.alterType = TSDB_ALTER_TABLE_DROP_COLUMN; + + SField field = {0}; + field.bytes = 12; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, colname); + taosArrayPush(req.pFields, &field); + + int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + void* pHead = rpcMallocCont(contLen); + void* pBuf = pHead; + tSerializeSMAlterStbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, + int32_t* pContLen) { + SMAltertbReq req = {0}; + strcpy(req.name, stbname); + req.numOfFields = 1; + req.pFields = taosArrayInit(1, sizeof(SField)); + req.alterType = TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES; + + SField field = {0}; + field.bytes = bytes; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, colname); + taosArrayPush(req.pFields, &field); + + int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + void* pHead = rpcMallocCont(contLen); + void* pBuf = pHead; + tSerializeSMAlterStbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { + const char* dbname = "1.d1"; + const char* stbname = "1.d1.stb"; + + { + int32_t contLen = 0; + void* pReq = BuildCreateDbReq(dbname, &contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } { - int32_t cols = 2; - int32_t tags = 3; - int32_t contLen = (tags + cols) * sizeof(SSchema) + sizeof(SMCreateStbReq); - - SMCreateStbReq* pReq = (SMCreateStbReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "1.d1.stb"); - pReq->numOfTags = htonl(tags); - pReq->numOfColumns = htonl(cols); - - { - SSchema* pSchema = &pReq->pSchema[0]; - pSchema->bytes = htonl(8); - pSchema->type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema->name, "ts"); - } - - { - SSchema* pSchema = &pReq->pSchema[1]; - pSchema->bytes = htonl(4); - pSchema->type = TSDB_DATA_TYPE_INT; - strcpy(pSchema->name, "col1"); - } - - { - SSchema* pSchema = &pReq->pSchema[2]; - pSchema->bytes = htonl(2); - pSchema->type = TSDB_DATA_TYPE_TINYINT; - strcpy(pSchema->name, "tag1"); - } - - { - SSchema* pSchema = &pReq->pSchema[3]; - pSchema->bytes = htonl(8); - pSchema->type = TSDB_DATA_TYPE_BIGINT; - strcpy(pSchema->name, "tag2"); - } - - { - SSchema* pSchema = &pReq->pSchema[4]; - pSchema->bytes = htonl(16); - pSchema->type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema->name, "tag3"); - } - + int32_t contLen = 0; + void* pReq = BuildCreateStbReq(stbname, &contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, "1.d1"); - CHECK_META("show stables", 4); + { + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + CHECK_META("show stables", 4); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_INT, 4, "columns"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_INT, 4, "tags"); - CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); - CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - CHECK_SCHEMA(2, TSDB_DATA_TYPE_INT, 4, "columns"); - CHECK_SCHEMA(3, TSDB_DATA_TYPE_INT, 4, "tags"); - - test.SendShowRetrieveReq(); - EXPECT_EQ(test.GetShowRows(), 1); - CheckBinary("stb", TSDB_TABLE_NAME_LEN); - CheckTimestamp(); - CheckInt32(2); - CheckInt32(3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(3); + } // ----- meta ------ { - int32_t contLen = sizeof(STableInfoReq); - + int32_t contLen = sizeof(STableInfoReq); STableInfoReq* pReq = (STableInfoReq*)rpcMallocCont(contLen); - strcpy(pReq->tableFname, "1.d1.stb"); + strcpy(pReq->dbFName, dbname); + strcpy(pReq->tbName, "stb"); SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); @@ -146,8 +358,9 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pSchema->bytes = htonl(pSchema->bytes); } - EXPECT_STREQ(pRsp->tbFname, "1.d1.stb"); - EXPECT_STREQ(pRsp->stbFname, ""); + EXPECT_STREQ(pRsp->dbFName, dbname); + EXPECT_STREQ(pRsp->tbName, "stb"); + EXPECT_STREQ(pRsp->stbName, "stb"); EXPECT_EQ(pRsp->numOfColumns, 2); EXPECT_EQ(pRsp->numOfTags, 3); EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); @@ -169,9 +382,9 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { SSchema* pSchema = &pRsp->pSchema[1]; - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->colId, 2); - EXPECT_EQ(pSchema->bytes, 4); + EXPECT_EQ(pSchema->bytes, 12); EXPECT_STREQ(pSchema->name, "col1"); } @@ -203,29 +416,476 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { // restart test.Restart(); - test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, "1.d1"); - CHECK_META("show stables", 4); - test.SendShowRetrieveReq(); - EXPECT_EQ(test.GetShowRows(), 1); + { + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + CHECK_META("show stables", 4); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); - CheckBinary("stb", TSDB_TABLE_NAME_LEN); - CheckTimestamp(); - CheckInt32(2); - CheckInt32(3); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(3); + } { - int32_t contLen = sizeof(SMDropStbReq); + SMDropStbReq dropReq = {0}; + strcpy(dropReq.name, stbname); - SMDropStbReq* pReq = (SMDropStbReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "1.d1.stb"); + int32_t contLen = tSerializeSMDropStbReq(NULL, &dropReq); + void* pHead = rpcMallocCont(contLen); + void* pBuf = pHead; + tSerializeSMDropStbReq(&pBuf, &dropReq); - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_STB, pHead, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, "1.d1"); - CHECK_META("show stables", 4); - test.SendShowRetrieveReq(); - EXPECT_EQ(test.GetShowRows(), 0); + { + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + CHECK_META("show stables", 4); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); + } + + { + int32_t contLen = 0; + void* pReq = BuildDropDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } +} + +TEST_F(MndTestStb, 02_Alter_Stb_AddTag) { + const char* dbname = "1.d2"; + const char* stbname = "1.d2.stb"; + int32_t contLen = 0; + + { + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildCreateStbReq(stbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildAlterStbAddTagReq("1.d3.stb", "tag4", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_DB); + } + + { + void* pReq = BuildAlterStbAddTagReq("1.d2.stb3", "tag4", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST); + } + + { + void* pReq = BuildAlterStbAddTagReq(stbname, "tag3", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST); + } + + { + void* pReq = BuildAlterStbAddTagReq(stbname, "col1", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST); + } + + { + void* pReq = BuildAlterStbAddTagReq(stbname, "tag4", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(4); + } + + { + void* pReq = BuildDropDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } +} + +TEST_F(MndTestStb, 03_Alter_Stb_DropTag) { + const char* dbname = "1.d3"; + const char* stbname = "1.d3.stb"; + int32_t contLen = 0; + + { + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildCreateStbReq(stbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildAlterStbDropTagReq(stbname, "tag5", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST); + } + + { + void* pReq = BuildAlterStbDropTagReq(stbname, "tag3", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(2); + } + + { + void* pReq = BuildDropDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } +} + +TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) { + const char* dbname = "1.d4"; + const char* stbname = "1.d4.stb"; + int32_t contLen = 0; + + { + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildCreateStbReq(stbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag5", "tag6", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST); + } + + { + void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "col1", "tag6", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST); + } + + { + void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "col1", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST); + } + + { + void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "tag2", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST); + } + { + void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "tag2", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST); + } + + { + void* pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "tag4", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(3); + } + + { + void* pReq = BuildDropDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } +} + +TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) { + const char* dbname = "1.d5"; + const char* stbname = "1.d5.stb"; + int32_t contLen = 0; + + { + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildCreateStbReq(stbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag5", 12, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST); + } + + { + void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag1", 13, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION); + } + + { + void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag3", 8, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); + } + + { + void* pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag3", 20, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(3); + } + + { + void* pReq = BuildDropDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } +} + +TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) { + const char* dbname = "1.d6"; + const char* stbname = "1.d6.stb"; + int32_t contLen = 0; + + { + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildCreateStbReq(stbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildAlterStbAddColumnReq("1.d7.stb", "tag4", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_DB); + } + + { + void* pReq = BuildAlterStbAddColumnReq("1.d6.stb3", "tag4", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST); + } + + { + void* pReq = BuildAlterStbAddColumnReq(stbname, "tag3", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST); + } + + { + void* pReq = BuildAlterStbAddColumnReq(stbname, "col1", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST); + } + + { + void* pReq = BuildAlterStbAddColumnReq(stbname, "col2", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(3); + CheckInt32(3); + } + + { + void* pReq = BuildDropDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } +} + +TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) { + const char* dbname = "1.d7"; + const char* stbname = "1.d7.stb"; + int32_t contLen = 0; + + { + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildCreateStbReq(stbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildAlterStbDropColumnReq(stbname, "col4", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST); + } + + { + void* pReq = BuildAlterStbDropColumnReq(stbname, "col1", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_ALTER_OPTION); + } + + { + void* pReq = BuildAlterStbDropColumnReq(stbname, "ts", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_ALTER_OPTION); + } + + { + void* pReq = BuildAlterStbAddColumnReq(stbname, "col2", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildAlterStbDropColumnReq(stbname, "col1", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(3); + } + + { + void* pReq = BuildDropDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } +} + +TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) { + const char* dbname = "1.d8"; + const char* stbname = "1.d8.stb"; + int32_t contLen = 0; + + { + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildCreateStbReq(stbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col5", 12, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST); + } + + { + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "ts", 8, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION); + } + + { + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 8, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); + } + + { + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", TSDB_MAX_BYTES_PER_ROW, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); + } + + { + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 20, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(3); + } + + { + void* pReq = BuildDropDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 21f5d027ad..a45a9d5a72 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -84,7 +84,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { int msgLen = 0; int32_t code = TSDB_CODE_VND_APP_ERROR; - pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid); + pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tbName, &uid); if (pTbCfg == NULL) { code = TSDB_CODE_VND_TB_NOT_EXIST; goto _exit; @@ -119,13 +119,13 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - memcpy(pTbMetaMsg->dbFname, pReq->dbFname, sizeof(pTbMetaMsg->dbFname)); - strcpy(pTbMetaMsg->tbFname, pTbCfg->name); + memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName)); + strcpy(pTbMetaMsg->tbName, pReq->tbName); if (pTbCfg->type == META_CHILD_TABLE) { - strcpy(pTbMetaMsg->stbFname, pStbCfg->name); + strcpy(pTbMetaMsg->stbName, pStbCfg->name); pTbMetaMsg->suid = htobe64(pTbCfg->ctbCfg.suid); } else if (pTbCfg->type == META_SUPER_TABLE) { - strcpy(pTbMetaMsg->stbFname, pTbCfg->name); + strcpy(pTbMetaMsg->stbName, pTbCfg->name); pTbMetaMsg->suid = htobe64(uid); } pTbMetaMsg->numOfTags = htonl(nTagCols); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index faf3171d76..28487821e6 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -83,7 +83,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { free(vCreateTbReq.name); break; case TDMT_VND_CREATE_TABLE: - tSVCreateTbBatchReqDeserialize(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); + tDeserializeSVCreateTbBatchReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); for (int i = 0; i < taosArrayGetSize(vCreateTbBatchReq.pArray); i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { @@ -105,7 +105,16 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { taosArrayDestroy(vCreateTbBatchReq.pArray); break; + case TDMT_VND_ALTER_STB: + vTrace("vgId:%d, process alter stb req", pVnode->vgId); + tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); + free(vCreateTbReq.stbCfg.pSchema); + free(vCreateTbReq.stbCfg.pTagSchema); + free(vCreateTbReq.name); + break; case TDMT_VND_DROP_STB: + vTrace("vgId:%d, process drop stb req", pVnode->vgId); + break; case TDMT_VND_DROP_TABLE: // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { // // TODO: handle error diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 7128e143bf..8f24374387 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -27,7 +27,7 @@ extern "C" { #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 #define CTG_DEFAULT_CACHE_DB_NUMBER 20 -#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 100000 +#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 10000 #define CTG_DEFAULT_RENT_SECOND 10 #define CTG_DEFAULT_RENT_SLOT_SIZE 10 @@ -47,55 +47,57 @@ enum { CTG_RENT_STABLE, }; -typedef struct SCTGDebug { - int32_t lockDebug; -} SCTGDebug; +typedef struct SCtgDebug { + bool lockDebug; + bool cacheDebug; + uint32_t showCachePeriodSec; +} SCtgDebug; -typedef struct SVgroupListCache { - int32_t vgroupVersion; - SHashObj *cache; // key:vgId, value:SVgroupInfo -} SVgroupListCache; +typedef struct SCtgTbMetaCache { + SRWLatch stbLock; + SRWLatch metaLock; // RC between cache destroy and all other operations + SHashObj *metaCache; //key:tbname, value:STableMeta + SHashObj *stbCache; //key:suid, value:STableMeta* +} SCtgTbMetaCache; -typedef struct SDBVgroupCache { - SHashObj *cache; //key:dbname, value:SDBVgroupInfo -} SDBVgroupCache; +typedef struct SCtgDBCache { + SRWLatch vgLock; + uint64_t dbId; + int8_t deleted; + SDBVgroupInfo *vgInfo; + SCtgTbMetaCache tbCache; +} SCtgDBCache; -typedef struct STableMetaCache { - SRWLatch stableLock; - SHashObj *cache; //key:fulltablename, value:STableMeta - SHashObj *stableCache; //key:suid, value:STableMeta* -} STableMetaCache; - -typedef struct SRentSlotInfo { +typedef struct SCtgRentSlot { SRWLatch lock; bool needSort; SArray *meta; // element is SDbVgVersion or SSTableMetaVersion -} SRentSlotInfo; +} SCtgRentSlot; -typedef struct SMetaRentMgmt { +typedef struct SCtgRentMgmt { int8_t type; uint16_t slotNum; uint16_t slotRIdx; int64_t lastReadMsec; - SRentSlotInfo *slots; -} SMetaRentMgmt; + SCtgRentSlot *slots; +} SCtgRentMgmt; typedef struct SCatalog { - uint64_t clusterId; - SDBVgroupCache dbCache; - STableMetaCache tableCache; - SMetaRentMgmt dbRent; - SMetaRentMgmt stableRent; + uint64_t clusterId; + SRWLatch dbLock; + SHashObj *dbCache; //key:dbname, value:SCtgDBCache + SCtgRentMgmt dbRent; + SCtgRentMgmt stbRent; } SCatalog; typedef struct SCtgApiStat { } SCtgApiStat; -typedef struct SCtgResourceStat { +typedef struct SCtgRuntimeStat { -} SCtgResourceStat; +} SCtgRuntimeStat; typedef struct SCtgCacheStat { @@ -103,11 +105,13 @@ typedef struct SCtgCacheStat { typedef struct SCatalogStat { SCtgApiStat api; - SCtgResourceStat resource; + SCtgRuntimeStat runtime; SCtgCacheStat cache; } SCatalogStat; typedef struct SCatalogMgmt { + bool exit; + SRWLatch lock; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; @@ -135,11 +139,8 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define ctgDebug(param, ...) qDebug("CTG:%p " param, pCatalog, __VA_ARGS__) #define ctgTrace(param, ...) qTrace("CTG:%p " param, pCatalog, __VA_ARGS__) -#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) -#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) -#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) - #define CTG_LOCK_DEBUG(...) do { if (gCTGDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) +#define CTG_CACHE_DEBUG(...) do { if (gCTGDebug.cacheDebug) { qDebug(__VA_ARGS__); } } while (0) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 @@ -175,6 +176,15 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); } \ } while (0) + +#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + +#define CTG_API_ENTER() do { CTG_LOCK(CTG_READ, &ctgMgmt.lock); if (atomic_load_8(&ctgMgmt.exit)) { CTG_RET(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) +#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_RET(__code); } while (0) + + #ifdef __cplusplus } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1b4d711aad..6ecff87a89 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -20,32 +20,265 @@ SCatalogMgmt ctgMgmt = {0}; -SCTGDebug gCTGDebug = {0}; +SCtgDebug gCTGDebug = {0}; + +int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) { + return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; +} + +int32_t ctgDbgGetStbNum(SCtgDBCache *dbCache) { + return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; +} + +int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) { + int32_t num = 0; + for (uint16_t i = 0; i < rent->slotNum; ++i) { + SCtgRentSlot *slot = &rent->slots[i]; + if (NULL == slot->meta) { + continue; + } + + num += taosArrayGetSize(slot->meta); + } + + return num; +} + +int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type) { + if (NULL == pCatalog || NULL == pCatalog->dbCache) { + return 0; + } + + switch (type) { + case CTG_DBG_DB_NUM: + return (int32_t)taosHashGetSize(pCatalog->dbCache); + case CTG_DBG_DB_RENT_NUM: + return ctgDbgGetRentNum(&pCatalog->dbRent); + case CTG_DBG_STB_RENT_NUM: + return ctgDbgGetRentNum(&pCatalog->stbRent); + default: + break; + } + + SCtgDBCache *dbCache = NULL; + int32_t num = 0; + void *pIter = taosHashIterate(pCatalog->dbCache, NULL); + while (pIter) { + dbCache = (SCtgDBCache *)pIter; + switch (type) { + case CTG_DBG_META_NUM: + num += ctgDbgGetTbMetaNum(dbCache); + break; + case CTG_DBG_STB_NUM: + num += ctgDbgGetStbNum(dbCache); + break; + default: + ctgError("invalid type:%d", type); + break; + } + pIter = taosHashIterate(pCatalog->dbCache, pIter); + } + + return num; +} -int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, bool *inCache) { - if (NULL == pCatalog->dbCache.cache) { +void ctgDbgShowDBCache(SHashObj *dbHash) { + if (NULL == dbHash) { + return; + } + + int32_t i = 0; + SCtgDBCache *dbCache = NULL; + void *pIter = taosHashIterate(dbHash, NULL); + while (pIter) { + char *dbFName = NULL; + size_t len = 0; + + dbCache = (SCtgDBCache *)pIter; + + taosHashGetKey(dbCache, (void **)&dbFName, &len); + + CTG_CACHE_DEBUG("** %dth db [%.*s][%"PRIx64"] **", i, (int32_t)len, dbFName, dbCache->dbId); + + pIter = taosHashIterate(dbHash, pIter); + } +} + + + + +void ctgDbgShowClusterCache(struct SCatalog* pCatalog) { + if (NULL == pCatalog) { + return; + } + + CTG_CACHE_DEBUG("## cluster %"PRIx64" %p cache Info ##", pCatalog->clusterId, pCatalog); + CTG_CACHE_DEBUG("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_META_NUM), + ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_STB_RENT_NUM)); + + ctgDbgShowDBCache(pCatalog->dbCache); +} + +int32_t ctgInitDBCache(struct SCatalog* pCatalog) { + if (NULL == pCatalog->dbCache) { + SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == cache) { + ctgError("taosHashInit %d failed", CTG_DEFAULT_CACHE_DB_NUMBER); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache, NULL, cache)) { + taosHashCleanup(cache); + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgInitTbMetaCache(struct SCatalog* pCatalog, SCtgDBCache *dbCache) { + if (NULL == dbCache->tbCache.metaCache) { + if (dbCache->deleted) { + ctgInfo("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + SHashObj *metaCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == metaCache) { + ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.metaCache, NULL, metaCache)) { + taosHashCleanup(metaCache); + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitStbCache(struct SCatalog* pCatalog, SCtgDBCache *dbCache) { + if (NULL == dbCache->tbCache.stbCache) { + if (dbCache->deleted) { + ctgInfo("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == cache) { + ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.stbCache, NULL, cache)) { + taosHashCleanup(cache); + } + } + + return TSDB_CODE_SUCCESS; +} + + + +void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { + if (NULL == mgmt->slots) { + return; + } + + for (int32_t i = 0; i < mgmt->slotNum; ++i) { + SCtgRentSlot *slot = &mgmt->slots[i]; + if (slot->meta) { + taosArrayDestroy(slot->meta); + slot->meta = NULL; + } + } + + tfree(mgmt->slots); +} + + +void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { + CTG_LOCK(CTG_WRITE, &cache->stbLock); + if (cache->stbCache) { + taosHashCleanup(cache->stbCache); + cache->stbCache = NULL; + } + CTG_UNLOCK(CTG_WRITE, &cache->stbLock); + + CTG_LOCK(CTG_WRITE, &cache->metaLock); + if (cache->metaCache) { + taosHashCleanup(cache->metaCache); + cache->metaCache = NULL; + } + CTG_UNLOCK(CTG_WRITE, &cache->metaLock); +} + +void ctgFreeDbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache) { + return; + } + + atomic_store_8(&dbCache->deleted, 1); + + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + if (dbCache->vgInfo) { + + if (dbCache->vgInfo->vgHash) { + taosHashCleanup(dbCache->vgInfo->vgHash); + dbCache->vgInfo->vgHash = NULL; + } + + tfree(dbCache->vgInfo); + } + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + + ctgFreeTableMetaCache(&dbCache->tbCache); +} + + +void ctgFreeHandle(struct SCatalog* pCatalog) { + ctgFreeMetaRent(&pCatalog->dbRent); + ctgFreeMetaRent(&pCatalog->stbRent); + if (pCatalog->dbCache) { + void *pIter = taosHashIterate(pCatalog->dbCache, NULL); + while (pIter) { + SCtgDBCache *dbCache = pIter; + + ctgFreeDbCache(dbCache); + + pIter = taosHashIterate(pCatalog->dbCache, pIter); + } + + taosHashCleanup(pCatalog->dbCache); + } + + free(pCatalog); +} + +int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbFName, SCtgDBCache **dbCache, bool *inCache) { + if (NULL == pCatalog->dbCache) { *inCache = false; - ctgWarn("empty db cache, dbName:%s", dbName); + ctgWarn("empty db cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } - SDBVgroupInfo *info = NULL; + SCtgDBCache *cache = NULL; while (true) { - info = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); + cache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - if (NULL == info) { + if (NULL == cache) { *inCache = false; - ctgWarn("not in db vgroup cache, dbName:%s", dbName); + ctgWarn("not in db vgroup cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &info->lock); - if (NULL == info->vgInfo) { - CTG_UNLOCK(CTG_READ, &info->lock); - taosHashRelease(pCatalog->dbCache.cache, info); - ctgWarn("db cache vgInfo is NULL, dbName:%s", dbName); + CTG_LOCK(CTG_READ, &cache->vgLock); + if (NULL == cache->vgInfo) { + CTG_UNLOCK(CTG_READ, &cache->vgLock); + taosHashRelease(pCatalog->dbCache, cache); + ctgWarn("db cache vgInfo is NULL, dbFName:%s", dbFName); continue; } @@ -53,10 +286,10 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S break; } - *dbInfo = info; + *dbCache = cache; *inCache = true; - ctgDebug("Got db vgroup from cache, dbName:%s", dbName); + ctgDebug("Got db vgroup from cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } @@ -98,48 +331,79 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp return TSDB_CODE_SUCCESS; } -int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, const char* tbFullName, int32_t *exist) { - if (NULL == pCatalog->tableCache.cache) { +int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, char* tbName, int32_t *exist) { + if (NULL == pCatalog->dbCache) { *exist = 0; - ctgWarn("empty tablemeta cache, tbName:%s", tbFullName); + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - size_t sz = 0; - STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName)); - - if (NULL == tbMeta) { + SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); + if (NULL == dbCache) { *exist = 0; - ctgDebug("tablemeta not in cache, tbName:%s", tbFullName); + ctgWarn("db not exist in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + + size_t sz = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == tbMeta) { + taosHashRelease(pCatalog->dbCache, dbCache); + + *exist = 0; + ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } *exist = 1; + + taosHashRelease(pCatalog->dbCache, dbCache); - ctgDebug("tablemeta is in cache, tbName:%s", tbFullName); + ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { - if (NULL == pCatalog->tableCache.cache) { + if (NULL == pCatalog->dbCache) { *exist = 0; - ctgWarn("empty tablemeta cache, tbName:%s", pTableName->tname); + ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, db); *pTableMeta = NULL; - size_t sz = 0; - STableMeta *tbMeta = taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)pTableMeta, &sz); + SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, db, strlen(db)); + if (NULL == dbCache) { + *exist = 0; + ctgWarn("no db cache, dbFName:%s, tbName:%s", db, pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + if (NULL == dbCache->tbCache.metaCache) { + *exist = 0; + taosHashRelease(pCatalog->dbCache, dbCache); + ctgWarn("empty tbmeta cache, dbFName:%s, tbName:%s", db, pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + size_t sz = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta *tbMeta = taosHashGetCloneExt(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), NULL, (void **)pTableMeta, &sz); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == *pTableMeta) { *exist = 0; - ctgDebug("tablemeta not in cache, tbName:%s", tbFullName); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", db, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -148,80 +412,94 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { - ctgDebug("Got tablemeta from cache, tbName:%s", tbFullName); - + taosHashRelease(pCatalog->dbCache, dbCache); + ctgDebug("Got tbmeta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, db, pTableName->tname); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - STableMeta **stbMeta = taosHashGet(pCatalog->tableCache.stableCache, &tbMeta->suid, sizeof(tbMeta->suid)); + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); - ctgError("stable not in stableCache, suid:%"PRIx64, tbMeta->suid); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgError("stable not in stbCache, suid:%"PRIx64, tbMeta->suid); tfree(*pTableMeta); *exist = 0; return TSDB_CODE_SUCCESS; } if ((*stbMeta)->suid != tbMeta->suid) { - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); tfree(*pTableMeta); - ctgError("stable suid in stableCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); *pTableMeta = realloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); ctgError("realloc size[%d] failed", metaSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgDebug("Got tablemeta from cache, tbName:%s", tbFullName); + taosHashRelease(pCatalog->dbCache, dbCache); + + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", db, pTableName->tname); return TSDB_CODE_SUCCESS; } int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableName, int32_t *tbType) { - if (NULL == pCatalog->tableCache.cache) { - ctgWarn("empty tablemeta cache, tbName:%s", pTableName->tname); + if (NULL == pCatalog->dbCache) { + ctgWarn("empty db cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char dbName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbName); - size_t sz = 0; - STableMeta *pTableMeta = NULL; - - taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)&pTableMeta, &sz); + SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == dbCache) { + ctgInfo("db not in cache, dbFName:%s", dbName); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == pTableMeta) { - ctgWarn("tablemeta not in cache, tbName:%s", tbFullName); - + ctgWarn("tbmeta not in cache, dbFName:%s, tbName:%s", dbName, pTableName->tname); + taosHashRelease(pCatalog->dbCache, dbCache); + return TSDB_CODE_SUCCESS; } - *tbType = pTableMeta->tableType; + *tbType = atomic_load_8(&pTableMeta->tableType); - ctgDebug("Got tabletype from cache, tbName:%s, type:%d", tbFullName, *tbType); + taosHashRelease(dbCache->tbCache.metaCache, dbCache); + taosHashRelease(pCatalog->dbCache, dbCache); + + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbName, pTableName->tname, *tbType); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char* tbFullName, STableMetaOutput* output) { - SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName}; +int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { + SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - ctgDebug("try to get table meta from mnode, tbName:%s", tbFullName); + ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)](&bInput, &msg, 0, &msgLen); if (code) { @@ -242,30 +520,30 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransport if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { SET_META_TYPE_NULL(output->metaType); - ctgDebug("stablemeta not exist in mnode, tbName:%s", tbFullName); + ctgDebug("stablemeta not exist in mnode, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - ctgError("error rsp for stablemeta from mnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tbFullName); + ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tbName); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process mnode stablemeta rsp failed, code:%x, tbName:%s", code, tbFullName); + ctgError("Process mnode stablemeta rsp failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tbName); CTG_ERR_RET(code); } - ctgDebug("Got table meta from mnode, tbName:%s", tbFullName); + ctgDebug("Got table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); - return ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, tbFullName, output); + return ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); } int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { @@ -273,18 +551,18 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - char dbFullName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFullName); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); - ctgDebug("try to get table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); + ctgDebug("try to get table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)tNameGetTableName(pTableName)}; + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; char *msg = NULL; int32_t msgLen = 0; int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen); if (code) { - ctgError("Build vnode tablemeta msg failed, code:%x, tbName:%s", code, tNameGetTableName(pTableName)); + ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tNameGetTableName(pTableName)); CTG_ERR_RET(code); } @@ -300,21 +578,21 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { SET_META_TYPE_NULL(output->metaType); - ctgDebug("tablemeta not exist in vnode, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("tablemeta not exist in vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } - ctgError("error rsp for table meta from vnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tNameGetTableName(pTableName)); + ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tNameGetTableName(pTableName)); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%s, tbName:%s", tstrerror(code), tNameGetTableName(pTableName)); + ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName, tNameGetTableName(pTableName)); CTG_ERR_RET(code); } - ctgDebug("Got table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); + ctgDebug("Got table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } @@ -334,7 +612,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { @@ -342,7 +620,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; @@ -351,7 +629,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - pIter = taosHashIterate(dbInfo->vgInfo, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } @@ -374,7 +652,7 @@ _return: int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -386,40 +664,40 @@ int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbIn tableNameHashFp fp = NULL; SVgroupInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); char tbFullName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFullName); uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); - void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgInfo, pIter); + taosHashCancelIterate(dbInfo->vgHash, pIter); break; } - pIter = taosHashIterate(dbInfo->vgInfo, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgInfo)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } *pVgroup = *vgInfo; -_return: CTG_RET(code); } +#if 1 int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { return 1; } else { return 0; @@ -435,19 +713,41 @@ int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { return 0; } } +#else +int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { + if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + return -1; + } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + return 1; + } else { + return 0; + } +} -int32_t ctgMetaRentInit(SMetaRentMgmt *mgmt, uint32_t rentSec, int8_t type) { +int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { + if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { + return -1; + } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { + return 1; + } else { + return 0; + } +} + +#endif + +int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; mgmt->type = type; - size_t msgSize = sizeof(SRentSlotInfo) * mgmt->slotNum; + size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; mgmt->slots = calloc(1, msgSize); if (NULL == mgmt->slots) { qError("calloc %d failed", (int32_t)msgSize); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); @@ -456,10 +756,10 @@ int32_t ctgMetaRentInit(SMetaRentMgmt *mgmt, uint32_t rentSec, int8_t type) { } -int32_t ctgMetaRentAdd(SMetaRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { +int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { int16_t widx = abs(id % mgmt->slotNum); - SRentSlotInfo *slot = &mgmt->slots[widx]; + SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); @@ -486,10 +786,10 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentUpdate(SMetaRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t compare) { +int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t compare) { int16_t widx = abs(id % mgmt->slotNum); - SRentSlotInfo *slot = &mgmt->slots[widx]; + SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); @@ -499,14 +799,15 @@ int32_t ctgMetaRentUpdate(SMetaRentMgmt *mgmt, void *meta, int64_t id, int32_t s } if (slot->needSort) { + qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); taosArraySort(slot->meta, compare); slot->needSort = false; - qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type); + qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); } void *orig = taosArraySearch(slot->meta, &id, compare, TD_EQ); if (NULL == orig) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -526,10 +827,10 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentRemove(SMetaRentMgmt *mgmt, int64_t id, __compar_fn_t compare) { +int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t compare) { int16_t widx = abs(id % mgmt->slotNum); - SRentSlotInfo *slot = &mgmt->slots[widx]; + SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); @@ -562,14 +863,14 @@ _return: } -int32_t ctgMetaRentGetImpl(SMetaRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { +int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); if (ridx >= mgmt->slotNum) { ridx %= mgmt->slotNum; atomic_store_16(&mgmt->slotRIdx, ridx); } - SRentSlotInfo *slot = &mgmt->slots[ridx]; + SCtgRentSlot *slot = &mgmt->slots[ridx]; int32_t code = 0; CTG_LOCK(CTG_READ, &slot->lock); @@ -608,7 +909,7 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentGet(SMetaRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { +int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { while (true) { int64_t msec = taosGetTimestampMs(); int64_t lsec = atomic_load_64(&mgmt->lastReadMsec); @@ -631,122 +932,294 @@ int32_t ctgMetaRentGet(SMetaRentMgmt *mgmt, void **res, uint32_t *num, int32_t s return TSDB_CODE_SUCCESS; } +int32_t ctgAddDBCache(struct SCatalog *pCatalog, const char *dbFName, SCtgDBCache *dbCache) { + int32_t code = 0; + + code = taosHashPut(pCatalog->dbCache, dbFName, strlen(dbFName), dbCache, sizeof(SCtgDBCache)); + if (code) { + if (HASH_NODE_EXIST(code)) { + ctgDebug("db already in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + SDbVgVersion vgVersion = {.dbId = dbCache->dbId, .vgVersion = dbCache->vgInfo ? dbCache->vgInfo->vgVersion : -1}; + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + + ctgDebug("dbCache added, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbCache->dbId); + + CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion))); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgFreeDbCache(dbCache); + + CTG_RET(code); +} + + +void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache) { + CTG_LOCK(CTG_WRITE, &cache->stbLock); + if (cache->stbCache) { + void *pIter = taosHashIterate(cache->stbCache, NULL); + while (pIter) { + uint64_t *suid = NULL; + taosHashGetKey(pIter, (void **)&suid, NULL); + + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCatalog->stbRent, *suid, ctgSTableVersionCompare)) { + ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + } + + pIter = taosHashIterate(cache->stbCache, pIter); + } + } + CTG_UNLOCK(CTG_WRITE, &cache->stbLock); + + ctgFreeTableMetaCache(cache); +} + + +int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, SCtgDBCache *dbCache, const char* dbFName) { + if (taosHashRemove(pCatalog->dbCache, dbFName, strlen(dbFName))) { + ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + atomic_store_8(&dbCache->deleted, 1); + + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + if (dbCache->vgInfo) { + ctgInfo("cleanup db vgInfo, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + if (dbCache->vgInfo->vgHash) { + taosHashCleanup(dbCache->vgInfo->vgHash); + } + + tfree(dbCache->vgInfo); + } + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + + ctgRemoveAndFreeTableMeta(pCatalog, &dbCache->tbCache); + + ctgInfo("db removed from cache, dbFName:%s, uid:%"PRIx64, dbFName, dbCache->dbId); + + CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->dbRent, dbCache->dbId, ctgDbVgVersionCompare)); + + ctgDebug("db removed from rent, dbFName:%s, uid:%"PRIx64, dbFName, dbCache->dbId); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgAcquireDBCache(struct SCatalog* pCatalog, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + + CTG_LOCK(CTG_WRITE, &pCatalog->dbLock); + + while (true) { + dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); + if (dbCache) { + // TODO OPEN IT +#if 0 + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#else + if (0 == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbId && (dbCache->dbId == 0)) { + dbCache->dbId = dbId; + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#endif + code = ctgValidateAndRemoveDb(pCatalog, dbCache, dbFName); + taosHashRelease(pCatalog->dbCache, dbCache); + dbCache = NULL; + if (code) { + if (TSDB_CODE_CTG_DB_DROPPED == code) { + continue; + } + + CTG_ERR_JRET(code); + } + } + + SCtgDBCache newDBCache = {0}; + newDBCache.dbId = dbId; + + CTG_ERR_JRET(ctgAddDBCache(pCatalog, dbFName, &newDBCache)); + } + +_return: + + if (dbCache) { + taosHashRelease(pCatalog->dbCache, dbCache); + } + + CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); + + CTG_RET(code); +} + + + +int32_t ctgUpdateTbMetaImpl(struct SCatalog *pCatalog, SCtgTbMetaCache *tbCache, char *dbFName, char *tbName, STableMeta *meta, int32_t metaSize) { + CTG_LOCK(CTG_READ, &tbCache->metaLock); + if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgUpdateStbMetaImpl(struct SCatalog *pCatalog, SCtgTbMetaCache *tbCache, char *dbFName, char *tbName, STableMeta *meta, int32_t metaSize) { + bool newAdded = false; + int32_t code = 0; + SSTableMetaVersion metaRent = {.suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); + + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + + CTG_LOCK(CTG_READ, &tbCache->metaLock); + STableMeta *orig = taosHashAcquire(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { + if (orig->suid != meta->suid) { + if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + ctgError("stb not exist in stbCache, db:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } + + ctgMetaRentRemove(&pCatalog->stbRent, orig->suid, ctgSTableVersionCompare); + } + + taosHashRelease(tbCache->metaCache, orig); + } + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, tbCache, dbFName, tbName, meta, metaSize)); + + CTG_LOCK(CTG_READ, &tbCache->metaLock); + STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (taosHashPutExt(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES, &newAdded) != 0) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, meta->suid); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + ctgDebug("update stable to cache, suid:%"PRIx64, meta->suid); + + if (newAdded) { + CTG_ERR_RET(ctgMetaRentAdd(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + } else { + CTG_ERR_RET(ctgMetaRentUpdate(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion), ctgSTableVersionCompare)); + } + + return TSDB_CODE_SUCCESS; + +_return: + + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + CTG_RET(code); +} int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { int32_t code = 0; + SCtgDBCache *dbCache = NULL; if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid table meta got from meta rsp, tbName:%s", output->tbFname); + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (NULL == pCatalog->tableCache.cache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == cache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } + CTG_ERR_RET(ctgInitDBCache(pCatalog)); - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.cache, NULL, cache)) { - taosHashCleanup(cache); - } - } + CTG_ERR_JRET(ctgAcquireDBCache(pCatalog, output->dbFName, output->dbId, &dbCache)); - if (NULL == pCatalog->tableCache.stableCache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == cache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.stableCache, NULL, cache)) { - taosHashCleanup(cache); - } - } + CTG_ERR_JRET(ctgInitTbMetaCache(pCatalog, dbCache)); + CTG_ERR_JRET(ctgInitStbCache(pCatalog, dbCache)); if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { - ctgError("taosHashPut ctablemeta to cache failed, ctbName:%s", output->ctbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - ctgDebug("update child tablemeta to cache, tbName:%s", output->ctbFname); + CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); } if (CTG_IS_META_CTABLE(output->metaType)) { - return TSDB_CODE_SUCCESS; + goto _return; } if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags); if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { - bool newAdded = false; - SSTableMetaVersion metaRent = {.suid = output->tbMeta->suid, .sversion = output->tbMeta->sversion, .tversion = output->tbMeta->tversion}; - - CTG_LOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); - if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { - CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); - ctgError("taosHashPut tablemeta to cache failed, tbName:%s", output->tbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname)); - if (taosHashPutExt(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &tbMeta, POINTER_BYTES, &newAdded) != 0) { - CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); - ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, output->tbMeta->suid); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); - - ctgDebug("update stable to cache, suid:%"PRIx64, output->tbMeta->suid); - - if (newAdded) { - CTG_ERR_RET(ctgMetaRentAdd(&pCatalog->stableRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); - } else { - CTG_ERR_RET(ctgMetaRentUpdate(&pCatalog->stableRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion), ctgSTableVersionCompare)); - } + CTG_ERR_JRET(ctgUpdateStbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->tbName, output->tbMeta, tbSize)); } else { - if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { - ctgError("taosHashPut tablemeta to cache failed, tbName:%s", output->tbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } + CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->tbName, output->tbMeta, tbSize)); } - ctgDebug("update tablemeta to cache, tbName:%s", output->tbFname); +_return: + + if (dbCache) { + taosHashRelease(pCatalog->dbCache, dbCache); + CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); + } CTG_RET(code); } -int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, bool forceUpdate, SDBVgroupInfo** dbInfo) { +int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache) { bool inCache = false; if (!forceUpdate) { - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbFName, dbCache, &inCache)); if (inCache) { return TSDB_CODE_SUCCESS; } - ctgDebug("failed to get DB vgroupInfo from cache, dbName:%s, load it from mnode, update:%d", dbName, forceUpdate); + ctgDebug("failed to get DB vgroupInfo from cache, dbName:%s, load it from mnode, update:%d", dbFName, forceUpdate); } SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; - tstrncpy(input.db, dbName, tListLen(input.db)); + tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; while (true) { CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); - CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, &DbOut.dbVgroup)); - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); + CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbFName, DbOut.dbId, DbOut.dbVgroup)); + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbFName, dbCache, &inCache)); if (!inCache) { - ctgWarn("can't get db vgroup from cache, will retry, db:%s", dbName); + ctgWarn("can't get db vgroup from cache, will retry, db:%s", dbFName); continue; } @@ -756,67 +1229,36 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm return TSDB_CODE_SUCCESS; } - -int32_t ctgValidateAndFreeDbInfo(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { - SDBVgroupInfo *oldInfo = (SDBVgroupInfo *)taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (oldInfo) { - CTG_LOCK(CTG_WRITE, &oldInfo->lock); - if (dbInfo->vgVersion <= oldInfo->vgVersion) { - ctgInfo("db vgVersion is not new, db:%s, vgVersion:%d, current:%d", dbName, dbInfo->vgVersion, oldInfo->vgVersion); - CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); - taosHashRelease(pCatalog->dbCache.cache, oldInfo); - - return TSDB_CODE_SUCCESS; - } - - if (oldInfo->vgInfo) { - ctgInfo("cleanup db vgInfo, db:%s", dbName); - taosHashCleanup(oldInfo->vgInfo); - oldInfo->vgInfo = NULL; - } - - CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); - - taosHashRelease(pCatalog->dbCache.cache, oldInfo); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgValidateAndRemoveDbInfo(struct SCatalog* pCatalog, SDbVgVersion* target, bool *removed) { +int32_t ctgValidateAndRemoveStbMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid, bool *removed) { *removed = false; - SDBVgroupInfo *info = (SDBVgroupInfo *)taosHashAcquire(pCatalog->dbCache.cache, target->dbName, strlen(target->dbName)); - if (NULL == info) { - ctgInfo("db not exist in dbCache, may be removed, db:%s", target->dbName); + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == dbCache) { + ctgInfo("db not exist in dbCache, may be removed, db:%s", dbName); return TSDB_CODE_SUCCESS; } - - CTG_LOCK(CTG_WRITE, &info->lock); - - if (info->dbId != target->dbId) { - ctgInfo("db id already updated, db:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, target->dbName, info->dbId, target->dbId); - CTG_UNLOCK(CTG_WRITE, &info->lock); - taosHashRelease(pCatalog->dbCache.cache, info); - return TSDB_CODE_SUCCESS; - } - - if (info->vgInfo) { - ctgInfo("cleanup db vgInfo, db:%s", target->dbName); - taosHashCleanup(info->vgInfo); - info->vgInfo = NULL; - } - if (taosHashRemove(pCatalog->dbCache.cache, target->dbName, strlen(target->dbName))) { - ctgError("taosHashRemove from dbCache failed, db:%s", target->dbName); - CTG_UNLOCK(CTG_WRITE, &info->lock); - taosHashRelease(pCatalog->dbCache.cache, info); + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashRemove(dbCache->tbCache.stbCache, &suid, sizeof(suid))) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgInfo("stb not exist in stbCache, may be removed, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, stbName, strlen(stbName))) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgError("stb not exist in cache, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - CTG_UNLOCK(CTG_WRITE, &info->lock); - - taosHashRelease(pCatalog->dbCache.cache, info); + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + + taosHashRelease(pCatalog->dbCache, dbCache); *removed = true; @@ -824,6 +1266,7 @@ int32_t ctgValidateAndRemoveDbInfo(struct SCatalog* pCatalog, SDbVgVersion* targ } + int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); @@ -839,7 +1282,7 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con STableMetaOutput *output = &voutput; if (CTG_IS_STABLE(isSTable)) { - ctgDebug("will renew table meta, supposed to be stable, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("will renew tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput)); @@ -850,15 +1293,15 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con output = &moutput; } } else { - ctgDebug("will renew table meta, not supposed to be stable, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); + ctgDebug("will renew tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); // if get from vnode failed or no table meta, will not try mnode CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); if (CTG_IS_META_TABLE(voutput.metaType) && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) { - ctgDebug("will continue to renew table meta since got stable, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); + ctgDebug("will continue to renew tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); voutput.metaType = moutput.metaType; @@ -867,9 +1310,9 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con moutput.tbMeta = NULL; } else if (CTG_IS_META_BOTH(voutput.metaType)) { int32_t exist = 0; - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCatalog, voutput.tbFname, &exist)); + CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCatalog, voutput.dbFName, voutput.tbName, &exist)); if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(voutput.metaType); @@ -934,71 +1377,15 @@ int32_t ctgGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMg return TSDB_CODE_SUCCESS; } -void ctgFreeMetaRent(SMetaRentMgmt *mgmt) { - if (NULL == mgmt->slots) { - return; - } - - for (int32_t i = 0; i < mgmt->slotNum; ++i) { - SRentSlotInfo *slot = &mgmt->slots[i]; - if (slot->meta) { - taosArrayDestroy(slot->meta); - slot->meta = NULL; - } - } - - tfree(mgmt->slots); -} - -void ctgFreeDbCache(SDBVgroupCache *db) { - if (NULL == db->cache) { - return; - } - - SDBVgroupInfo *dbInfo = NULL; - void *pIter = taosHashIterate(db->cache, NULL); - while (pIter) { - dbInfo = pIter; - - if (dbInfo->vgInfo) { - taosHashCleanup(dbInfo->vgInfo); - dbInfo->vgInfo = NULL; - } - - pIter = taosHashIterate(db->cache, pIter); - } - - taosHashCleanup(db->cache); - db->cache = NULL; -} - -void ctgFreeTableMetaCache(STableMetaCache *table) { - if (table->stableCache) { - taosHashCleanup(table->stableCache); - table->stableCache = NULL; - } - - if (table->cache) { - taosHashCleanup(table->cache); - table->cache = NULL; - } -} - -void ctgFreeHandle(struct SCatalog* pCatalog) { - ctgFreeMetaRent(&pCatalog->dbRent); - ctgFreeMetaRent(&pCatalog->stableRent); - ctgFreeDbCache(&pCatalog->dbCache); - ctgFreeTableMetaCache(&pCatalog->tableCache); - - free(pCatalog); -} int32_t catalogInit(SCatalogCfg *cfg) { if (ctgMgmt.pCluster) { - qError("catalog already init"); + qError("catalog already initialized"); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } + atomic_store_8(&ctgMgmt.exit, false); + if (cfg) { memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg)); @@ -1014,14 +1401,14 @@ int32_t catalogInit(SCatalogCfg *cfg) { ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; } - if (ctgMgmt.cfg.stableRentSec == 0) { - ctgMgmt.cfg.stableRentSec = CTG_DEFAULT_RENT_SECOND; + if (ctgMgmt.cfg.stbRentSec == 0) { + ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } } else { ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; - ctgMgmt.cfg.stableRentSec = CTG_DEFAULT_RENT_SECOND; + ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); @@ -1030,7 +1417,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stableRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stableRentSec); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } @@ -1041,7 +1428,7 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { } if (NULL == ctgMgmt.pCluster) { - qError("cluster cache are not ready, clusterId:%"PRIx64, clusterId); + qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } @@ -1066,7 +1453,7 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { clusterCtg->clusterId = clusterId; CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, ctgMgmt.cfg.dbRentSec, CTG_RENT_DB)); - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stableRent, ctgMgmt.cfg.stableRentSec, CTG_RENT_STABLE)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, ctgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); code = taosHashPut(ctgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { @@ -1117,55 +1504,73 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == pCatalog->dbCache.cache) { + CTG_API_ENTER(); + + if (NULL == pCatalog->dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; ctgInfo("empty db cache, dbName:%s", dbName); - return TSDB_CODE_SUCCESS; + CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SDBVgroupInfo * dbInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (NULL == dbInfo) { + SCtgDBCache *db = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == db) { *version = CTG_DEFAULT_INVALID_VERSION; ctgInfo("db not in cache, dbName:%s", dbName); - return TSDB_CODE_SUCCESS; + CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - *version = dbInfo->vgVersion; - taosHashRelease(pCatalog->dbCache.cache, dbInfo); + CTG_LOCK(CTG_READ, &db->vgLock); + + if (NULL == db->vgInfo) { + CTG_UNLOCK(CTG_READ, &db->vgLock); + + *version = CTG_DEFAULT_INVALID_VERSION; + ctgInfo("db not in cache, dbName:%s", dbName); + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } + + *version = db->vgInfo->vgVersion; + CTG_UNLOCK(CTG_READ, &db->vgLock); + + taosHashRelease(pCatalog->dbCache, db); ctgDebug("Got db vgVersion from cache, dbName:%s, vgVersion:%d", dbName, *version); - return TSDB_CODE_SUCCESS; + CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, bool forceUpdate, SArray** vgroupList) { - if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SArray** vgroupList) { + if (NULL == pCatalog || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - SDBVgroupInfo* db = NULL; + CTG_API_ENTER(); + + SCtgDBCache* dbCache = NULL; SVgroupInfo *vgInfo = NULL; int32_t code = 0; SArray *vgList = NULL; - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbName, forceUpdate, &db)); + CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbFName, forceUpdate, &dbCache)); - vgList = taosArrayInit(taosHashGetSize(db->vgInfo), sizeof(SVgroupInfo)); + int32_t vgNum = (int32_t)taosHashGetSize(dbCache->vgInfo->vgHash); + vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { - ctgError("taosArrayInit %d failed", taosHashGetSize(db->vgInfo)); + ctgError("taosArrayInit %d failed", vgNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - void *pIter = taosHashIterate(db->vgInfo, NULL); + void *pIter = taosHashIterate(dbCache->vgInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; if (NULL == taosArrayPush(vgList, vgInfo)) { ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); + taosHashCancelIterate(dbCache->vgInfo->vgHash, pIter); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - pIter = taosHashIterate(db->vgInfo, pIter); + pIter = taosHashIterate(dbCache->vgInfo->vgHash, pIter); vgInfo = NULL; } @@ -1173,9 +1578,10 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* vgList = NULL; _return: - if (db) { - CTG_UNLOCK(CTG_READ, &db->lock); - taosHashRelease(pCatalog->dbCache.cache, db); + + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); } if (vgList) { @@ -1183,125 +1589,214 @@ _return: vgList = NULL; } - CTG_RET(code); + CTG_API_LEAVE(code); } -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { +int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbFName, uint64_t dbId, SDBVgroupInfo* dbInfo) { int32_t code = 0; + + CTG_API_ENTER(); - if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) { + if (NULL == pCatalog || NULL == dbFName || NULL == dbInfo) { CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == dbInfo->vgInfo || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgInfo) <= 0) { - ctgError("invalid db vgInfo, dbName:%s, vgInfo:%p, vgVersion:%d", dbName, dbInfo->vgInfo, dbInfo->vgVersion); + if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - if (NULL == pCatalog->dbCache.cache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == cache) { - ctgError("taosHashInit %d failed", CTG_DEFAULT_CACHE_DB_NUMBER); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache.cache, NULL, cache)) { - taosHashCleanup(cache); - } - } else { - CTG_ERR_JRET(ctgValidateAndFreeDbInfo(pCatalog, dbName, dbInfo)); - } + CTG_ERR_JRET(ctgInitDBCache(pCatalog)); bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion}; + + SCtgDBCache *dbCache = NULL; + CTG_ERR_JRET(ctgAcquireDBCache(pCatalog, dbFName, dbId, &dbCache)); - dbInfo->lock = 0; - if (taosHashPutExt(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo), &newAdded) != 0) { - ctgError("taosHashPutExt db vgroup to cache failed, db:%s", dbName); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + if (dbCache->deleted) { + ctgInfo("db is dropping, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + CTG_ERR_JRET(TSDB_CODE_CTG_DB_DROPPED); } - - dbInfo->vgInfo = NULL; - - SDbVgVersion vgVersion = {.dbId = dbInfo->dbId, .vgVersion = dbInfo->vgVersion}; - strncpy(vgVersion.dbName, dbName, sizeof(vgVersion.dbName)); - if (newAdded) { - CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, dbInfo->dbId, sizeof(SDbVgVersion))); + if (NULL == dbCache->vgInfo) { + dbCache->vgInfo = dbInfo; } else { - CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->dbRent, &vgVersion, dbInfo->dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); - } - - ctgDebug("dbName:%s vgroup updated, vgVersion:%d", dbName, dbInfo->vgVersion); + if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { + ctgInfo("db vgVersion is old, dbFName:%s, vgVersion:%d, current:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + + goto _return; + } + + if (dbCache->vgInfo->vgHash) { + ctgDebug("cleanup db vgHash, dbFName:%s", dbFName); + taosHashCleanup(dbCache->vgInfo->vgHash); + dbCache->vgInfo->vgHash = NULL; + } + tfree(dbCache->vgInfo); + dbCache->vgInfo = dbInfo; + } + + dbInfo = NULL; + + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); + + ctgDebug("dbCache updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); _return: - if (dbInfo && dbInfo->vgInfo) { - taosHashCleanup(dbInfo->vgInfo); - dbInfo->vgInfo = NULL; + if (dbCache) { + CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); + } + + if (dbInfo) { + taosHashCleanup(dbInfo->vgHash); + dbInfo->vgHash = NULL; + tfree(dbInfo); } - CTG_RET(code); + CTG_API_LEAVE(code); } -int32_t catalogRemoveDBVgroup(struct SCatalog* pCatalog, SDbVgVersion* dbInfo) { +int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbFName, uint64_t dbId) { int32_t code = 0; - bool removed = false; - if (NULL == pCatalog || NULL == dbInfo) { + if (NULL == pCatalog || NULL == dbFName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == pCatalog->dbCache.cache) { - return TSDB_CODE_SUCCESS; - } - - CTG_ERR_RET(ctgValidateAndRemoveDbInfo(pCatalog, dbInfo, &removed)); - if (!removed) { - return TSDB_CODE_SUCCESS; - } - - ctgInfo("db removed from cache, db:%s", dbInfo->dbName); + CTG_API_ENTER(); - CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->dbRent, dbInfo->dbId, ctgDbVgVersionCompare)); + if (NULL == pCatalog->dbCache) { + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } + + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); + if (NULL == dbCache) { + ctgInfo("db not exist in dbCache, may be removed, dbFName:%s", dbFName); + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } + + if (dbCache->dbId != dbId) { + ctgInfo("db id already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, dbFName, dbCache->dbId, dbId); + taosHashRelease(pCatalog->dbCache, dbCache); + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } - ctgDebug("db removed from rent, db:%s", dbInfo->dbName); + CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbCache, dbFName)); + +_return: - CTG_RET(code); + taosHashRelease(pCatalog->dbCache, dbCache); + + CTG_API_LEAVE(code); +} + +int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid) { + int32_t code = 0; + bool removed = false; + + if (NULL == pCatalog || NULL == dbName || NULL == stbName) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_ENTER(); + + if (NULL == pCatalog->dbCache) { + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } + + CTG_ERR_RET(ctgValidateAndRemoveStbMeta(pCatalog, dbName, stbName, suid, &removed)); + if (!removed) { + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } + + ctgInfo("stb removed from cache, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); + + CTG_ERR_JRET(ctgMetaRentRemove(&pCatalog->stbRent, suid, ctgSTableVersionCompare)); + + ctgDebug("stb removed from rent, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); + +_return: + + CTG_API_LEAVE(code); } int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { - return ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1); + CTG_API_ENTER(); + + CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); } int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { - return ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1); + CTG_API_ENTER(); + + CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); } +int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg) { + STableMetaOutput output = {0}; + int32_t code = 0; + + CTG_API_ENTER(); + + strcpy(output.dbFName, rspMsg->dbFName); + strcpy(output.tbName, rspMsg->tbName); + + SET_META_TYPE_TABLE(output.metaType); + + CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output.tbMeta)); + + CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output)); + +_return: + + tfree(output.tbMeta); + + CTG_API_LEAVE(code); +} + + int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - return ctgRenewTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, isSTable); + CTG_API_ENTER(); + + CTG_API_LEAVE(ctgRenewTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, isSTable)); } int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { - return ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable); + CTG_API_ENTER(); + + CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); } int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgroupList) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } + + CTG_API_ENTER(); STableMeta *tbMeta = NULL; int32_t code = 0; SVgroupInfo vgroupInfo = {0}; - SDBVgroupInfo* dbVgroup = NULL; + SCtgDBCache* dbCache = NULL; SArray *vgList = NULL; *pVgroupList = NULL; @@ -1310,9 +1805,9 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbVgroup)); + CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbCache)); - // REMOEV THIS .... + // TODO REMOEV THIS .... if (0 == tbMeta->vgId) { SVgroupInfo vgroup = {0}; @@ -1320,13 +1815,13 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S tbMeta->vgId = vgroup.vgId; } - // REMOVE THIS .... + // TODO REMOVE THIS .... if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbVgroup, pVgroupList)); + CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbCache->vgInfo, pVgroupList)); } else { int32_t vgId = tbMeta->vgId; - if (NULL == taosHashGetClone(dbVgroup->vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { + if (NULL == taosHashGetClone(dbCache->vgInfo->vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1349,9 +1844,9 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S _return: tfree(tbMeta); - if (dbVgroup) { - CTG_UNLOCK(CTG_READ, &dbVgroup->lock); - taosHashRelease(pCatalog->dbCache.cache, dbVgroup); + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); } if (vgList) { @@ -1359,29 +1854,31 @@ _return: vgList = NULL; } - CTG_RET(code); + CTG_API_LEAVE(code); } int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { - SDBVgroupInfo* dbInfo = NULL; + SCtgDBCache* dbCache = NULL; int32_t code = 0; + CTG_API_ENTER(); + char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_RET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbInfo)); + CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbCache)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCatalog, dbInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCatalog, dbCache->vgInfo, pTableName, pVgroup)); _return: - if (dbInfo) { - CTG_UNLOCK(CTG_READ, &dbInfo->lock); - taosHashRelease(pCatalog->dbCache.cache, dbInfo); + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); } - CTG_RET(code); + CTG_API_LEAVE(code); } @@ -1390,19 +1887,22 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const S CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } + CTG_API_ENTER(); + int32_t code = 0; + pRsp->pTableMeta = NULL; if (pReq->pTableName) { int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName); if (tbNum <= 0) { ctgError("empty table name list, tbNum:%d", tbNum); - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES); if (NULL == pRsp->pTableMeta) { ctgError("taosArrayInit %d failed", tbNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } for (int32_t i = 0; i < tbNum; ++i) { @@ -1419,9 +1919,10 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const S } } - return TSDB_CODE_SUCCESS; + CTG_API_LEAVE(TSDB_CODE_SUCCESS); _return: + if (pRsp->pTableMeta) { int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); for (int32_t i = 0; i < aSize; ++i) { @@ -1433,7 +1934,7 @@ _return: pRsp->pTableMeta = NULL; } - CTG_RET(code); + CTG_API_LEAVE(code); } int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { @@ -1441,9 +1942,11 @@ int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } + CTG_API_ENTER(); + //TODO - return TSDB_CODE_SUCCESS; + CTG_API_LEAVE(TSDB_CODE_SUCCESS); } int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num) { @@ -1451,7 +1954,9 @@ int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion * CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_RET(ctgMetaRentGet(&pCatalog->stableRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + CTG_API_ENTER(); + + CTG_API_LEAVE(ctgMetaRentGet(&pCatalog->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); } int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num) { @@ -1459,15 +1964,21 @@ int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_RET(ctgMetaRentGet(&pCatalog->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); + CTG_API_ENTER(); + + CTG_API_LEAVE(ctgMetaRentGet(&pCatalog->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } void catalogDestroy(void) { - if (NULL == ctgMgmt.pCluster) { + if (NULL == ctgMgmt.pCluster || atomic_load_8(&ctgMgmt.exit)) { return; } + atomic_store_8(&ctgMgmt.exit, true); + + CTG_LOCK(CTG_WRITE, &ctgMgmt.lock); + SCatalog *pCatalog = NULL; void *pIter = taosHashIterate(ctgMgmt.pCluster, NULL); while (pIter) { @@ -1483,6 +1994,8 @@ void catalogDestroy(void) { taosHashCleanup(ctgMgmt.pCluster); ctgMgmt.pCluster = NULL; + CTG_UNLOCK(CTG_WRITE, &ctgMgmt.lock); + qInfo("catalog destroyed"); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index c6e5e359bf..d0f98e3c2a 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -39,6 +39,7 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, int32_t *exist); extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output); +extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); void ctgTestSetPrepareTableMeta(); void ctgTestSetPrepareCTableMeta(); @@ -49,7 +50,7 @@ bool ctgTestStop = false; bool ctgTestEnableSleep = false; bool ctgTestDeadLoop = false; int32_t ctgTestPrintNum = 200000; -int32_t ctgTestMTRunSec = 30; +int32_t ctgTestMTRunSec = 5; int32_t ctgTestCurrentVgVersion = 0; int32_t ctgTestVgVersion = 1; @@ -107,6 +108,7 @@ void ctgTestInitLogFile() { const int32_t maxLogFileNum = 10; tsAsyncLog = 0; + qDebugFlag = 159; char temp[128] = {0}; sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); @@ -128,15 +130,14 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(sn.dbname, "db1"); strcpy(sn.tname, ctgTestSTablename); - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&cn, tbFullName); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&cn, db); + strcpy(output->dbFName, db); SET_META_TYPE_BOTH_TABLE(output->metaType); - strcpy(output->ctbFname, tbFullName); - - tNameExtractFullName(&cn, tbFullName); - strcpy(output->tbFname, tbFullName); + strcpy(output->ctbName, cn.tname); + strcpy(output->tbName, sn.tname); output->ctbMeta.vgId = 9; output->ctbMeta.tableType = TSDB_CHILD_TABLE; @@ -175,18 +176,18 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(s->name, "tag1s"); } -void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { +void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; + SDBVgroupInfo *dbVgroup = (SDBVgroupInfo *)calloc(1, sizeof(SDBVgroupInfo)); dbVgroup->vgVersion = vgVersion++; ctgTestCurrentVgVersion = dbVgroup->vgVersion; dbVgroup->hashMethod = 0; - dbVgroup->dbId = ctgTestDbId; - dbVgroup->vgInfo = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + dbVgroup->vgHash = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion); uint32_t hashUnit = UINT32_MAX / vgNum; @@ -203,10 +204,51 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { addr->port = htons(n + 22); } - taosHashPut(dbVgroup->vgInfo, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); + taosHashPut(dbVgroup->vgHash, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); } + + *pdbVgroup = dbVgroup; } + +void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { + strcpy(rspMsg->dbFName, ctgTestDbname); + sprintf(rspMsg->tbName, "%s", ctgTestSTablename); + sprintf(rspMsg->stbName, "%s", ctgTestSTablename); + rspMsg->numOfTags = ctgTestTagNum; + rspMsg->numOfColumns = ctgTestColNum; + rspMsg->precision = 1 + 1; + rspMsg->tableType = TSDB_SUPER_TABLE; + rspMsg->update = 1 + 1; + rspMsg->sversion = ctgTestSVersion + 1; + rspMsg->tversion = ctgTestTVersion + 1; + rspMsg->suid = ctgTestSuid + 1; + rspMsg->tuid = ctgTestSuid + 1; + rspMsg->vgId = 1; + + SSchema *s = NULL; + s = &rspMsg->pSchema[0]; + s->type = TSDB_DATA_TYPE_TIMESTAMP; + s->colId = 1; + s->bytes = 8; + strcpy(s->name, "ts"); + + s = &rspMsg->pSchema[1]; + s->type = TSDB_DATA_TYPE_INT; + s->colId = 2; + s->bytes = 4; + strcpy(s->name, "col1s"); + + s = &rspMsg->pSchema[2]; + s->type = TSDB_DATA_TYPE_BINARY; + s->colId = 3; + s->bytes = 12 + 1; + strcpy(s->name, "tag1s"); + + return; +} + + void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SUseDbRsp *rspMsg = NULL; // todo @@ -250,7 +292,8 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestTablename); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestTablename); rspMsg->numOfTags = 0; rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -285,8 +328,9 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestCTablename); - sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestCTablename); + strcpy(rspMsg->stbName, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -327,8 +371,9 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); - sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestSTablename); + strcpy(rspMsg->stbName, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -370,8 +415,9 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx); - sprintf(rspMsg->stbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx); + strcpy(rspMsg->dbFName, ctgTestDbname); + sprintf(rspMsg->tbName, "%s_%d", ctgTestSTablename, idx); + sprintf(rspMsg->stbName, "%s_%d", ctgTestSTablename, idx); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -586,15 +632,15 @@ void *ctgTestGetDbVgroupThread(void *param) { return NULL; } -void *ctgTestSetDbVgroupThread(void *param) { +void *ctgTestSetSameDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; + SDBVgroupInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); + code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); if (code) { assert(0); } @@ -610,6 +656,32 @@ void *ctgTestSetDbVgroupThread(void *param) { return NULL; } + +void *ctgTestSetDiffDbVgroupThread(void *param) { + struct SCatalog *pCtg = (struct SCatalog *)param; + int32_t code = 0; + SDBVgroupInfo *dbVgroup = NULL; + int32_t n = 0; + + while (!ctgTestStop) { + ctgTestBuildDBVgroup(&dbVgroup); + code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup); + if (code) { + assert(0); + } + + if (ctgTestEnableSleep) { + usleep(rand() % 5); + } + if (++n % ctgTestPrintNum == 0) { + printf("Set:%d\n", n); + } + } + + return NULL; +} + + void *ctgTestGetCtableMetaThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; @@ -669,11 +741,14 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } + TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroups(); initQueryModuleMsgHandle(); @@ -741,7 +816,7 @@ TEST(tableMeta, normalTable) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 "\n", stb->suid); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); free(stb); stb = NULL; } else { @@ -764,6 +839,8 @@ TEST(tableMeta, childTableCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -837,7 +914,7 @@ TEST(tableMeta, childTableCase) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 "\n", stb->suid); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); free(stb); stb = NULL; } else { @@ -938,7 +1015,8 @@ TEST(tableMeta, superTableCase) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 "\n", stb->suid); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); + free(stb); stb = NULL; } else { @@ -956,6 +1034,124 @@ TEST(tableMeta, superTableCase) { catalogDestroy(); } +TEST(tableMeta, rmStbMeta) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + + ctgTestInitLogFile(); + + ctgTestSetPrepareDbVgroupsAndSuperMeta(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestSTablename); + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid); + ASSERT_EQ(tableMeta->suid, ctgTestSuid); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + code = catalogRemoveSTableMeta(pCtg, "1.db1", ctgTestSTablename, ctgTestSuid); + ASSERT_EQ(code, 0); + + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); + + catalogDestroy(); +} + +TEST(tableMeta, updateStbMeta) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + + ctgTestInitLogFile(); + + ctgTestSetPrepareDbVgroupsAndSuperMeta(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestSTablename); + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid); + ASSERT_EQ(tableMeta->suid, ctgTestSuid); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + tfree(tableMeta); + + STableMetaRsp rsp = {0}; + ctgTestBuildSTableMetaRsp(&rsp); + + code = catalogUpdateSTableMeta(pCtg, &rsp); + ASSERT_EQ(code, 0); + + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1); + + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion + 1); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion + 1); + ASSERT_EQ(tableMeta->uid, ctgTestSuid + 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid + 1); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1 + 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + tfree(tableMeta); + + catalogDestroy(); +} + + + TEST(tableDistVgroup, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1062,9 +1258,11 @@ TEST(dbVgroup, getSetDbVgroupCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgroupInfo *dbVgroup = NULL; SArray *vgList = NULL; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndNormalMeta(); initQueryModuleMsgHandle(); @@ -1099,7 +1297,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { taosArrayDestroy(vgList); ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); + code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); ASSERT_EQ(code, 0); code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); @@ -1118,7 +1316,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { catalogDestroy(); } -TEST(multiThread, getSetDbVgroupCase) { +TEST(multiThread, getSetRmSameDbVgroup) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; @@ -1149,10 +1347,10 @@ TEST(multiThread, getSetDbVgroupCase) { pthread_attr_init(&thattr); pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetDbVgroupThread, pCtg); + pthread_create(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); sleep(1); - pthread_create(&(thread1), &thattr, ctgTestGetDbVgroupThread, pCtg); + pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -1169,6 +1367,59 @@ TEST(multiThread, getSetDbVgroupCase) { catalogDestroy(); } +TEST(multiThread, getSetRmDiffDbVgroup) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; + ctgTestStop = false; + + ctgTestInitLogFile(); + + ctgTestSetPrepareDbVgroups(); + + initQueryModuleMsgHandle(); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t thread1, thread2; + pthread_create(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); + + sleep(1); + pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); + + while (true) { + if (ctgTestDeadLoop) { + sleep(1); + } else { + sleep(ctgTestMTRunSec); + break; + } + } + + ctgTestStop = true; + sleep(1); + + catalogDestroy(); +} + + + TEST(multiThread, ctableMeta) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1178,6 +1429,8 @@ TEST(multiThread, ctableMeta) { SArray *vgList = NULL; ctgTestStop = false; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -1212,11 +1465,13 @@ TEST(multiThread, ctableMeta) { } ctgTestStop = true; - sleep(1); + sleep(2); catalogDestroy(); } + + TEST(rentTest, allRent) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1229,6 +1484,8 @@ TEST(rentTest, allRent) { SSTableMetaVersion *stable = NULL; uint32_t num = 0; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndMultiSuperMeta(); initQueryModuleMsgHandle(); @@ -1273,7 +1530,7 @@ TEST(rentTest, allRent) { printf("%d - expired stableNum:%d\n", i, num); if (stable) { for (int32_t n = 0; n < num; ++n) { - printf("suid:%" PRId64 ", sversion:%d, tversion:%d\n", stable[n].suid, stable[n].sversion, stable[n].tversion); + printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } free(stable); stable = NULL; @@ -1291,4 +1548,4 @@ int main(int argc, char **argv) { return RUN_ALL_TESTS(); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index c6ed2c8c03..ca3d5b2afe 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -78,3 +78,11 @@ FuncDef setExecFuncs(FuncDef def, FExecGetEnv getEnv, FExecInit init, FExecProce int32_t registerFunc(FuncDef func) { } + +int32_t fmGetFuncResultType(FuncMgtHandle handle, SFunctionNode* pFunc) { + return TSDB_CODE_SUCCESS; +} + +bool fmIsAggFunc(int32_t funcId) { + return false; +} diff --git a/source/libs/parser/inc/astCreateContext.h b/source/libs/parser/inc/astCreateContext.h index dcdc6a045b..a0bac9ea7b 100644 --- a/source/libs/parser/inc/astCreateContext.h +++ b/source/libs/parser/inc/astCreateContext.h @@ -30,12 +30,9 @@ typedef struct SAstCreateContext { SNode* pRootNode; } SAstCreateContext; -int32_t createAstCreateContext(const SParseContext* pQueryCxt, SAstCreateContext* pCxt); +int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt); int32_t destroyAstCreateContext(SAstCreateContext* pCxt); -void* acquireRaii(SAstCreateContext* pCxt, void* p); -void* releaseRaii(SAstCreateContext* pCxt, void* p); - #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h index c4c5dcba6c..82315a5ba4 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -13,11 +13,6 @@ * along with this program. If not, see . */ -#include "nodes.h" -#include "nodesShowStmts.h" -#include "astCreateContext.h" -#include "ttoken.h" - #ifndef _TD_AST_CREATE_FUNCS_H_ #define _TD_AST_CREATE_FUNCS_H_ @@ -25,20 +20,55 @@ extern "C" { #endif -bool checkTableName(const SToken* pTableName); -SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); -SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList); -SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit); -SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); -SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName); -SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); +#include "nodes.h" +#include "nodesShowStmts.h" +#include "astCreateContext.h" +#include "ttoken.h" + +extern SToken nil_token; + +SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode); +SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SToken* pEnd, SNode* pNode); +SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode); +SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode); + SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode); +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); + +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableAlias, const SToken* pColumnName); +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull); +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias); +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); -SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal); +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); + +SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); +SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); +SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow); +SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList); +SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving); +SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList); +SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit); +SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); + SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type); -SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); + #ifdef __cplusplus } diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 8f2c2ad4b3..82648e0bfa 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -10,8 +10,8 @@ SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf); -SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y index 9237f63f35..d12e76000a 100644 --- a/source/libs/parser/inc/new_sql.y +++ b/source/libs/parser/inc/new_sql.y @@ -6,7 +6,7 @@ %token_prefix NEW_TK_ %token_type { SToken } %default_type { SNode* } -%default_destructor { nodesDestroyNode($$); } +%default_destructor { PARSER_DESTRUCTOR_TRACE; nodesDestroyNode($$); } %extra_argument { SAstCreateContext* pCxt } @@ -22,7 +22,15 @@ #include "ttokendef.h" #include "astCreateFuncs.h" -#define PARSER_TRACE printf("rule = %s\n", yyRuleName[yyruleno]) +#if 0 +#define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) +#define PARSER_COMPLETE printf("parsing complete!\n" ) +#else +#define PARSER_TRACE +#define PARSER_DESTRUCTOR_TRACE +#define PARSER_COMPLETE +#endif } %syntax_error { @@ -43,157 +51,332 @@ pCxt->valid = false; } -%parse_accept { printf("parsing complete!\n" );} +%parse_accept { PARSER_COMPLETE; } -//%left OR. -//%left AND. -//%right NOT. +%left OR. +%left AND. %left UNION ALL MINUS EXCEPT INTERSECT. -//%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN. -//%left GT GE LT LE. //%left BITAND BITOR LSHIFT RSHIFT. %left NK_PLUS NK_MINUS. //%left DIVIDE TIMES. -%left NK_STAR NK_SLASH. //REM. +%left NK_STAR NK_SLASH NK_REM. //%left CONCAT. //%right UMINUS UPLUS BITNOT. -cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } +cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } +cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; } -cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; } +/************************************************ literal *************************************************************/ +literal(A) ::= NK_INTEGER(B). { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } +literal(A) ::= NK_FLOAT(B). { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B)); } +literal(A) ::= NK_STRING(B). { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B)); } +literal(A) ::= NK_BOOL(B). { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B)); } +literal(A) ::= TIMESTAMP(B) NK_STRING(C). { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &C, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &C)); } +literal(A) ::= duration_literal(B). { PARSER_TRACE; A = B; } -//////////////////////// value_function ///////////////////////////////// -value_function ::= NK_ID NK_LP value_expression NK_RP. -value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP. +duration_literal(A) ::= NK_VARIABLE(B). { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } -//////////////////////// value_expression_primary ///////////////////////////////// -value_expression_primary ::= NK_LP value_expression NK_RP. -value_expression_primary ::= nonparenthesized_value_expression_primary. +%type literal_list { SNodeList* } +%destructor literal_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +literal_list(A) ::= literal(B). { PARSER_TRACE; A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +literal_list(A) ::= literal_list(B) NK_COMMA literal(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } -nonparenthesized_value_expression_primary ::= literal. -// ? -nonparenthesized_value_expression_primary ::= column_reference. +/************************************************ names and identifiers ***********************************************/ +%type db_name { SToken } +%destructor db_name { PARSER_DESTRUCTOR_TRACE; } +db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -literal ::= NK_LITERAL. +%type table_name { SToken } +%destructor table_name { PARSER_DESTRUCTOR_TRACE; } +table_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -column_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); } -column_reference(A) ::= table_name(B) NK_DOT NK_ID(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } +%type column_name { SToken } +%destructor column_name { PARSER_DESTRUCTOR_TRACE; } +column_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -//////////////////////// value_expression ///////////////////////////////// -value_expression ::= common_value_expression. +%type function_name { SToken } +%destructor function_name { PARSER_DESTRUCTOR_TRACE; } +function_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -common_value_expression ::= numeric_value_expression. +%type table_alias { SToken } +%destructor table_alias { PARSER_DESTRUCTOR_TRACE; } +table_alias(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -numeric_value_expression ::= numeric_primary. -numeric_value_expression ::= NK_PLUS numeric_primary. -numeric_value_expression ::= NK_MINUS numeric_primary. -numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression. -numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression. -numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression. -numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression. +%type column_alias { SToken } +%destructor column_alias { PARSER_DESTRUCTOR_TRACE; } +column_alias(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -numeric_primary ::= value_expression_primary. -numeric_primary ::= value_function. +/************************************************ expression **********************************************************/ +expression(A) ::= literal(B). { PARSER_TRACE; A = B; } +//expression(A) ::= NK_QUESTION(B). { PARSER_TRACE; A = B; } +//expression(A) ::= pseudo_column(B). { PARSER_TRACE; A = B; } +expression(A) ::= column_reference(B). { PARSER_TRACE; A = B; } +expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } +//expression(A) ::= cast_expression(B). { PARSER_TRACE; A = B; } +//expression(A) ::= case_expression(B). { PARSER_TRACE; A = B; } +expression(A) ::= subquery(B). { PARSER_TRACE; A = B; } +expression(A) ::= NK_LP(B) expression(C) NK_RP(D). { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &D, releaseRawExprNode(pCxt, C)); } +expression(A) ::= NK_PLUS(B) expression(C). { + PARSER_TRACE; + SToken t = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &B, &t, releaseRawExprNode(pCxt, C)); + } +expression(A) ::= NK_MINUS(B) expression(C). { + PARSER_TRACE; + SToken t = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, C), NULL)); + } +expression(A) ::= expression(B) NK_PLUS expression(C). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_MINUS expression(C). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_STAR expression(C). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_SLASH expression(C). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_REM expression(C). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } -//////////////////////// query_specification ///////////////////////////////// -query_specification(A) ::= SELECT set_quantifier_opt(B) select_list(C) from_clause(D). { PARSER_TRACE; A = createSelectStmt(pCxt, B, C, D); } +%type expression_list { SNodeList* } +%destructor expression_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +expression_list(A) ::= expression(B). { PARSER_TRACE; A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +expression_list(A) ::= expression_list(B) NK_COMMA expression(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } -%type set_quantifier_opt { bool } -%destructor set_quantifier_opt {} -set_quantifier_opt(A) ::= . { PARSER_TRACE; A = false; } -set_quantifier_opt(A) ::= DISTINCT. { PARSER_TRACE; A = true; } -set_quantifier_opt(A) ::= ALL. { PARSER_TRACE; A = false; } +column_reference(A) ::= column_name(B). { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); } +column_reference(A) ::= table_name(B) NK_DOT column_name(C). { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); } -%type select_list { SNodeList* } -%destructor select_list { nodesDestroyNodeList($$); } -select_list(A) ::= NK_STAR. { PARSER_TRACE; A = NULL; } -select_list(A) ::= select_sublist(B). { PARSER_TRACE; A = B; } +//pseudo_column(A) ::= NK_NOW. { PARSER_TRACE; A = createFunctionNode(pCxt, NULL, NULL); } -%type select_sublist { SNodeList* } -%destructor select_sublist { nodesDestroyNodeList($$); } -select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } -select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } +/************************************************ predicate ***********************************************************/ +predicate(A) ::= expression(B) compare_op(C) expression(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)); } +//predicate(A) ::= expression(B) compare_op sub_type expression(B). +predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D)); } +predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)); } +predicate(A) ::= expression(B) IS NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), true); } +predicate(A) ::= expression(B) IS NOT NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), false); } +predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), D); } -select_item(A) ::= value_expression(B). { PARSER_TRACE; A = B; } -select_item(A) ::= value_expression(B) AS NK_ID(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } -select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } +%type compare_op { EOperatorType } +%destructor compare_op { PARSER_DESTRUCTOR_TRACE; } +compare_op(A) ::= NK_LT. { PARSER_TRACE; A = OP_TYPE_LOWER_THAN; } +compare_op(A) ::= NK_GT. { PARSER_TRACE; A = OP_TYPE_GREATER_THAN; } +compare_op(A) ::= NK_LE. { PARSER_TRACE; A = OP_TYPE_LOWER_EQUAL; } +compare_op(A) ::= NK_GE. { PARSER_TRACE; A = OP_TYPE_GREATER_EQUAL; } +compare_op(A) ::= NK_NE. { PARSER_TRACE; A = OP_TYPE_NOT_EQUAL; } +compare_op(A) ::= NK_EQ. { PARSER_TRACE; A = OP_TYPE_EQUAL; } +compare_op(A) ::= LIKE. { PARSER_TRACE; A = OP_TYPE_LIKE; } +compare_op(A) ::= NOT LIKE. { PARSER_TRACE; A = OP_TYPE_NOT_LIKE; } +compare_op(A) ::= MATCH. { PARSER_TRACE; A = OP_TYPE_MATCH; } +compare_op(A) ::= NMATCH. { PARSER_TRACE; A = OP_TYPE_NMATCH; } -from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } +%type in_op { EOperatorType } +%destructor in_op { PARSER_DESTRUCTOR_TRACE; } +in_op(A) ::= IN. { PARSER_TRACE; A = OP_TYPE_IN; } +in_op(A) ::= NOT IN. { PARSER_TRACE; A = OP_TYPE_NOT_IN; } -//%type table_reference_list { SNodeList* } -//%destructor table_reference_list { nodesDestroyNodeList($$); } -table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; } -//table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, B, C); } +in_predicate_value(A) ::= NK_LP expression_list(B) NK_RP. { PARSER_TRACE; A = createNodeListNode(pCxt, B); } -//table_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createRealTableNode(pCxt, ); } -table_reference(A) ::= table_factor(B). { PARSER_TRACE; A = B; } -//table_reference ::= joined_table. +/************************************************ boolean_value_expression ********************************************/ +boolean_value_expression(A) ::= boolean_primary(B). { PARSER_TRACE; A = B; } +boolean_value_expression(A) ::= NOT boolean_primary(B). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, B, NULL); } +boolean_value_expression(A) ::= + boolean_value_expression(B) OR boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, B, C); } +boolean_value_expression(A) ::= + boolean_value_expression(B) AND boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, B, C); } -table_factor(A) ::= table_primary(B). { PARSER_TRACE; A = B; } +boolean_primary(A) ::= predicate(B). { PARSER_TRACE; A = B; } +boolean_primary(A) ::= NK_LP boolean_value_expression(B) NK_RP. { PARSER_TRACE; A = B; } -table_primary(A) ::= table_name(B). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B); } -table_primary(A) ::= db_name(B) NK_DOT table_name(C). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C); } -table_primary ::= derived_table. +/************************************************ from_clause *********************************************************/ +from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } -derived_table ::= table_subquery. +table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; } +table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } -%type db_name { SToken } -db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -%type table_name { SToken } -table_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } +/************************************************ table_reference *****************************************************/ +table_reference(A) ::= table_primary(B). { PARSER_TRACE; A = B; } +table_reference(A) ::= joined_table(B). { PARSER_TRACE; A = B; } -//////////////////////// subquery ///////////////////////////////// -subquery ::= NK_LR query_expression NK_RP. +table_primary(A) ::= table_name(B) alias_opt(C). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B, &C); } +table_primary(A) ::= db_name(B) NK_DOT table_name(C) alias_opt(D). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C, &D); } +table_primary(A) ::= subquery(B) alias_opt(C). { PARSER_TRACE; A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), &C); } +table_primary(A) ::= parenthesized_joined_table(B). { PARSER_TRACE; A = B; } -table_subquery ::= subquery. +%type alias_opt { SToken } +%destructor alias_opt { PARSER_DESTRUCTOR_TRACE; } +alias_opt(A) ::= . { PARSER_TRACE; A = nil_token; } +alias_opt(A) ::= table_alias(B). { PARSER_TRACE; A = B; } +alias_opt(A) ::= AS table_alias(B). { PARSER_TRACE; A = B; } -// query_expression -query_expression(A) ::= with_clause_opt query_expression_body(B) order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { - PARSER_TRACE; - addOrderByList(pCxt, B, C); - addSlimit(pCxt, B, D); - addLimit(pCxt, B, E); - A = B; - } +parenthesized_joined_table(A) ::= NK_LP joined_table(B) NK_RP. { PARSER_TRACE; A = B; } +parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP. { PARSER_TRACE; A = B; } -// WITH AS -with_clause_opt ::= . {} +/************************************************ joined_table ********************************************************/ +joined_table(A) ::= + table_reference(B) join_type(C) JOIN table_reference(D) ON search_condition(E). { PARSER_TRACE; A = createJoinTableNode(pCxt, C, B, D, E); } -query_expression_body(A) ::= query_primary(B). { PARSER_TRACE; A = B; } -query_expression_body(A) ::= query_expression_body(B) UNION ALL query_expression_body(D). { PARSER_TRACE; A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } +%type join_type { EJoinType } +%destructor join_type { PARSER_DESTRUCTOR_TRACE; } +join_type(A) ::= INNER. { PARSER_TRACE; A = JOIN_TYPE_INNER; } -query_primary(A) ::= query_specification(B). { PARSER_TRACE; A = B; } -query_primary(A) ::= NK_LP query_expression_body(B) order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP. { PARSER_TRACE; A = B;} +/************************************************ query_specification *************************************************/ +query_specification(A) ::= + SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) + partition_by_clause_opt(F) twindow_clause_opt(G) + group_by_clause_opt(H) having_clause_opt(I). { + PARSER_TRACE; + A = createSelectStmt(pCxt, B, C, D); + A = addWhereClause(pCxt, A, E); + A = addPartitionByClause(pCxt, A, F); + A = addWindowClauseClause(pCxt, A, G); + A = addGroupByClause(pCxt, A, H); + A = addHavingClause(pCxt, A, I); + } -%type order_by_clause_opt { SNodeList* } -%destructor order_by_clause_opt { nodesDestroyNodeList($$); } -order_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } -order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; } +%type set_quantifier_opt { bool } +%destructor set_quantifier_opt { PARSER_DESTRUCTOR_TRACE; } +set_quantifier_opt(A) ::= . { PARSER_TRACE; A = false; } +set_quantifier_opt(A) ::= DISTINCT. { PARSER_TRACE; A = true; } +set_quantifier_opt(A) ::= ALL. { PARSER_TRACE; A = false; } -slimit_clause_opt(A) ::= . { A = NULL; } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } +%type select_list { SNodeList* } +%destructor select_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +select_list(A) ::= NK_STAR. { PARSER_TRACE; A = NULL; } +select_list(A) ::= select_sublist(B). { PARSER_TRACE; A = B; } -limit_clause_opt(A) ::= . { A = NULL; } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } +%type select_sublist { SNodeList* } +%destructor select_sublist { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } -//////////////////////// sort_specification_list ///////////////////////////////// -%type sort_specification_list { SNodeList* } -%destructor sort_specification_list { nodesDestroyNodeList($$); } -sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } -sort_specification_list(A) ::= sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } +select_item(A) ::= expression(B). { + PARSER_TRACE; + SToken t = getTokenFromRawExprNode(pCxt, B); + A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &t); + } +select_item(A) ::= expression(B) column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } +select_item(A) ::= expression(B) AS column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } +select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } -sort_specification(A) ::= value_expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); } +where_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +where_clause_opt(A) ::= WHERE search_condition(B). { PARSER_TRACE; A = B; } + +%type partition_by_clause_opt { SNodeList* } +%destructor partition_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +partition_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). { PARSER_TRACE; A = B; } + +twindow_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +twindow_clause_opt(A) ::= + SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { PARSER_TRACE; A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); } +twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { PARSER_TRACE; A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { PARSER_TRACE; A = createIntervalWindowNode(pCxt, B, NULL, C, D); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP + sliding_opt(D) fill_opt(E). { PARSER_TRACE; A = createIntervalWindowNode(pCxt, B, C, D, E); } + +sliding_opt(A) ::= . { PARSER_TRACE; A = NULL; } +sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { PARSER_TRACE; A = B; } + +fill_opt(A) ::= . { PARSER_TRACE; A = NULL; } +fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { PARSER_TRACE; A = createFillNode(pCxt, B, NULL); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { PARSER_TRACE; A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } + +%type fill_mode { EFillMode } +%destructor fill_mode { PARSER_DESTRUCTOR_TRACE; } +fill_mode(A) ::= NONE. { PARSER_TRACE; A = FILL_MODE_NONE; } +fill_mode(A) ::= PREV. { PARSER_TRACE; A = FILL_MODE_PREV; } +fill_mode(A) ::= NULL. { PARSER_TRACE; A = FILL_MODE_NULL; } +fill_mode(A) ::= LINEAR. { PARSER_TRACE; A = FILL_MODE_LINEAR; } +fill_mode(A) ::= NEXT. { PARSER_TRACE; A = FILL_MODE_NEXT; } + +%type group_by_clause_opt { SNodeList* } +%destructor group_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +group_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +group_by_clause_opt(A) ::= GROUP BY expression_list(B). { PARSER_TRACE; A = B; } + +having_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +having_clause_opt(A) ::= HAVING search_condition(B). { PARSER_TRACE; A = B; } + +/************************************************ query_expression ****************************************************/ +query_expression(A) ::= + query_expression_body(B) + order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { + PARSER_TRACE; + A = addOrderByClause(pCxt, B, C); + A = addSlimitClause(pCxt, A, D); + A = addLimitClause(pCxt, A, E); + } + +query_expression_body(A) ::= query_primary(B). { PARSER_TRACE; A = B; } +query_expression_body(A) ::= + query_expression_body(B) UNION ALL query_expression_body(D). { PARSER_TRACE; A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } + +query_primary(A) ::= query_specification(B). { PARSER_TRACE; A = B; } +//query_primary(A) ::= +// NK_LP query_expression_body(B) +// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { PARSER_TRACE; A = B;} + +%type order_by_clause_opt { SNodeList* } +%destructor order_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +order_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; } + +slimit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, NULL); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } + +limit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, NULL); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } + +/************************************************ subquery ************************************************************/ +subquery(A) ::= NK_LP(B) query_expression(C) NK_RP(D). { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &D, C); } + +/************************************************ search_condition ****************************************************/ +search_condition(A) ::= boolean_value_expression(B). { PARSER_TRACE; A = B; } + +/************************************************ sort_specification_list *********************************************/ +%type sort_specification_list { SNodeList* } +%destructor sort_specification_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +sort_specification_list(A) ::= + sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +sort_specification(A) ::= + expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, B), C, D); } %type ordering_specification_opt EOrder -%destructor ordering_specification_opt {} -ordering_specification_opt(A) ::= . { PARSER_TRACE; A = ORDER_ASC; } -ordering_specification_opt(A) ::= ASC. { PARSER_TRACE; A = ORDER_ASC; } -ordering_specification_opt(A) ::= DESC. { PARSER_TRACE; A = ORDER_DESC; } +%destructor ordering_specification_opt { PARSER_DESTRUCTOR_TRACE; } +ordering_specification_opt(A) ::= . { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= ASC. { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= DESC. { PARSER_TRACE; A = ORDER_DESC; } %type null_ordering_opt ENullOrder -%destructor null_ordering_opt {} -null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; } -null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; } -null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; } +%destructor null_ordering_opt { PARSER_DESTRUCTOR_TRACE; } +null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; } +null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; } +null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; } diff --git a/source/libs/parser/inc/parserImpl.h b/source/libs/parser/inc/parserImpl.h index 57012e2fd6..183075d465 100644 --- a/source/libs/parser/inc/parserImpl.h +++ b/source/libs/parser/inc/parserImpl.h @@ -28,6 +28,7 @@ typedef struct SQuery { } SQuery; int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery); +int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 410e4fb187..1222138b5e 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -813,14 +813,14 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } //////////////////////////////////ALTER TAGS statement///////////////////////////////////// cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { @@ -829,7 +829,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { toTSDBType(Y.type); SArray* A = tListItemAppendToken(NULL, &Y, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } @@ -842,7 +842,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { toTSDBType(Z.type); A = tListItemAppendToken(A, &Z, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } @@ -859,7 +859,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } @@ -882,14 +882,14 @@ cmd ::= ALTER STABLE ids(X) cpxName(F) DROP COLUMN ids(A). { cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } //////////////////////////////////ALTER TAGS statement///////////////////////////////////// cmd ::= ALTER STABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y). { @@ -898,7 +898,7 @@ cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y). { toTSDBType(Y.type); SArray* A = tListItemAppendToken(NULL, &Y, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } @@ -911,7 +911,7 @@ cmd ::= ALTER STABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { toTSDBType(Z.type); A = tListItemAppendToken(A, &Z, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } @@ -928,7 +928,7 @@ cmd ::= ALTER STABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } diff --git a/source/libs/parser/src/astCreateContext.c b/source/libs/parser/src/astCreateContext.c index 42c5257360..a8c82780ca 100644 --- a/source/libs/parser/src/astCreateContext.c +++ b/source/libs/parser/src/astCreateContext.c @@ -16,25 +16,14 @@ #include "ttoken.h" #include "astCreateContext.h" -void* acquireRaii(SAstCreateContext* pCxt, void* p) { - if (NULL == p) { - return NULL; - } - return p; +int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt) { + pCxt->pQueryCxt = pQueryCxt; + pCxt->notSupport = false; + pCxt->valid = true; + pCxt->pRootNode = NULL; + return TSDB_CODE_SUCCESS; } -void* releaseRaii(SAstCreateContext* pCxt, void* p) { - if (NULL == p) { - return NULL; - } - return p; +int32_t destroyAstCreateContext(SAstCreateContext* pCxt) { + return TSDB_CODE_SUCCESS; } - -int32_t createAstCreater(const SParseContext* pQueryCxt, SAstCreateContext* pCxt) { - -} - -int32_t destroyAstCreater(SAstCreateContext* pCxt) { - -} - diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c index 337ae78d75..bfcebfd8b8 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/astCreateFuncs.c @@ -16,76 +16,351 @@ #include "astCreateFuncs.h" -#include "astCreateContext.h" +#define CHECK_OUT_OF_MEM(p) \ + do { \ + if (NULL == (p)) { \ + pCxt->valid = false; \ + return NULL; \ + } \ + } while (0) -bool checkTableName(const SToken* pTableName) { - printf("%p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z); - return pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; +#define CHECK_RAW_EXPR_NODE(node) \ + do { \ + if (NULL == (node) || QUERY_NODE_RAW_EXPR != nodeType(node)) { \ + pCxt->valid = false; \ + return NULL; \ + } \ + } while (0) + +SToken nil_token = { .type = TK_NIL, .n = 0, .z = NULL }; + +static bool checkDbName(SAstCreateContext* pCxt, const SToken* pDbName) { + if (NULL == pDbName) { + return true; + } + pCxt->valid = pDbName->n < TSDB_DB_NAME_LEN ? true : false; + return pCxt->valid; } -SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { - +static bool checkTableName(SAstCreateContext* pCxt, const SToken* pTableName) { + if (NULL == pTableName) { + return true; + } + pCxt->valid = pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; + return pCxt->valid; } -SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { - +static bool checkColumnName(SAstCreateContext* pCxt, const SToken* pColumnName) { + if (NULL == pColumnName) { + return true; + } + pCxt->valid = pColumnName->n < TSDB_COL_NAME_LEN ? true : false; + return pCxt->valid; } -SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { - +SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode) { + SRawExprNode* target = (SRawExprNode*)nodesMakeNode(QUERY_NODE_RAW_EXPR); + CHECK_OUT_OF_MEM(target); + target->p = pToken->z; + target->n = pToken->n; + target->pNode = pNode; + return (SNode*)target; } -SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { - +SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SToken* pEnd, SNode* pNode) { + SRawExprNode* target = (SRawExprNode*)nodesMakeNode(QUERY_NODE_RAW_EXPR); + CHECK_OUT_OF_MEM(target); + target->p = pStart->z; + target->n = (pEnd->z + pEnd->n) - pStart->z; + target->pNode = pNode; + return (SNode*)target; } -SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName) { - +SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { + CHECK_RAW_EXPR_NODE(pNode); + SNode* tmp = ((SRawExprNode*)pNode)->pNode; + tfree(pNode); + return tmp; } -SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { - +SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { + SRawExprNode* target = (SRawExprNode*)pNode; + SToken t = { .type = 0, .z = target->p, .n = target->n}; + return t; } SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { + SNodeList* list = nodesMakeList(); + CHECK_OUT_OF_MEM(list); + return nodesListAppend(list, pNode); +} +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { + return nodesListAppend(pList, pNode); +} + +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableAlias, const SToken* pColumnName) { + if (!checkTableName(pCxt, pTableAlias) || !checkColumnName(pCxt, pColumnName)) { + return NULL; + } + SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + CHECK_OUT_OF_MEM(col); + if (NULL != pTableAlias) { + strncpy(col->tableAlias, pTableAlias->z, pTableAlias->n); + } + strncpy(col->colName, pColumnName->z, pColumnName->n); + return (SNode*)col; +} + +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral) { + SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + CHECK_OUT_OF_MEM(val); + val->literal = strndup(pLiteral->z, pLiteral->n); + CHECK_OUT_OF_MEM(val->literal); + val->node.resType.type = dataType; + val->node.resType.bytes = tDataTypes[dataType].bytes; + if (TSDB_DATA_TYPE_TIMESTAMP == dataType) { + val->node.resType.precision = TSDB_TIME_PRECISION_MILLI; + } + return (SNode*)val; +} + +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) { + SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + CHECK_OUT_OF_MEM(val); + val->literal = strndup(pLiteral->z, pLiteral->n); + CHECK_OUT_OF_MEM(val->literal); + val->isDuration = true; + val->node.resType.type = TSDB_DATA_TYPE_BIGINT; + val->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + val->node.resType.precision = TSDB_TIME_PRECISION_MILLI; + return (SNode*)val; +} + +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2) { + SLogicConditionNode* cond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + CHECK_OUT_OF_MEM(cond); + cond->condType = type; + cond->pParameterList = nodesMakeList(); + nodesListAppend(cond->pParameterList, pParam1); + nodesListAppend(cond->pParameterList, pParam2); + return (SNode*)cond; +} + +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight) { + SOperatorNode* op = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + CHECK_OUT_OF_MEM(op); + op->opType = type; + op->pLeft = pLeft; + op->pRight = pRight; + return (SNode*)op; +} + +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + return createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, + createOperatorNode(pCxt, OP_TYPE_GREATER_EQUAL, pExpr, pLeft), createOperatorNode(pCxt, OP_TYPE_LOWER_EQUAL, pExpr, pRight)); +} + +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + return createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, + createOperatorNode(pCxt, OP_TYPE_LOWER_THAN, pExpr, pLeft), createOperatorNode(pCxt, OP_TYPE_GREATER_THAN, pExpr, pRight)); +} + +SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull) { + SIsNullCondNode* cond = (SIsNullCondNode*)nodesMakeNode(QUERY_NODE_IS_NULL_CONDITION); + CHECK_OUT_OF_MEM(cond); + cond->pExpr = pExpr; + cond->isNull = isNull; + return (SNode*)cond; +} + +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList) { + SFunctionNode* func = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + CHECK_OUT_OF_MEM(func); + strncpy(func->functionName, pFuncName->z, pFuncName->n); + func->pParameterList = pParameterList; + return (SNode*)func; +} + +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList) { + SNodeListNode* list = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); + CHECK_OUT_OF_MEM(list); + list->pNodeList = pList; + return (SNode*)list; +} + +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias) { + if (!checkDbName(pCxt, pDbName) || !checkTableName(pCxt, pTableName)) { + return NULL; + } + SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + CHECK_OUT_OF_MEM(realTable); + if (NULL != pDbName) { + strncpy(realTable->table.dbName, pDbName->z, pDbName->n); + } else { + strcpy(realTable->table.dbName, pCxt->pQueryCxt->db); + } + if (NULL != pTableAlias && TK_NIL != pTableAlias->type) { + strncpy(realTable->table.tableAlias, pTableAlias->z, pTableAlias->n); + } else { + strncpy(realTable->table.tableAlias, pTableName->z, pTableName->n); + } + strncpy(realTable->table.tableName, pTableName->z, pTableName->n); + return (SNode*)realTable; +} + +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias) { + STempTableNode* tempTable = (STempTableNode*)nodesMakeNode(QUERY_NODE_TEMP_TABLE); + CHECK_OUT_OF_MEM(tempTable); + tempTable->pSubquery = pSubquery; + if (NULL != pTableAlias && TK_NIL != pTableAlias->type) { + strncpy(tempTable->table.tableAlias, pTableAlias->z, pTableAlias->n); + } + return (SNode*)tempTable; +} + +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond) { + SJoinTableNode* joinTable = (SJoinTableNode*)nodesMakeNode(QUERY_NODE_JOIN_TABLE); + CHECK_OUT_OF_MEM(joinTable); + joinTable->joinType = type; + joinTable->pLeft = pLeft; + joinTable->pRight = pRight; + joinTable->pOnCond = pJoinCond; + return (SNode*)joinTable; +} + +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { + SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); + CHECK_OUT_OF_MEM(limitNode); + // limitNode->limit = limit; + // limitNode->offset = offset; + return (SNode*)limitNode; } SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder) { - + SOrderByExprNode* orderByExpr = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + CHECK_OUT_OF_MEM(orderByExpr); + orderByExpr->pExpr = pExpr; + orderByExpr->order = order; + orderByExpr->nullOrder = nullOrder; + return (SNode*)orderByExpr; } -SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName) { - SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); - if (NULL != pDbName) { - printf("DbName %p : %d, %d, %s\n", pDbName, pDbName->type, pDbName->n, pDbName->z); - strncpy(realTable->dbName, pDbName->z, pDbName->n); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal) { + SSessionWindowNode* session = (SSessionWindowNode*)nodesMakeNode(QUERY_NODE_SESSION_WINDOW); + CHECK_OUT_OF_MEM(session); + session->pCol = pCol; + // session->gap = getInteger(pVal); + return (SNode*)session; +} + +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol) { + SStateWindowNode* state = (SStateWindowNode*)nodesMakeNode(QUERY_NODE_STATE_WINDOW); + CHECK_OUT_OF_MEM(state); + state->pCol = pCol; + return (SNode*)state; +} + +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { + SIntervalWindowNode* interval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); + CHECK_OUT_OF_MEM(interval); + interval->pInterval = pInterval; + interval->pOffset = pOffset; + interval->pSliding = pSliding; + interval->pFill = pFill; + return (SNode*)interval; +} + +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { + SFillNode* fill = (SFillNode*)nodesMakeNode(QUERY_NODE_FILL); + CHECK_OUT_OF_MEM(fill); + fill->mode = mode; + fill->pValues = pValues; + return (SNode*)fill; +} + +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { + strncpy(((SExprNode*)pNode)->aliasName, pAlias->z, pAlias->n); + return pNode; +} + +SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pWhere = pWhere; } - printf("TableName %p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z); - strncpy(realTable->table.tableName, pTableName->z, pTableName->n); - return acquireRaii(pCxt, realTable); + return pStmt; +} + +SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pPartitionByList = pPartitionByList; + } + return pStmt; +} + +SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pWindow = pWindow; + } + return pStmt; +} + +SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pGroupByList = pGroupByList; + } + return pStmt; +} + +SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pHaving = pHaving; + } + return pStmt; +} + +SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pOrderByList = pOrderByList; + } + return pStmt; +} + +SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pSlimit = pSlimit; + } + return pStmt; +} + +SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pLimit = pLimit; + } + return pStmt; } SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + CHECK_OUT_OF_MEM(select); select->isDistinct = isDistinct; - if (NULL == pProjectionList) { - select->isStar = true; - } - select->pProjectionList = releaseRaii(pCxt, pProjectionList); - printf("pTable = %p, name = %s\n", pTable, ((SRealTableNode*)pTable)->table.tableName); - select->pFromTable = releaseRaii(pCxt, pTable); - return acquireRaii(pCxt, select); + select->pProjectionList = pProjectionList; + select->pFromTable = pTable; + return (SNode*)select; } SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight) { - + SSetOperator* setOp = (SSetOperator*)nodesMakeNode(QUERY_NODE_SET_OPERATOR); + CHECK_OUT_OF_MEM(setOp); + setOp->opType = type; + setOp->pLeft = pLeft; + setOp->pRight = pRight; + return (SNode*)setOp; } SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type) { - -} - -SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { - + SShowStmt* show = (SShowStmt*)nodesMakeNode(QUERY_NODE_SHOW_STMT); + CHECK_OUT_OF_MEM(show); + show->showType = type; + return (SNode*)show; } diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index 8e8da92bf5..5dce5120f9 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -610,7 +610,7 @@ SAlterTableInfo *tSetAlterTableInfo(SToken *pTableName, SArray *pCols, SArray *p pAlterTable->type = type; pAlterTable->tableType = tableType; - if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || type == TSDB_ALTER_TABLE_CHANGE_COLUMN || type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) { + if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG || type == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES || type == TSDB_ALTER_TABLE_UPDATE_TAG_BYTES) { pAlterTable->pAddColumns = pCols; assert(pVals == NULL); } else { diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 697fd0c4cb..e1828b6a26 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -249,112 +249,64 @@ SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx return pCreateMsg; } -SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { - SSchema* pSchema; +char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { + SMCreateStbReq createReq = {0}; + createReq.igExists = pCreateTableSql->existCheck ? 1 : 0; + createReq.pColumns = pCreateTableSql->colInfo.pColumns; + createReq.pTags = pCreateTableSql->colInfo.pTagColumns; + createReq.numOfColumns = (int32_t)taosArrayGetSize(pCreateTableSql->colInfo.pColumns); + createReq.numOfTags = (int32_t)taosArrayGetSize(pCreateTableSql->colInfo.pTagColumns); - int32_t numOfTags = 0; - int32_t numOfCols = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pColumns); - if (pCreateTableSql->colInfo.pTagColumns != NULL) { - numOfTags = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pTagColumns); - } - - SMCreateStbReq* pCreateStbMsg = (SMCreateStbReq*)calloc(1, sizeof(SMCreateStbReq) + (numOfCols + numOfTags) * sizeof(SSchema)); - if (pCreateStbMsg == NULL) { + SName n = {0}; + if (createSName(&n, &pCreateTableSql->name, pParseCtx, pMsgBuf) != 0) { return NULL; } - char* pMsg = NULL; -#if 0 - int32_t tableType = pCreateTableSql->type; - if (tableType != TSQL_CREATE_TABLE && tableType != TSQL_CREATE_STABLE) { // create by using super table, tags value - SArray* list = pInfo->pCreateTableInfo->childTableInfo; + if (tNameExtractFullName(&n, createReq.name) != 0) { + buildInvalidOperationMsg(pMsgBuf, "invalid table name or database not specified"); + return NULL; + } - int32_t numOfTables = (int32_t)taosArrayGetSize(list); - pCreateStbMsg->numOfTables = htonl(numOfTables); + int32_t tlen = tSerializeSMCreateStbReq(NULL, &createReq); + void* req = malloc(tlen); + if (req == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - pMsg = (char*)pCreateMsg; - for (int32_t i = 0; i < numOfTables; ++i) { - SCreateTableMsg* pCreate = (SCreateTableMsg*)pMsg; - - pCreate->numOfColumns = htons(pCmd->numOfCols); - pCreate->numOfTags = htons(pCmd->count); - pMsg += sizeof(SCreateTableMsg); - - SCreatedTableInfo* p = taosArrayGet(list, i); - strcpy(pCreate->tableName, p->fullname); - pCreate->igExists = (p->igExist) ? 1 : 0; - - // use dbinfo from table id without modifying current db info - pMsg = serializeTagData(&p->tagdata, pMsg); - - int32_t len = (int32_t)(pMsg - (char*)pCreate); - pCreate->len = htonl(len); - } - - } else { -#endif - // create (super) table - SName n = {0}; - int32_t code = createSName(&n, &pCreateTableSql->name, pParseCtx, pMsgBuf); - if (code != 0) { - return NULL; - } - - code = tNameExtractFullName(&n, pCreateStbMsg->name); - if (code != 0) { - buildInvalidOperationMsg(pMsgBuf, "invalid table name or database not specified"); - return NULL; - } - - pCreateStbMsg->igExists = pCreateTableSql->existCheck ? 1 : 0; - pCreateStbMsg->numOfColumns = htonl(numOfCols); - pCreateStbMsg->numOfTags = htonl(numOfTags); - - pSchema = (SSchema*)pCreateStbMsg->pSchema; - for (int i = 0; i < numOfCols; ++i) { - SField* pField = taosArrayGet(pCreateTableSql->colInfo.pColumns, i); - pSchema->type = pField->type; - pSchema->bytes = htonl(pField->bytes); - strcpy(pSchema->name, pField->name); - - pSchema++; - } - - for(int32_t i = 0; i < numOfTags; ++i) { - SField* pField = taosArrayGet(pCreateTableSql->colInfo.pTagColumns, i); - pSchema->type = pField->type; - pSchema->bytes = htonl(pField->bytes); - strcpy(pSchema->name, pField->name); - - pSchema++; - } - - pMsg = (char*)pSchema; - - int32_t msgLen = (int32_t)(pMsg - (char*)pCreateStbMsg); - *len = msgLen; - - return pCreateStbMsg; + void* buf = req; + tSerializeSMCreateStbReq(&buf, &createReq); + *len = tlen; + return req; } -SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { +char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SToken* tableName = taosArrayGet(pInfo->pMiscInfo->a, 0); - SName name = {0}; + SName name = {0}; int32_t code = createSName(&name, tableName, pParseCtx, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { terrno = buildInvalidOperationMsg(pMsgBuf, "invalid table name"); return NULL; } - SMDropStbReq *pDropTableMsg = (SMDropStbReq*) calloc(1, sizeof(SMDropStbReq)); + SMDropStbReq dropReq = {0}; + code = tNameExtractFullName(&name, dropReq.name); - code = tNameExtractFullName(&name, pDropTableMsg->name); assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_TABLE_NAME_T); + dropReq.igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - *len = sizeof(SMDropStbReq); - return pDropTableMsg; + int32_t tlen = tSerializeSMDropStbReq(NULL, &dropReq); + void* req = malloc(tlen); + if (req == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + void* buf = req; + tSerializeSMDropStbReq(&buf, &dropReq); + *len = tlen; + return req; } SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { diff --git a/source/libs/parser/src/astTranslate.c b/source/libs/parser/src/astTranslate.c deleted file mode 100644 index ff68c4b22c..0000000000 --- a/source/libs/parser/src/astTranslate.c +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// int32_t doTranslate() { - -// } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 3ae89bca0a..4b70d2c6c3 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -598,7 +598,7 @@ static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SPa } static int32_t serializeVgroupTablesBatchImpl(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) { - int tlen = sizeof(SMsgHead) + tSVCreateTbBatchReqSerialize(NULL, &(pTbBatch->req)); + int tlen = sizeof(SMsgHead) + tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req)); void* buf = malloc(tlen); if (buf == NULL) { // TODO: handle error @@ -608,7 +608,7 @@ static int32_t serializeVgroupTablesBatchImpl(SVgroupTablesBatch* pTbBatch, SArr ((SMsgHead*)buf)->contLen = htonl(tlen); void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tSVCreateTbBatchReqSerialize(&pBuf, &(pTbBatch->req)); + tSerializeSVCreateTbBatchReq(&pBuf, &(pTbBatch->req)); SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks)); pVgData->vg = pTbBatch->info; @@ -924,13 +924,13 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - pDcl->pMsg = (char*)buildCreateStbMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf); + pDcl->pMsg = buildCreateStbReq(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf); pDcl->msgType = TDMT_MND_CREATE_STB; break; } case TSDB_SQL_DROP_TABLE: { - pDcl->pMsg = (char*)buildDropStableMsg(pInfo, &pDcl->msgLen, pCtx, pMsgBuf); + pDcl->pMsg = buildDropStableReq(pInfo, &pDcl->msgLen, pCtx, pMsgBuf); if (pDcl->pMsg == NULL) { goto _error; } diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/new_sql.c index 7bf29c45f8..cb2b277df8 100644 --- a/source/libs/parser/src/new_sql.c +++ b/source/libs/parser/src/new_sql.c @@ -37,7 +37,15 @@ #include "ttokendef.h" #include "astCreateFuncs.h" -#define PARSER_TRACE printf("rule = %s\n", yyRuleName[yyruleno]) +#if 0 +#define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) +#define PARSER_COMPLETE printf("parsing complete!\n" ) +#else +#define PARSER_TRACE +#define PARSER_DESTRUCTOR_TRACE +#define PARSER_COMPLETE +#endif /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols ** in a format understandable to "makeheaders". This section is blank unless @@ -101,18 +109,21 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 71 -#define YYACTIONTYPE unsigned char +#define YYNOCODE 124 +#define YYACTIONTYPE unsigned short int #define NewParseTOKENTYPE SToken typedef union { int yyinit; NewParseTOKENTYPE yy0; - bool yy9; - SNodeList* yy30; - SToken yy67; - ENullOrder yy68; - EOrder yy108; - SNode* yy130; + EOrder yy10; + EFillMode yy14; + SNode* yy168; + ENullOrder yy177; + SNodeList* yy192; + bool yy209; + EOperatorType yy228; + EJoinType yy229; + SToken yy241; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -127,17 +138,17 @@ typedef union { #define NewParseCTX_PARAM #define NewParseCTX_FETCH #define NewParseCTX_STORE -#define YYNSTATE 63 -#define YYNRULE 68 -#define YYNTOKEN 35 -#define YY_MAX_SHIFT 62 -#define YY_MIN_SHIFTREDUCE 114 -#define YY_MAX_SHIFTREDUCE 181 -#define YY_ERROR_ACTION 182 -#define YY_ACCEPT_ACTION 183 -#define YY_NO_ACTION 184 -#define YY_MIN_REDUCE 185 -#define YY_MAX_REDUCE 252 +#define YYNSTATE 143 +#define YYNRULE 134 +#define YYNTOKEN 72 +#define YY_MAX_SHIFT 142 +#define YY_MIN_SHIFTREDUCE 238 +#define YY_MAX_SHIFTREDUCE 371 +#define YY_ERROR_ACTION 372 +#define YY_ACCEPT_ACTION 373 +#define YY_NO_ACTION 374 +#define YY_MIN_REDUCE 375 +#define YY_MAX_REDUCE 508 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -204,89 +215,202 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (229) +#define YY_ACTTAB_COUNT (705) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 30, 50, 30, 30, 30, 30, 55, 50, 30, 30, - /* 10 */ 10, 9, 29, 62, 51, 195, 30, 19, 30, 30, - /* 20 */ 30, 30, 57, 19, 30, 30, 30, 19, 30, 30, - /* 30 */ 30, 30, 57, 19, 30, 30, 30, 50, 30, 30, - /* 40 */ 30, 30, 55, 50, 30, 30, 39, 222, 155, 156, - /* 50 */ 23, 196, 12, 11, 10, 9, 25, 223, 30, 58, - /* 60 */ 30, 30, 30, 30, 57, 58, 30, 30, 30, 59, - /* 70 */ 30, 30, 30, 30, 57, 59, 30, 30, 30, 37, - /* 80 */ 30, 30, 30, 30, 57, 37, 30, 30, 204, 38, - /* 90 */ 5, 183, 60, 166, 158, 159, 121, 53, 200, 201, - /* 100 */ 202, 203, 54, 203, 203, 203, 246, 18, 246, 246, - /* 110 */ 246, 246, 57, 120, 246, 246, 245, 17, 245, 245, - /* 120 */ 245, 245, 57, 46, 245, 245, 35, 27, 35, 35, - /* 130 */ 35, 35, 57, 31, 35, 35, 36, 47, 36, 36, - /* 140 */ 36, 36, 57, 128, 36, 36, 242, 117, 242, 242, - /* 150 */ 242, 242, 57, 42, 241, 242, 241, 241, 241, 241, - /* 160 */ 57, 52, 212, 241, 14, 13, 122, 1, 212, 212, - /* 170 */ 38, 5, 14, 13, 166, 40, 224, 20, 38, 5, - /* 180 */ 15, 210, 166, 211, 21, 22, 17, 210, 210, 43, - /* 190 */ 160, 6, 7, 45, 41, 26, 207, 48, 24, 208, - /* 200 */ 136, 44, 205, 8, 189, 3, 142, 2, 16, 127, - /* 210 */ 147, 146, 32, 150, 49, 149, 4, 33, 180, 206, - /* 220 */ 28, 117, 34, 186, 56, 162, 161, 185, 61, + /* 0 */ 384, 382, 89, 22, 64, 385, 382, 458, 29, 27, + /* 10 */ 25, 24, 23, 392, 382, 137, 406, 126, 406, 138, + /* 20 */ 71, 109, 44, 393, 259, 395, 21, 84, 93, 142, + /* 30 */ 277, 278, 279, 280, 281, 282, 283, 285, 286, 287, + /* 40 */ 29, 27, 25, 24, 23, 25, 24, 23, 125, 9, + /* 50 */ 456, 240, 241, 242, 243, 139, 246, 106, 21, 84, + /* 60 */ 93, 51, 277, 278, 279, 280, 281, 282, 283, 285, + /* 70 */ 286, 287, 127, 392, 382, 137, 406, 137, 406, 138, + /* 80 */ 315, 113, 38, 393, 114, 395, 431, 253, 28, 26, + /* 90 */ 83, 427, 487, 443, 131, 240, 241, 242, 243, 139, + /* 100 */ 246, 487, 98, 1, 340, 486, 67, 10, 51, 485, + /* 110 */ 440, 392, 382, 60, 50, 137, 406, 138, 485, 123, + /* 120 */ 39, 393, 314, 395, 431, 51, 125, 9, 91, 427, + /* 130 */ 105, 338, 339, 341, 342, 392, 382, 132, 462, 137, + /* 140 */ 406, 138, 4, 311, 39, 393, 407, 395, 431, 51, + /* 150 */ 443, 443, 91, 427, 29, 27, 25, 24, 23, 323, + /* 160 */ 392, 382, 483, 73, 137, 406, 138, 439, 438, 39, + /* 170 */ 393, 251, 395, 431, 389, 18, 387, 91, 427, 7, + /* 180 */ 6, 29, 27, 25, 24, 23, 134, 447, 127, 392, + /* 190 */ 382, 7, 6, 137, 406, 138, 28, 26, 77, 393, + /* 200 */ 8, 395, 295, 240, 241, 242, 243, 139, 246, 107, + /* 210 */ 98, 1, 28, 26, 469, 10, 104, 487, 52, 240, + /* 220 */ 241, 242, 243, 139, 246, 246, 98, 5, 102, 40, + /* 230 */ 50, 135, 337, 19, 485, 392, 382, 130, 468, 137, + /* 240 */ 406, 138, 103, 284, 39, 393, 288, 395, 431, 51, + /* 250 */ 56, 54, 430, 427, 57, 370, 371, 29, 27, 25, + /* 260 */ 24, 23, 30, 392, 382, 289, 90, 137, 406, 138, + /* 270 */ 3, 254, 39, 393, 117, 395, 431, 28, 26, 316, + /* 280 */ 118, 427, 47, 449, 240, 241, 242, 243, 139, 246, + /* 290 */ 70, 98, 5, 101, 392, 382, 129, 127, 126, 406, + /* 300 */ 138, 122, 43, 44, 393, 119, 395, 274, 30, 59, + /* 310 */ 41, 257, 2, 29, 27, 25, 24, 23, 61, 65, + /* 320 */ 436, 121, 15, 120, 69, 311, 487, 20, 413, 250, + /* 330 */ 99, 455, 42, 29, 27, 25, 24, 23, 253, 50, + /* 340 */ 28, 26, 444, 485, 31, 62, 254, 240, 241, 242, + /* 350 */ 243, 139, 246, 459, 98, 1, 392, 382, 94, 502, + /* 360 */ 137, 406, 138, 136, 484, 39, 393, 72, 395, 431, + /* 370 */ 28, 26, 367, 368, 428, 133, 251, 240, 241, 242, + /* 380 */ 243, 139, 246, 13, 98, 5, 29, 27, 25, 24, + /* 390 */ 23, 115, 110, 108, 392, 382, 12, 30, 137, 406, + /* 400 */ 138, 53, 259, 45, 393, 334, 395, 55, 34, 336, + /* 410 */ 330, 46, 58, 35, 392, 382, 373, 140, 137, 406, + /* 420 */ 138, 329, 111, 81, 393, 100, 395, 392, 382, 112, + /* 430 */ 387, 137, 406, 138, 36, 6, 81, 393, 116, 395, + /* 440 */ 128, 500, 14, 392, 382, 275, 487, 137, 406, 138, + /* 450 */ 309, 308, 81, 393, 92, 395, 392, 382, 33, 50, + /* 460 */ 137, 406, 138, 485, 66, 45, 393, 32, 395, 392, + /* 470 */ 382, 386, 49, 137, 406, 138, 361, 16, 81, 393, + /* 480 */ 97, 395, 37, 11, 356, 355, 74, 95, 360, 392, + /* 490 */ 382, 359, 96, 137, 406, 138, 244, 17, 78, 393, + /* 500 */ 376, 395, 375, 501, 374, 141, 392, 382, 374, 374, + /* 510 */ 137, 406, 138, 374, 374, 75, 393, 374, 395, 392, + /* 520 */ 382, 374, 374, 137, 406, 138, 374, 374, 79, 393, + /* 530 */ 374, 395, 392, 382, 374, 374, 137, 406, 138, 374, + /* 540 */ 374, 76, 393, 374, 395, 392, 382, 374, 374, 137, + /* 550 */ 406, 138, 374, 374, 80, 393, 374, 395, 374, 374, + /* 560 */ 374, 374, 374, 374, 392, 382, 374, 374, 137, 406, + /* 570 */ 138, 374, 374, 403, 393, 374, 395, 374, 392, 382, + /* 580 */ 374, 374, 137, 406, 138, 374, 374, 402, 393, 374, + /* 590 */ 395, 392, 382, 374, 374, 137, 406, 138, 374, 374, + /* 600 */ 401, 393, 374, 395, 392, 382, 374, 374, 137, 406, + /* 610 */ 138, 374, 374, 87, 393, 374, 395, 392, 382, 374, + /* 620 */ 374, 137, 406, 138, 374, 374, 86, 393, 374, 395, + /* 630 */ 392, 382, 374, 374, 137, 406, 138, 374, 374, 88, + /* 640 */ 393, 374, 395, 392, 382, 374, 374, 137, 406, 138, + /* 650 */ 374, 374, 85, 393, 374, 395, 374, 392, 382, 374, + /* 660 */ 374, 137, 406, 138, 374, 374, 82, 393, 374, 395, + /* 670 */ 122, 43, 374, 374, 374, 122, 43, 374, 374, 41, + /* 680 */ 374, 374, 122, 43, 41, 374, 374, 124, 63, 436, + /* 690 */ 437, 41, 441, 48, 436, 437, 374, 441, 374, 374, + /* 700 */ 68, 436, 437, 374, 441, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, - /* 10 */ 8, 9, 49, 10, 51, 52, 37, 38, 39, 40, - /* 20 */ 41, 42, 43, 44, 45, 46, 37, 38, 39, 40, - /* 30 */ 41, 42, 43, 44, 45, 46, 37, 38, 39, 40, - /* 40 */ 41, 42, 43, 44, 45, 46, 67, 68, 30, 31, - /* 50 */ 69, 52, 6, 7, 8, 9, 63, 68, 37, 38, - /* 60 */ 39, 40, 41, 42, 43, 44, 45, 46, 37, 38, - /* 70 */ 39, 40, 41, 42, 43, 44, 45, 46, 37, 38, - /* 80 */ 39, 40, 41, 42, 43, 44, 45, 46, 43, 12, - /* 90 */ 13, 35, 36, 16, 33, 34, 2, 12, 53, 54, - /* 100 */ 55, 56, 57, 58, 59, 60, 37, 22, 39, 40, - /* 110 */ 41, 42, 43, 19, 45, 46, 37, 61, 39, 40, - /* 120 */ 41, 42, 43, 15, 45, 46, 37, 63, 39, 40, - /* 130 */ 41, 42, 43, 32, 45, 46, 37, 29, 39, 40, - /* 140 */ 41, 42, 43, 8, 45, 46, 37, 12, 39, 40, - /* 150 */ 41, 42, 43, 1, 37, 46, 39, 40, 41, 42, - /* 160 */ 43, 36, 47, 46, 6, 7, 8, 48, 47, 47, - /* 170 */ 12, 13, 6, 7, 16, 23, 70, 62, 12, 13, - /* 180 */ 13, 66, 16, 62, 62, 18, 61, 66, 66, 15, - /* 190 */ 14, 15, 13, 25, 64, 64, 17, 28, 65, 65, - /* 200 */ 12, 27, 43, 21, 50, 15, 14, 24, 2, 12, - /* 210 */ 26, 26, 26, 26, 20, 26, 15, 26, 14, 17, - /* 220 */ 17, 12, 17, 0, 17, 14, 14, 0, 11, 71, - /* 230 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - /* 240 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - /* 250 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - /* 260 */ 71, 71, 71, 71, + /* 0 */ 74, 75, 76, 88, 89, 74, 75, 82, 8, 9, + /* 10 */ 10, 11, 12, 74, 75, 78, 79, 78, 79, 80, + /* 20 */ 122, 84, 83, 84, 24, 86, 26, 27, 28, 13, + /* 30 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + /* 40 */ 8, 9, 10, 11, 12, 10, 11, 12, 22, 23, + /* 50 */ 111, 15, 16, 17, 18, 19, 20, 114, 26, 27, + /* 60 */ 28, 45, 30, 31, 32, 33, 34, 35, 36, 37, + /* 70 */ 38, 39, 73, 74, 75, 78, 79, 78, 79, 80, + /* 80 */ 4, 84, 83, 84, 22, 86, 87, 22, 8, 9, + /* 90 */ 91, 92, 102, 81, 21, 15, 16, 17, 18, 19, + /* 100 */ 20, 102, 22, 23, 29, 115, 41, 27, 45, 119, + /* 110 */ 98, 74, 75, 107, 115, 78, 79, 80, 119, 100, + /* 120 */ 83, 84, 46, 86, 87, 45, 22, 23, 91, 92, + /* 130 */ 55, 56, 57, 58, 59, 74, 75, 64, 101, 78, + /* 140 */ 79, 80, 43, 44, 83, 84, 79, 86, 87, 45, + /* 150 */ 81, 81, 91, 92, 8, 9, 10, 11, 12, 10, + /* 160 */ 74, 75, 101, 116, 78, 79, 80, 98, 98, 83, + /* 170 */ 84, 22, 86, 87, 23, 2, 25, 91, 92, 1, + /* 180 */ 2, 8, 9, 10, 11, 12, 21, 101, 73, 74, + /* 190 */ 75, 1, 2, 78, 79, 80, 8, 9, 83, 84, + /* 200 */ 103, 86, 24, 15, 16, 17, 18, 19, 20, 54, + /* 210 */ 22, 23, 8, 9, 113, 27, 53, 102, 112, 15, + /* 220 */ 16, 17, 18, 19, 20, 20, 22, 23, 75, 21, + /* 230 */ 115, 66, 24, 26, 119, 74, 75, 3, 113, 78, + /* 240 */ 79, 80, 75, 36, 83, 84, 39, 86, 87, 45, + /* 250 */ 112, 21, 91, 92, 24, 70, 71, 8, 9, 10, + /* 260 */ 11, 12, 21, 74, 75, 24, 75, 78, 79, 80, + /* 270 */ 61, 22, 83, 84, 60, 86, 87, 8, 9, 10, + /* 280 */ 91, 92, 106, 109, 15, 16, 17, 18, 19, 20, + /* 290 */ 41, 22, 23, 48, 74, 75, 62, 73, 78, 79, + /* 300 */ 80, 77, 78, 83, 84, 27, 86, 29, 21, 108, + /* 310 */ 86, 24, 47, 8, 9, 10, 11, 12, 105, 95, + /* 320 */ 96, 97, 23, 99, 104, 44, 102, 2, 90, 22, + /* 330 */ 110, 111, 78, 8, 9, 10, 11, 12, 22, 115, + /* 340 */ 8, 9, 81, 119, 40, 93, 22, 15, 16, 17, + /* 350 */ 18, 19, 20, 82, 22, 23, 74, 75, 69, 123, + /* 360 */ 78, 79, 80, 65, 118, 83, 84, 117, 86, 87, + /* 370 */ 8, 9, 67, 68, 92, 63, 22, 15, 16, 17, + /* 380 */ 18, 19, 20, 49, 22, 23, 8, 9, 10, 11, + /* 390 */ 12, 50, 51, 52, 74, 75, 21, 21, 78, 79, + /* 400 */ 80, 24, 24, 83, 84, 24, 86, 23, 21, 24, + /* 410 */ 24, 23, 23, 23, 74, 75, 72, 73, 78, 79, + /* 420 */ 80, 24, 15, 83, 84, 85, 86, 74, 75, 21, + /* 430 */ 25, 78, 79, 80, 23, 2, 83, 84, 85, 86, + /* 440 */ 120, 121, 49, 74, 75, 29, 102, 78, 79, 80, + /* 450 */ 24, 24, 83, 84, 85, 86, 74, 75, 21, 115, + /* 460 */ 78, 79, 80, 119, 25, 83, 84, 42, 86, 74, + /* 470 */ 75, 25, 25, 78, 79, 80, 24, 21, 83, 84, + /* 480 */ 85, 86, 4, 49, 15, 15, 25, 15, 15, 74, + /* 490 */ 75, 15, 15, 78, 79, 80, 17, 23, 83, 84, + /* 500 */ 0, 86, 0, 121, 124, 14, 74, 75, 124, 124, + /* 510 */ 78, 79, 80, 124, 124, 83, 84, 124, 86, 74, + /* 520 */ 75, 124, 124, 78, 79, 80, 124, 124, 83, 84, + /* 530 */ 124, 86, 74, 75, 124, 124, 78, 79, 80, 124, + /* 540 */ 124, 83, 84, 124, 86, 74, 75, 124, 124, 78, + /* 550 */ 79, 80, 124, 124, 83, 84, 124, 86, 124, 124, + /* 560 */ 124, 124, 124, 124, 74, 75, 124, 124, 78, 79, + /* 570 */ 80, 124, 124, 83, 84, 124, 86, 124, 74, 75, + /* 580 */ 124, 124, 78, 79, 80, 124, 124, 83, 84, 124, + /* 590 */ 86, 74, 75, 124, 124, 78, 79, 80, 124, 124, + /* 600 */ 83, 84, 124, 86, 74, 75, 124, 124, 78, 79, + /* 610 */ 80, 124, 124, 83, 84, 124, 86, 74, 75, 124, + /* 620 */ 124, 78, 79, 80, 124, 124, 83, 84, 124, 86, + /* 630 */ 74, 75, 124, 124, 78, 79, 80, 124, 124, 83, + /* 640 */ 84, 124, 86, 74, 75, 124, 124, 78, 79, 80, + /* 650 */ 124, 124, 83, 84, 124, 86, 124, 74, 75, 124, + /* 660 */ 124, 78, 79, 80, 124, 124, 83, 84, 124, 86, + /* 670 */ 77, 78, 124, 124, 124, 77, 78, 124, 124, 86, + /* 680 */ 124, 124, 77, 78, 86, 124, 124, 94, 95, 96, + /* 690 */ 97, 86, 99, 95, 96, 97, 124, 99, 124, 124, + /* 700 */ 95, 96, 97, 124, 99, }; -#define YY_SHIFT_COUNT (62) +#define YY_SHIFT_COUNT (142) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (227) +#define YY_SHIFT_MAX (502) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 3, 158, 166, 166, 166, 166, 166, 166, 85, 166, - /* 10 */ 166, 166, 166, 77, 77, 167, 167, 167, 229, 18, - /* 20 */ 152, 152, 94, 101, 168, 169, 169, 168, 188, 182, - /* 30 */ 46, 61, 174, 108, 135, 2, 2, 176, 179, 190, - /* 40 */ 183, 192, 206, 184, 185, 186, 187, 189, 191, 197, - /* 50 */ 194, 201, 204, 202, 203, 205, 209, 207, 211, 212, - /* 60 */ 223, 227, 217, + /* 0 */ 16, 80, 188, 188, 188, 204, 188, 188, 269, 104, + /* 10 */ 332, 362, 362, 362, 362, 362, 362, 362, 362, 362, + /* 20 */ 362, 362, 362, 362, 362, 362, 362, 362, 362, 362, + /* 30 */ 362, 26, 26, 26, 36, 62, 62, 63, 0, 32, + /* 40 */ 36, 65, 65, 65, 249, 305, 75, 341, 99, 149, + /* 50 */ 234, 76, 155, 163, 205, 205, 155, 163, 205, 209, + /* 60 */ 214, 245, 265, 281, 299, 281, 307, 316, 281, 304, + /* 70 */ 324, 289, 298, 312, 354, 173, 325, 378, 146, 146, + /* 80 */ 146, 146, 146, 178, 207, 35, 35, 35, 35, 208, + /* 90 */ 230, 190, 241, 278, 185, 73, 165, 287, 151, 375, + /* 100 */ 376, 334, 377, 381, 384, 387, 385, 388, 389, 386, + /* 110 */ 390, 397, 407, 408, 405, 411, 376, 393, 433, 416, + /* 120 */ 426, 427, 439, 425, 437, 446, 447, 452, 456, 434, + /* 130 */ 478, 469, 470, 472, 473, 476, 477, 461, 474, 479, + /* 140 */ 500, 502, 491, }; -#define YY_REDUCE_COUNT (29) -#define YY_REDUCE_MIN (-37) -#define YY_REDUCE_MAX (159) +#define YY_REDUCE_COUNT (74) +#define YY_REDUCE_MIN (-102) +#define YY_REDUCE_MAX (605) static const short yy_reduce_ofst[] = { - /* 0 */ 56, -37, -21, -11, -1, 21, 31, 41, 45, 69, - /* 10 */ 79, 89, 99, 109, 117, 115, 121, 122, 125, -19, - /* 20 */ -7, 64, 119, 106, 130, 133, 134, 131, 159, 154, + /* 0 */ 344, -1, 37, 61, 86, 115, 161, 189, 220, 224, + /* 10 */ 282, 320, -61, 340, 353, 369, 382, 395, 415, 432, + /* 20 */ 445, 458, 471, 490, 504, 517, 530, 543, 556, 569, + /* 30 */ 583, 593, 598, 605, -74, -63, -3, -10, -85, -85, + /* 40 */ -69, 12, 69, 70, -75, -102, -57, 6, 19, 67, + /* 50 */ 47, 97, 101, 106, 153, 167, 125, 138, 191, 174, + /* 60 */ 201, 176, 213, 19, 238, 19, 254, 261, 19, 252, + /* 70 */ 271, 236, 246, 250, 67, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 209, 182, 182, 182, 182, 182, 182, 182, 182, 182, - /* 10 */ 182, 182, 182, 182, 182, 182, 182, 182, 209, 225, - /* 20 */ 214, 214, 190, 228, 216, 219, 219, 216, 182, 182, - /* 30 */ 239, 182, 182, 182, 182, 244, 243, 182, 187, 215, - /* 40 */ 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, - /* 50 */ 197, 194, 182, 207, 182, 182, 182, 182, 182, 182, - /* 60 */ 182, 182, 182, + /* 0 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 10 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 20 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 30 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 40 */ 372, 442, 442, 442, 457, 503, 372, 465, 372, 372, + /* 50 */ 488, 450, 472, 470, 372, 372, 472, 470, 372, 482, + /* 60 */ 480, 463, 461, 434, 372, 372, 372, 372, 435, 372, + /* 70 */ 372, 506, 494, 490, 372, 372, 372, 372, 410, 409, + /* 80 */ 408, 404, 405, 372, 372, 399, 400, 398, 397, 372, + /* 90 */ 372, 499, 372, 372, 372, 491, 495, 372, 388, 454, + /* 100 */ 464, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 110 */ 372, 372, 372, 372, 388, 372, 481, 372, 429, 372, + /* 120 */ 441, 437, 372, 372, 433, 387, 372, 372, 489, 372, + /* 130 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 140 */ 372, 372, 372, }; /********** End of lemon-generated parsing tables *****************************/ @@ -394,76 +518,129 @@ void NewParseTrace(FILE *TraceFILE, char *zTracePrompt){ ** are required. The following table supplies these names */ static const char *const yyTokenName[] = { /* 0 */ "$", - /* 1 */ "UNION", - /* 2 */ "ALL", - /* 3 */ "MINUS", - /* 4 */ "EXCEPT", - /* 5 */ "INTERSECT", - /* 6 */ "NK_PLUS", - /* 7 */ "NK_MINUS", - /* 8 */ "NK_STAR", - /* 9 */ "NK_SLASH", - /* 10 */ "SHOW", - /* 11 */ "DATABASES", - /* 12 */ "NK_ID", - /* 13 */ "NK_LP", - /* 14 */ "NK_RP", - /* 15 */ "NK_COMMA", - /* 16 */ "NK_LITERAL", - /* 17 */ "NK_DOT", - /* 18 */ "SELECT", - /* 19 */ "DISTINCT", - /* 20 */ "AS", - /* 21 */ "FROM", - /* 22 */ "NK_LR", - /* 23 */ "ORDER", - /* 24 */ "BY", - /* 25 */ "SLIMIT", - /* 26 */ "NK_INTEGER", - /* 27 */ "SOFFSET", - /* 28 */ "LIMIT", - /* 29 */ "OFFSET", - /* 30 */ "ASC", - /* 31 */ "DESC", - /* 32 */ "NULLS", - /* 33 */ "FIRST", - /* 34 */ "LAST", - /* 35 */ "cmd", - /* 36 */ "query_expression", - /* 37 */ "value_function", - /* 38 */ "value_expression", - /* 39 */ "value_expression_primary", - /* 40 */ "nonparenthesized_value_expression_primary", - /* 41 */ "literal", - /* 42 */ "column_reference", - /* 43 */ "table_name", - /* 44 */ "common_value_expression", - /* 45 */ "numeric_value_expression", - /* 46 */ "numeric_primary", - /* 47 */ "query_specification", - /* 48 */ "set_quantifier_opt", - /* 49 */ "select_list", - /* 50 */ "from_clause", - /* 51 */ "select_sublist", - /* 52 */ "select_item", - /* 53 */ "table_reference_list", - /* 54 */ "table_reference", - /* 55 */ "table_factor", - /* 56 */ "table_primary", - /* 57 */ "db_name", - /* 58 */ "derived_table", - /* 59 */ "table_subquery", - /* 60 */ "subquery", - /* 61 */ "with_clause_opt", - /* 62 */ "query_expression_body", - /* 63 */ "order_by_clause_opt", - /* 64 */ "slimit_clause_opt", - /* 65 */ "limit_clause_opt", - /* 66 */ "query_primary", - /* 67 */ "sort_specification_list", - /* 68 */ "sort_specification", - /* 69 */ "ordering_specification_opt", - /* 70 */ "null_ordering_opt", + /* 1 */ "OR", + /* 2 */ "AND", + /* 3 */ "UNION", + /* 4 */ "ALL", + /* 5 */ "MINUS", + /* 6 */ "EXCEPT", + /* 7 */ "INTERSECT", + /* 8 */ "NK_PLUS", + /* 9 */ "NK_MINUS", + /* 10 */ "NK_STAR", + /* 11 */ "NK_SLASH", + /* 12 */ "NK_REM", + /* 13 */ "SHOW", + /* 14 */ "DATABASES", + /* 15 */ "NK_INTEGER", + /* 16 */ "NK_FLOAT", + /* 17 */ "NK_STRING", + /* 18 */ "NK_BOOL", + /* 19 */ "TIMESTAMP", + /* 20 */ "NK_VARIABLE", + /* 21 */ "NK_COMMA", + /* 22 */ "NK_ID", + /* 23 */ "NK_LP", + /* 24 */ "NK_RP", + /* 25 */ "NK_DOT", + /* 26 */ "BETWEEN", + /* 27 */ "NOT", + /* 28 */ "IS", + /* 29 */ "NULL", + /* 30 */ "NK_LT", + /* 31 */ "NK_GT", + /* 32 */ "NK_LE", + /* 33 */ "NK_GE", + /* 34 */ "NK_NE", + /* 35 */ "NK_EQ", + /* 36 */ "LIKE", + /* 37 */ "MATCH", + /* 38 */ "NMATCH", + /* 39 */ "IN", + /* 40 */ "FROM", + /* 41 */ "AS", + /* 42 */ "JOIN", + /* 43 */ "ON", + /* 44 */ "INNER", + /* 45 */ "SELECT", + /* 46 */ "DISTINCT", + /* 47 */ "WHERE", + /* 48 */ "PARTITION", + /* 49 */ "BY", + /* 50 */ "SESSION", + /* 51 */ "STATE_WINDOW", + /* 52 */ "INTERVAL", + /* 53 */ "SLIDING", + /* 54 */ "FILL", + /* 55 */ "VALUE", + /* 56 */ "NONE", + /* 57 */ "PREV", + /* 58 */ "LINEAR", + /* 59 */ "NEXT", + /* 60 */ "GROUP", + /* 61 */ "HAVING", + /* 62 */ "ORDER", + /* 63 */ "SLIMIT", + /* 64 */ "SOFFSET", + /* 65 */ "LIMIT", + /* 66 */ "OFFSET", + /* 67 */ "ASC", + /* 68 */ "DESC", + /* 69 */ "NULLS", + /* 70 */ "FIRST", + /* 71 */ "LAST", + /* 72 */ "cmd", + /* 73 */ "query_expression", + /* 74 */ "literal", + /* 75 */ "duration_literal", + /* 76 */ "literal_list", + /* 77 */ "db_name", + /* 78 */ "table_name", + /* 79 */ "column_name", + /* 80 */ "function_name", + /* 81 */ "table_alias", + /* 82 */ "column_alias", + /* 83 */ "expression", + /* 84 */ "column_reference", + /* 85 */ "expression_list", + /* 86 */ "subquery", + /* 87 */ "predicate", + /* 88 */ "compare_op", + /* 89 */ "in_op", + /* 90 */ "in_predicate_value", + /* 91 */ "boolean_value_expression", + /* 92 */ "boolean_primary", + /* 93 */ "from_clause", + /* 94 */ "table_reference_list", + /* 95 */ "table_reference", + /* 96 */ "table_primary", + /* 97 */ "joined_table", + /* 98 */ "alias_opt", + /* 99 */ "parenthesized_joined_table", + /* 100 */ "join_type", + /* 101 */ "search_condition", + /* 102 */ "query_specification", + /* 103 */ "set_quantifier_opt", + /* 104 */ "select_list", + /* 105 */ "where_clause_opt", + /* 106 */ "partition_by_clause_opt", + /* 107 */ "twindow_clause_opt", + /* 108 */ "group_by_clause_opt", + /* 109 */ "having_clause_opt", + /* 110 */ "select_sublist", + /* 111 */ "select_item", + /* 112 */ "sliding_opt", + /* 113 */ "fill_opt", + /* 114 */ "fill_mode", + /* 115 */ "query_expression_body", + /* 116 */ "order_by_clause_opt", + /* 117 */ "slimit_clause_opt", + /* 118 */ "limit_clause_opt", + /* 119 */ "query_primary", + /* 120 */ "sort_specification_list", + /* 121 */ "sort_specification", + /* 122 */ "ordering_specification_opt", + /* 123 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -473,72 +650,138 @@ static const char *const yyTokenName[] = { static const char *const yyRuleName[] = { /* 0 */ "cmd ::= SHOW DATABASES", /* 1 */ "cmd ::= query_expression", - /* 2 */ "column_reference ::= NK_ID", - /* 3 */ "column_reference ::= table_name NK_DOT NK_ID", - /* 4 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause", - /* 5 */ "set_quantifier_opt ::=", - /* 6 */ "set_quantifier_opt ::= DISTINCT", - /* 7 */ "set_quantifier_opt ::= ALL", - /* 8 */ "select_list ::= NK_STAR", - /* 9 */ "select_list ::= select_sublist", - /* 10 */ "select_sublist ::= select_item", - /* 11 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 12 */ "select_item ::= value_expression", - /* 13 */ "select_item ::= value_expression AS NK_ID", - /* 14 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 15 */ "from_clause ::= FROM table_reference_list", - /* 16 */ "table_reference_list ::= table_reference", - /* 17 */ "table_reference ::= table_factor", - /* 18 */ "table_factor ::= table_primary", - /* 19 */ "table_primary ::= table_name", - /* 20 */ "table_primary ::= db_name NK_DOT table_name", - /* 21 */ "db_name ::= NK_ID", - /* 22 */ "table_name ::= NK_ID", - /* 23 */ "query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 24 */ "with_clause_opt ::=", - /* 25 */ "query_expression_body ::= query_primary", - /* 26 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 27 */ "query_primary ::= query_specification", - /* 28 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP", - /* 29 */ "order_by_clause_opt ::=", - /* 30 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 31 */ "slimit_clause_opt ::=", - /* 32 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 33 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 34 */ "limit_clause_opt ::=", - /* 35 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 36 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 37 */ "sort_specification_list ::= sort_specification", - /* 38 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 39 */ "sort_specification ::= value_expression ordering_specification_opt null_ordering_opt", - /* 40 */ "ordering_specification_opt ::=", - /* 41 */ "ordering_specification_opt ::= ASC", - /* 42 */ "ordering_specification_opt ::= DESC", - /* 43 */ "null_ordering_opt ::=", - /* 44 */ "null_ordering_opt ::= NULLS FIRST", - /* 45 */ "null_ordering_opt ::= NULLS LAST", - /* 46 */ "value_function ::= NK_ID NK_LP value_expression NK_RP", - /* 47 */ "value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP", - /* 48 */ "value_expression_primary ::= NK_LP value_expression NK_RP", - /* 49 */ "value_expression_primary ::= nonparenthesized_value_expression_primary", - /* 50 */ "nonparenthesized_value_expression_primary ::= literal", - /* 51 */ "nonparenthesized_value_expression_primary ::= column_reference", - /* 52 */ "literal ::= NK_LITERAL", - /* 53 */ "value_expression ::= common_value_expression", - /* 54 */ "common_value_expression ::= numeric_value_expression", - /* 55 */ "numeric_value_expression ::= numeric_primary", - /* 56 */ "numeric_value_expression ::= NK_PLUS numeric_primary", - /* 57 */ "numeric_value_expression ::= NK_MINUS numeric_primary", - /* 58 */ "numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression", - /* 59 */ "numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression", - /* 60 */ "numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression", - /* 61 */ "numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression", - /* 62 */ "numeric_primary ::= value_expression_primary", - /* 63 */ "numeric_primary ::= value_function", - /* 64 */ "table_primary ::= derived_table", - /* 65 */ "derived_table ::= table_subquery", - /* 66 */ "subquery ::= NK_LR query_expression NK_RP", - /* 67 */ "table_subquery ::= subquery", + /* 2 */ "literal ::= NK_INTEGER", + /* 3 */ "literal ::= NK_FLOAT", + /* 4 */ "literal ::= NK_STRING", + /* 5 */ "literal ::= NK_BOOL", + /* 6 */ "literal ::= TIMESTAMP NK_STRING", + /* 7 */ "literal ::= duration_literal", + /* 8 */ "duration_literal ::= NK_VARIABLE", + /* 9 */ "literal_list ::= literal", + /* 10 */ "literal_list ::= literal_list NK_COMMA literal", + /* 11 */ "db_name ::= NK_ID", + /* 12 */ "table_name ::= NK_ID", + /* 13 */ "column_name ::= NK_ID", + /* 14 */ "function_name ::= NK_ID", + /* 15 */ "table_alias ::= NK_ID", + /* 16 */ "column_alias ::= NK_ID", + /* 17 */ "expression ::= literal", + /* 18 */ "expression ::= column_reference", + /* 19 */ "expression ::= function_name NK_LP expression_list NK_RP", + /* 20 */ "expression ::= subquery", + /* 21 */ "expression ::= NK_LP expression NK_RP", + /* 22 */ "expression ::= NK_PLUS expression", + /* 23 */ "expression ::= NK_MINUS expression", + /* 24 */ "expression ::= expression NK_PLUS expression", + /* 25 */ "expression ::= expression NK_MINUS expression", + /* 26 */ "expression ::= expression NK_STAR expression", + /* 27 */ "expression ::= expression NK_SLASH expression", + /* 28 */ "expression ::= expression NK_REM expression", + /* 29 */ "expression_list ::= expression", + /* 30 */ "expression_list ::= expression_list NK_COMMA expression", + /* 31 */ "column_reference ::= column_name", + /* 32 */ "column_reference ::= table_name NK_DOT column_name", + /* 33 */ "predicate ::= expression compare_op expression", + /* 34 */ "predicate ::= expression BETWEEN expression AND expression", + /* 35 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 36 */ "predicate ::= expression IS NULL", + /* 37 */ "predicate ::= expression IS NOT NULL", + /* 38 */ "predicate ::= expression in_op in_predicate_value", + /* 39 */ "compare_op ::= NK_LT", + /* 40 */ "compare_op ::= NK_GT", + /* 41 */ "compare_op ::= NK_LE", + /* 42 */ "compare_op ::= NK_GE", + /* 43 */ "compare_op ::= NK_NE", + /* 44 */ "compare_op ::= NK_EQ", + /* 45 */ "compare_op ::= LIKE", + /* 46 */ "compare_op ::= NOT LIKE", + /* 47 */ "compare_op ::= MATCH", + /* 48 */ "compare_op ::= NMATCH", + /* 49 */ "in_op ::= IN", + /* 50 */ "in_op ::= NOT IN", + /* 51 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 52 */ "boolean_value_expression ::= boolean_primary", + /* 53 */ "boolean_value_expression ::= NOT boolean_primary", + /* 54 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 55 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 56 */ "boolean_primary ::= predicate", + /* 57 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 58 */ "from_clause ::= FROM table_reference_list", + /* 59 */ "table_reference_list ::= table_reference", + /* 60 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 61 */ "table_reference ::= table_primary", + /* 62 */ "table_reference ::= joined_table", + /* 63 */ "table_primary ::= table_name alias_opt", + /* 64 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 65 */ "table_primary ::= subquery alias_opt", + /* 66 */ "table_primary ::= parenthesized_joined_table", + /* 67 */ "alias_opt ::=", + /* 68 */ "alias_opt ::= table_alias", + /* 69 */ "alias_opt ::= AS table_alias", + /* 70 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 71 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 72 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 73 */ "join_type ::= INNER", + /* 74 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 75 */ "set_quantifier_opt ::=", + /* 76 */ "set_quantifier_opt ::= DISTINCT", + /* 77 */ "set_quantifier_opt ::= ALL", + /* 78 */ "select_list ::= NK_STAR", + /* 79 */ "select_list ::= select_sublist", + /* 80 */ "select_sublist ::= select_item", + /* 81 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 82 */ "select_item ::= expression", + /* 83 */ "select_item ::= expression column_alias", + /* 84 */ "select_item ::= expression AS column_alias", + /* 85 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 86 */ "where_clause_opt ::=", + /* 87 */ "where_clause_opt ::= WHERE search_condition", + /* 88 */ "partition_by_clause_opt ::=", + /* 89 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 90 */ "twindow_clause_opt ::=", + /* 91 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 92 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 93 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 94 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 95 */ "sliding_opt ::=", + /* 96 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 97 */ "fill_opt ::=", + /* 98 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 99 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 100 */ "fill_mode ::= NONE", + /* 101 */ "fill_mode ::= PREV", + /* 102 */ "fill_mode ::= NULL", + /* 103 */ "fill_mode ::= LINEAR", + /* 104 */ "fill_mode ::= NEXT", + /* 105 */ "group_by_clause_opt ::=", + /* 106 */ "group_by_clause_opt ::= GROUP BY expression_list", + /* 107 */ "having_clause_opt ::=", + /* 108 */ "having_clause_opt ::= HAVING search_condition", + /* 109 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 110 */ "query_expression_body ::= query_primary", + /* 111 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 112 */ "query_primary ::= query_specification", + /* 113 */ "order_by_clause_opt ::=", + /* 114 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 115 */ "slimit_clause_opt ::=", + /* 116 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 117 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 118 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 119 */ "limit_clause_opt ::=", + /* 120 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 121 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 122 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 123 */ "subquery ::= NK_LP query_expression NK_RP", + /* 124 */ "search_condition ::= boolean_value_expression", + /* 125 */ "sort_specification_list ::= sort_specification", + /* 126 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 127 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 128 */ "ordering_specification_opt ::=", + /* 129 */ "ordering_specification_opt ::= ASC", + /* 130 */ "ordering_specification_opt ::= DESC", + /* 131 */ "null_ordering_opt ::=", + /* 132 */ "null_ordering_opt ::= NULLS FIRST", + /* 133 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -665,60 +908,92 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 35: /* cmd */ - case 36: /* query_expression */ - case 37: /* value_function */ - case 38: /* value_expression */ - case 39: /* value_expression_primary */ - case 40: /* nonparenthesized_value_expression_primary */ - case 41: /* literal */ - case 42: /* column_reference */ - case 43: /* table_name */ - case 44: /* common_value_expression */ - case 45: /* numeric_value_expression */ - case 46: /* numeric_primary */ - case 47: /* query_specification */ - case 50: /* from_clause */ - case 52: /* select_item */ - case 53: /* table_reference_list */ - case 54: /* table_reference */ - case 55: /* table_factor */ - case 56: /* table_primary */ - case 57: /* db_name */ - case 58: /* derived_table */ - case 59: /* table_subquery */ - case 60: /* subquery */ - case 61: /* with_clause_opt */ - case 62: /* query_expression_body */ - case 64: /* slimit_clause_opt */ - case 65: /* limit_clause_opt */ - case 66: /* query_primary */ - case 68: /* sort_specification */ + case 72: /* cmd */ + case 73: /* query_expression */ + case 74: /* literal */ + case 75: /* duration_literal */ + case 83: /* expression */ + case 84: /* column_reference */ + case 86: /* subquery */ + case 87: /* predicate */ + case 90: /* in_predicate_value */ + case 91: /* boolean_value_expression */ + case 92: /* boolean_primary */ + case 93: /* from_clause */ + case 94: /* table_reference_list */ + case 95: /* table_reference */ + case 96: /* table_primary */ + case 97: /* joined_table */ + case 99: /* parenthesized_joined_table */ + case 101: /* search_condition */ + case 102: /* query_specification */ + case 105: /* where_clause_opt */ + case 107: /* twindow_clause_opt */ + case 109: /* having_clause_opt */ + case 111: /* select_item */ + case 112: /* sliding_opt */ + case 113: /* fill_opt */ + case 115: /* query_expression_body */ + case 117: /* slimit_clause_opt */ + case 118: /* limit_clause_opt */ + case 119: /* query_primary */ + case 121: /* sort_specification */ { - nodesDestroyNode((yypminor->yy130)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy168)); } break; - case 48: /* set_quantifier_opt */ + case 76: /* literal_list */ + case 85: /* expression_list */ + case 104: /* select_list */ + case 106: /* partition_by_clause_opt */ + case 108: /* group_by_clause_opt */ + case 110: /* select_sublist */ + case 116: /* order_by_clause_opt */ + case 120: /* sort_specification_list */ { - + PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy192)); } break; - case 49: /* select_list */ - case 51: /* select_sublist */ - case 63: /* order_by_clause_opt */ - case 67: /* sort_specification_list */ + case 77: /* db_name */ + case 78: /* table_name */ + case 79: /* column_name */ + case 80: /* function_name */ + case 81: /* table_alias */ + case 82: /* column_alias */ + case 98: /* alias_opt */ { - nodesDestroyNodeList((yypminor->yy30)); + PARSER_DESTRUCTOR_TRACE; } break; - case 69: /* ordering_specification_opt */ + case 88: /* compare_op */ + case 89: /* in_op */ { - + PARSER_DESTRUCTOR_TRACE; } break; - case 70: /* null_ordering_opt */ + case 100: /* join_type */ { - + PARSER_DESTRUCTOR_TRACE; +} + break; + case 103: /* set_quantifier_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 114: /* fill_mode */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 122: /* ordering_specification_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 123: /* null_ordering_opt */ +{ + PARSER_DESTRUCTOR_TRACE; } break; /********* End destructor definitions *****************************************/ @@ -1015,74 +1290,140 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 35, -2 }, /* (0) cmd ::= SHOW DATABASES */ - { 35, -1 }, /* (1) cmd ::= query_expression */ - { 42, -1 }, /* (2) column_reference ::= NK_ID */ - { 42, -3 }, /* (3) column_reference ::= table_name NK_DOT NK_ID */ - { 47, -4 }, /* (4) query_specification ::= SELECT set_quantifier_opt select_list from_clause */ - { 48, 0 }, /* (5) set_quantifier_opt ::= */ - { 48, -1 }, /* (6) set_quantifier_opt ::= DISTINCT */ - { 48, -1 }, /* (7) set_quantifier_opt ::= ALL */ - { 49, -1 }, /* (8) select_list ::= NK_STAR */ - { 49, -1 }, /* (9) select_list ::= select_sublist */ - { 51, -1 }, /* (10) select_sublist ::= select_item */ - { 51, -3 }, /* (11) select_sublist ::= select_sublist NK_COMMA select_item */ - { 52, -1 }, /* (12) select_item ::= value_expression */ - { 52, -3 }, /* (13) select_item ::= value_expression AS NK_ID */ - { 52, -3 }, /* (14) select_item ::= table_name NK_DOT NK_STAR */ - { 50, -2 }, /* (15) from_clause ::= FROM table_reference_list */ - { 53, -1 }, /* (16) table_reference_list ::= table_reference */ - { 54, -1 }, /* (17) table_reference ::= table_factor */ - { 55, -1 }, /* (18) table_factor ::= table_primary */ - { 56, -1 }, /* (19) table_primary ::= table_name */ - { 56, -3 }, /* (20) table_primary ::= db_name NK_DOT table_name */ - { 57, -1 }, /* (21) db_name ::= NK_ID */ - { 43, -1 }, /* (22) table_name ::= NK_ID */ - { 36, -5 }, /* (23) query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 61, 0 }, /* (24) with_clause_opt ::= */ - { 62, -1 }, /* (25) query_expression_body ::= query_primary */ - { 62, -4 }, /* (26) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 66, -1 }, /* (27) query_primary ::= query_specification */ - { 66, -6 }, /* (28) query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ - { 63, 0 }, /* (29) order_by_clause_opt ::= */ - { 63, -3 }, /* (30) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 64, 0 }, /* (31) slimit_clause_opt ::= */ - { 64, -4 }, /* (32) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 64, -4 }, /* (33) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 65, 0 }, /* (34) limit_clause_opt ::= */ - { 65, -4 }, /* (35) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 65, -4 }, /* (36) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 67, -1 }, /* (37) sort_specification_list ::= sort_specification */ - { 67, -3 }, /* (38) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 68, -3 }, /* (39) sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ - { 69, 0 }, /* (40) ordering_specification_opt ::= */ - { 69, -1 }, /* (41) ordering_specification_opt ::= ASC */ - { 69, -1 }, /* (42) ordering_specification_opt ::= DESC */ - { 70, 0 }, /* (43) null_ordering_opt ::= */ - { 70, -2 }, /* (44) null_ordering_opt ::= NULLS FIRST */ - { 70, -2 }, /* (45) null_ordering_opt ::= NULLS LAST */ - { 37, -4 }, /* (46) value_function ::= NK_ID NK_LP value_expression NK_RP */ - { 37, -6 }, /* (47) value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ - { 39, -3 }, /* (48) value_expression_primary ::= NK_LP value_expression NK_RP */ - { 39, -1 }, /* (49) value_expression_primary ::= nonparenthesized_value_expression_primary */ - { 40, -1 }, /* (50) nonparenthesized_value_expression_primary ::= literal */ - { 40, -1 }, /* (51) nonparenthesized_value_expression_primary ::= column_reference */ - { 41, -1 }, /* (52) literal ::= NK_LITERAL */ - { 38, -1 }, /* (53) value_expression ::= common_value_expression */ - { 44, -1 }, /* (54) common_value_expression ::= numeric_value_expression */ - { 45, -1 }, /* (55) numeric_value_expression ::= numeric_primary */ - { 45, -2 }, /* (56) numeric_value_expression ::= NK_PLUS numeric_primary */ - { 45, -2 }, /* (57) numeric_value_expression ::= NK_MINUS numeric_primary */ - { 45, -3 }, /* (58) numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ - { 45, -3 }, /* (59) numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ - { 45, -3 }, /* (60) numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ - { 45, -3 }, /* (61) numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ - { 46, -1 }, /* (62) numeric_primary ::= value_expression_primary */ - { 46, -1 }, /* (63) numeric_primary ::= value_function */ - { 56, -1 }, /* (64) table_primary ::= derived_table */ - { 58, -1 }, /* (65) derived_table ::= table_subquery */ - { 60, -3 }, /* (66) subquery ::= NK_LR query_expression NK_RP */ - { 59, -1 }, /* (67) table_subquery ::= subquery */ + { 72, -2 }, /* (0) cmd ::= SHOW DATABASES */ + { 72, -1 }, /* (1) cmd ::= query_expression */ + { 74, -1 }, /* (2) literal ::= NK_INTEGER */ + { 74, -1 }, /* (3) literal ::= NK_FLOAT */ + { 74, -1 }, /* (4) literal ::= NK_STRING */ + { 74, -1 }, /* (5) literal ::= NK_BOOL */ + { 74, -2 }, /* (6) literal ::= TIMESTAMP NK_STRING */ + { 74, -1 }, /* (7) literal ::= duration_literal */ + { 75, -1 }, /* (8) duration_literal ::= NK_VARIABLE */ + { 76, -1 }, /* (9) literal_list ::= literal */ + { 76, -3 }, /* (10) literal_list ::= literal_list NK_COMMA literal */ + { 77, -1 }, /* (11) db_name ::= NK_ID */ + { 78, -1 }, /* (12) table_name ::= NK_ID */ + { 79, -1 }, /* (13) column_name ::= NK_ID */ + { 80, -1 }, /* (14) function_name ::= NK_ID */ + { 81, -1 }, /* (15) table_alias ::= NK_ID */ + { 82, -1 }, /* (16) column_alias ::= NK_ID */ + { 83, -1 }, /* (17) expression ::= literal */ + { 83, -1 }, /* (18) expression ::= column_reference */ + { 83, -4 }, /* (19) expression ::= function_name NK_LP expression_list NK_RP */ + { 83, -1 }, /* (20) expression ::= subquery */ + { 83, -3 }, /* (21) expression ::= NK_LP expression NK_RP */ + { 83, -2 }, /* (22) expression ::= NK_PLUS expression */ + { 83, -2 }, /* (23) expression ::= NK_MINUS expression */ + { 83, -3 }, /* (24) expression ::= expression NK_PLUS expression */ + { 83, -3 }, /* (25) expression ::= expression NK_MINUS expression */ + { 83, -3 }, /* (26) expression ::= expression NK_STAR expression */ + { 83, -3 }, /* (27) expression ::= expression NK_SLASH expression */ + { 83, -3 }, /* (28) expression ::= expression NK_REM expression */ + { 85, -1 }, /* (29) expression_list ::= expression */ + { 85, -3 }, /* (30) expression_list ::= expression_list NK_COMMA expression */ + { 84, -1 }, /* (31) column_reference ::= column_name */ + { 84, -3 }, /* (32) column_reference ::= table_name NK_DOT column_name */ + { 87, -3 }, /* (33) predicate ::= expression compare_op expression */ + { 87, -5 }, /* (34) predicate ::= expression BETWEEN expression AND expression */ + { 87, -6 }, /* (35) predicate ::= expression NOT BETWEEN expression AND expression */ + { 87, -3 }, /* (36) predicate ::= expression IS NULL */ + { 87, -4 }, /* (37) predicate ::= expression IS NOT NULL */ + { 87, -3 }, /* (38) predicate ::= expression in_op in_predicate_value */ + { 88, -1 }, /* (39) compare_op ::= NK_LT */ + { 88, -1 }, /* (40) compare_op ::= NK_GT */ + { 88, -1 }, /* (41) compare_op ::= NK_LE */ + { 88, -1 }, /* (42) compare_op ::= NK_GE */ + { 88, -1 }, /* (43) compare_op ::= NK_NE */ + { 88, -1 }, /* (44) compare_op ::= NK_EQ */ + { 88, -1 }, /* (45) compare_op ::= LIKE */ + { 88, -2 }, /* (46) compare_op ::= NOT LIKE */ + { 88, -1 }, /* (47) compare_op ::= MATCH */ + { 88, -1 }, /* (48) compare_op ::= NMATCH */ + { 89, -1 }, /* (49) in_op ::= IN */ + { 89, -2 }, /* (50) in_op ::= NOT IN */ + { 90, -3 }, /* (51) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 91, -1 }, /* (52) boolean_value_expression ::= boolean_primary */ + { 91, -2 }, /* (53) boolean_value_expression ::= NOT boolean_primary */ + { 91, -3 }, /* (54) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 91, -3 }, /* (55) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 92, -1 }, /* (56) boolean_primary ::= predicate */ + { 92, -3 }, /* (57) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 93, -2 }, /* (58) from_clause ::= FROM table_reference_list */ + { 94, -1 }, /* (59) table_reference_list ::= table_reference */ + { 94, -3 }, /* (60) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 95, -1 }, /* (61) table_reference ::= table_primary */ + { 95, -1 }, /* (62) table_reference ::= joined_table */ + { 96, -2 }, /* (63) table_primary ::= table_name alias_opt */ + { 96, -4 }, /* (64) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 96, -2 }, /* (65) table_primary ::= subquery alias_opt */ + { 96, -1 }, /* (66) table_primary ::= parenthesized_joined_table */ + { 98, 0 }, /* (67) alias_opt ::= */ + { 98, -1 }, /* (68) alias_opt ::= table_alias */ + { 98, -2 }, /* (69) alias_opt ::= AS table_alias */ + { 99, -3 }, /* (70) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 99, -3 }, /* (71) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 97, -6 }, /* (72) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 100, -1 }, /* (73) join_type ::= INNER */ + { 102, -9 }, /* (74) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 103, 0 }, /* (75) set_quantifier_opt ::= */ + { 103, -1 }, /* (76) set_quantifier_opt ::= DISTINCT */ + { 103, -1 }, /* (77) set_quantifier_opt ::= ALL */ + { 104, -1 }, /* (78) select_list ::= NK_STAR */ + { 104, -1 }, /* (79) select_list ::= select_sublist */ + { 110, -1 }, /* (80) select_sublist ::= select_item */ + { 110, -3 }, /* (81) select_sublist ::= select_sublist NK_COMMA select_item */ + { 111, -1 }, /* (82) select_item ::= expression */ + { 111, -2 }, /* (83) select_item ::= expression column_alias */ + { 111, -3 }, /* (84) select_item ::= expression AS column_alias */ + { 111, -3 }, /* (85) select_item ::= table_name NK_DOT NK_STAR */ + { 105, 0 }, /* (86) where_clause_opt ::= */ + { 105, -2 }, /* (87) where_clause_opt ::= WHERE search_condition */ + { 106, 0 }, /* (88) partition_by_clause_opt ::= */ + { 106, -3 }, /* (89) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 107, 0 }, /* (90) twindow_clause_opt ::= */ + { 107, -6 }, /* (91) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 107, -4 }, /* (92) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 107, -6 }, /* (93) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 107, -8 }, /* (94) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 112, 0 }, /* (95) sliding_opt ::= */ + { 112, -4 }, /* (96) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 113, 0 }, /* (97) fill_opt ::= */ + { 113, -4 }, /* (98) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 113, -6 }, /* (99) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 114, -1 }, /* (100) fill_mode ::= NONE */ + { 114, -1 }, /* (101) fill_mode ::= PREV */ + { 114, -1 }, /* (102) fill_mode ::= NULL */ + { 114, -1 }, /* (103) fill_mode ::= LINEAR */ + { 114, -1 }, /* (104) fill_mode ::= NEXT */ + { 108, 0 }, /* (105) group_by_clause_opt ::= */ + { 108, -3 }, /* (106) group_by_clause_opt ::= GROUP BY expression_list */ + { 109, 0 }, /* (107) having_clause_opt ::= */ + { 109, -2 }, /* (108) having_clause_opt ::= HAVING search_condition */ + { 73, -4 }, /* (109) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 115, -1 }, /* (110) query_expression_body ::= query_primary */ + { 115, -4 }, /* (111) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 119, -1 }, /* (112) query_primary ::= query_specification */ + { 116, 0 }, /* (113) order_by_clause_opt ::= */ + { 116, -3 }, /* (114) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 117, 0 }, /* (115) slimit_clause_opt ::= */ + { 117, -2 }, /* (116) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 117, -4 }, /* (117) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 117, -4 }, /* (118) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 118, 0 }, /* (119) limit_clause_opt ::= */ + { 118, -2 }, /* (120) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 118, -4 }, /* (121) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 118, -4 }, /* (122) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 86, -3 }, /* (123) subquery ::= NK_LP query_expression NK_RP */ + { 101, -1 }, /* (124) search_condition ::= boolean_value_expression */ + { 120, -1 }, /* (125) sort_specification_list ::= sort_specification */ + { 120, -3 }, /* (126) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 121, -3 }, /* (127) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 122, 0 }, /* (128) ordering_specification_opt ::= */ + { 122, -1 }, /* (129) ordering_specification_opt ::= ASC */ + { 122, -1 }, /* (130) ordering_specification_opt ::= DESC */ + { 123, 0 }, /* (131) null_ordering_opt ::= */ + { 123, -2 }, /* (132) null_ordering_opt ::= NULLS FIRST */ + { 123, -2 }, /* (133) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1173,240 +1514,425 @@ static YYACTIONTYPE yy_reduce( { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } break; case 1: /* cmd ::= query_expression */ -{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy130; } +{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy168; } break; - case 2: /* column_reference ::= NK_ID */ -{ PARSER_TRACE; yylhsminor.yy130 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy130 = yylhsminor.yy130; + case 2: /* literal ::= NK_INTEGER */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 3: /* column_reference ::= table_name NK_DOT NK_ID */ - case 14: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==14); -{ PARSER_TRACE; yylhsminor.yy130 = createColumnNode(pCxt, &yymsp[-2].minor.yy67, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 3: /* literal ::= NK_FLOAT */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 4: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause */ -{ PARSER_TRACE; yymsp[-3].minor.yy130 = createSelectStmt(pCxt, yymsp[-2].minor.yy9, yymsp[-1].minor.yy30, yymsp[0].minor.yy130); } + case 4: /* literal ::= NK_STRING */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 5: /* set_quantifier_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy9 = false; } + case 5: /* literal ::= NK_BOOL */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 6: /* set_quantifier_opt ::= DISTINCT */ -{ PARSER_TRACE; yymsp[0].minor.yy9 = true; } + case 6: /* literal ::= TIMESTAMP NK_STRING */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; break; - case 7: /* set_quantifier_opt ::= ALL */ -{ PARSER_TRACE; yymsp[0].minor.yy9 = false; } + case 7: /* literal ::= duration_literal */ + case 17: /* expression ::= literal */ yytestcase(yyruleno==17); + case 18: /* expression ::= column_reference */ yytestcase(yyruleno==18); + case 20: /* expression ::= subquery */ yytestcase(yyruleno==20); + case 52: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==52); + case 56: /* boolean_primary ::= predicate */ yytestcase(yyruleno==56); + case 59: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==59); + case 61: /* table_reference ::= table_primary */ yytestcase(yyruleno==61); + case 62: /* table_reference ::= joined_table */ yytestcase(yyruleno==62); + case 66: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==66); + case 110: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==110); + case 112: /* query_primary ::= query_specification */ yytestcase(yyruleno==112); + case 124: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==124); +{ PARSER_TRACE; yylhsminor.yy168 = yymsp[0].minor.yy168; } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 8: /* select_list ::= NK_STAR */ -{ PARSER_TRACE; yymsp[0].minor.yy30 = NULL; } + case 8: /* duration_literal ::= NK_VARIABLE */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 9: /* select_list ::= select_sublist */ -{ PARSER_TRACE; yylhsminor.yy30 = yymsp[0].minor.yy30; } - yymsp[0].minor.yy30 = yylhsminor.yy30; + case 9: /* literal_list ::= literal */ + case 29: /* expression_list ::= expression */ yytestcase(yyruleno==29); +{ PARSER_TRACE; yylhsminor.yy192 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } + yymsp[0].minor.yy192 = yylhsminor.yy192; break; - case 10: /* select_sublist ::= select_item */ - case 37: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==37); -{ PARSER_TRACE; yylhsminor.yy30 = createNodeList(pCxt, yymsp[0].minor.yy130); } - yymsp[0].minor.yy30 = yylhsminor.yy30; + case 10: /* literal_list ::= literal_list NK_COMMA literal */ + case 30: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==30); +{ PARSER_TRACE; yylhsminor.yy192 = addNodeToList(pCxt, yymsp[-2].minor.yy192, releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } + yymsp[-2].minor.yy192 = yylhsminor.yy192; break; - case 11: /* select_sublist ::= select_sublist NK_COMMA select_item */ - case 38: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==38); -{ PARSER_TRACE; yylhsminor.yy30 = addNodeToList(pCxt, yymsp[-2].minor.yy30, yymsp[0].minor.yy130); } - yymsp[-2].minor.yy30 = yylhsminor.yy30; + case 11: /* db_name ::= NK_ID */ + case 12: /* table_name ::= NK_ID */ yytestcase(yyruleno==12); + case 13: /* column_name ::= NK_ID */ yytestcase(yyruleno==13); + case 14: /* function_name ::= NK_ID */ yytestcase(yyruleno==14); + case 15: /* table_alias ::= NK_ID */ yytestcase(yyruleno==15); + case 16: /* column_alias ::= NK_ID */ yytestcase(yyruleno==16); +{ PARSER_TRACE; yylhsminor.yy241 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy241 = yylhsminor.yy241; break; - case 12: /* select_item ::= value_expression */ - case 16: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==16); - case 17: /* table_reference ::= table_factor */ yytestcase(yyruleno==17); - case 18: /* table_factor ::= table_primary */ yytestcase(yyruleno==18); - case 25: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==25); - case 27: /* query_primary ::= query_specification */ yytestcase(yyruleno==27); -{ PARSER_TRACE; yylhsminor.yy130 = yymsp[0].minor.yy130; } - yymsp[0].minor.yy130 = yylhsminor.yy130; + case 19: /* expression ::= function_name NK_LP expression_list NK_RP */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy241, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy241, yymsp[-1].minor.yy192)); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; break; - case 13: /* select_item ::= value_expression AS NK_ID */ -{ PARSER_TRACE; yylhsminor.yy130 = setProjectionAlias(pCxt, yymsp[-2].minor.yy130, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 21: /* expression ::= NK_LP expression NK_RP */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168)); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 15: /* from_clause ::= FROM table_reference_list */ -{ PARSER_TRACE; yymsp[-1].minor.yy130 = yymsp[0].minor.yy130; } + case 22: /* expression ::= NK_PLUS expression */ +{ + PARSER_TRACE; + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); + yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); + } + yymsp[-1].minor.yy168 = yylhsminor.yy168; break; - case 19: /* table_primary ::= table_name */ -{ PARSER_TRACE; yylhsminor.yy130 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy67); } - yymsp[0].minor.yy130 = yylhsminor.yy130; + case 23: /* expression ::= NK_MINUS expression */ +{ + PARSER_TRACE; + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); + yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy168), NULL)); + } + yymsp[-1].minor.yy168 = yylhsminor.yy168; break; - case 20: /* table_primary ::= db_name NK_DOT table_name */ -{ PARSER_TRACE; yylhsminor.yy130 = createRealTableNode(pCxt, &yymsp[-2].minor.yy67, &yymsp[0].minor.yy67); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 24: /* expression ::= expression NK_PLUS expression */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); + yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 21: /* db_name ::= NK_ID */ - case 22: /* table_name ::= NK_ID */ yytestcase(yyruleno==22); -{ PARSER_TRACE; yylhsminor.yy67 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy67 = yylhsminor.yy67; + case 25: /* expression ::= expression NK_MINUS expression */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); + yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 23: /* query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ -{ yy_destructor(yypParser,61,&yymsp[-4].minor); + case 26: /* expression ::= expression NK_STAR expression */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); + yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 27: /* expression ::= expression NK_SLASH expression */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); + yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 28: /* expression ::= expression NK_REM expression */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); + yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 31: /* column_reference ::= column_name */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy241, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy241)); } + yymsp[0].minor.yy168 = yylhsminor.yy168; + break; + case 32: /* column_reference ::= table_name NK_DOT column_name */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241, createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241)); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 33: /* predicate ::= expression compare_op expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, yymsp[-1].minor.yy228, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 34: /* predicate ::= expression BETWEEN expression AND expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy168), releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } + yymsp[-4].minor.yy168 = yylhsminor.yy168; + break; + case 35: /* predicate ::= expression NOT BETWEEN expression AND expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[-5].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } + yymsp[-5].minor.yy168 = yylhsminor.yy168; + break; + case 36: /* predicate ::= expression IS NULL */ +{ PARSER_TRACE; yylhsminor.yy168 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), true); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 37: /* predicate ::= expression IS NOT NULL */ +{ PARSER_TRACE; yylhsminor.yy168 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy168), false); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 38: /* predicate ::= expression in_op in_predicate_value */ +{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, yymsp[-1].minor.yy228, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 39: /* compare_op ::= NK_LT */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LOWER_THAN; } + break; + case 40: /* compare_op ::= NK_GT */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_GREATER_THAN; } + break; + case 41: /* compare_op ::= NK_LE */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LOWER_EQUAL; } + break; + case 42: /* compare_op ::= NK_GE */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_GREATER_EQUAL; } + break; + case 43: /* compare_op ::= NK_NE */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_NOT_EQUAL; } + break; + case 44: /* compare_op ::= NK_EQ */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_EQUAL; } + break; + case 45: /* compare_op ::= LIKE */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LIKE; } + break; + case 46: /* compare_op ::= NOT LIKE */ +{ PARSER_TRACE; yymsp[-1].minor.yy228 = OP_TYPE_NOT_LIKE; } + break; + case 47: /* compare_op ::= MATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_MATCH; } + break; + case 48: /* compare_op ::= NMATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_NMATCH; } + break; + case 49: /* in_op ::= IN */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_IN; } + break; + case 50: /* in_op ::= NOT IN */ +{ PARSER_TRACE; yymsp[-1].minor.yy228 = OP_TYPE_NOT_IN; } + break; + case 51: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ PARSER_TRACE; yymsp[-2].minor.yy168 = createNodeListNode(pCxt, yymsp[-1].minor.yy192); } + break; + case 53: /* boolean_value_expression ::= NOT boolean_primary */ +{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy168, NULL); } + break; + case 54: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 55: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 57: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + case 70: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ yytestcase(yyruleno==70); + case 71: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==71); +{ PARSER_TRACE; yymsp[-2].minor.yy168 = yymsp[-1].minor.yy168; } + break; + case 58: /* from_clause ::= FROM table_reference_list */ + case 87: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==87); + case 108: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==108); +{ PARSER_TRACE; yymsp[-1].minor.yy168 = yymsp[0].minor.yy168; } + break; + case 60: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy168, yymsp[0].minor.yy168, NULL); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 63: /* table_primary ::= table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; + break; + case 64: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, &yymsp[-3].minor.yy241, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 65: /* table_primary ::= subquery alias_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168), &yymsp[0].minor.yy241); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; + break; + case 67: /* alias_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy241 = nil_token; } + break; + case 68: /* alias_opt ::= table_alias */ +{ PARSER_TRACE; yylhsminor.yy241 = yymsp[0].minor.yy241; } + yymsp[0].minor.yy241 = yylhsminor.yy241; + break; + case 69: /* alias_opt ::= AS table_alias */ +{ PARSER_TRACE; yymsp[-1].minor.yy241 = yymsp[0].minor.yy241; } + break; + case 72: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, yymsp[-4].minor.yy229, yymsp[-5].minor.yy168, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-5].minor.yy168 = yylhsminor.yy168; + break; + case 73: /* join_type ::= INNER */ +{ PARSER_TRACE; yymsp[0].minor.yy229 = JOIN_TYPE_INNER; } + break; + case 74: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - PARSER_TRACE; - addOrderByList(pCxt, yymsp[-3].minor.yy130, yymsp[-2].minor.yy30); - addSlimit(pCxt, yymsp[-3].minor.yy130, yymsp[-1].minor.yy130); - addLimit(pCxt, yymsp[-3].minor.yy130, yymsp[0].minor.yy130); - yymsp[-4].minor.yy130 = yymsp[-3].minor.yy130; - } -} + PARSER_TRACE; + yymsp[-8].minor.yy168 = createSelectStmt(pCxt, yymsp[-7].minor.yy209, yymsp[-6].minor.yy192, yymsp[-5].minor.yy168); + yymsp[-8].minor.yy168 = addWhereClause(pCxt, yymsp[-8].minor.yy168, yymsp[-4].minor.yy168); + yymsp[-8].minor.yy168 = addPartitionByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-3].minor.yy192); + yymsp[-8].minor.yy168 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy168, yymsp[-2].minor.yy168); + yymsp[-8].minor.yy168 = addGroupByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-1].minor.yy192); + yymsp[-8].minor.yy168 = addHavingClause(pCxt, yymsp[-8].minor.yy168, yymsp[0].minor.yy168); + } break; - case 24: /* with_clause_opt ::= */ -{} + case 75: /* set_quantifier_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy209 = false; } break; - case 26: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ PARSER_TRACE; yylhsminor.yy130 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy130, yymsp[0].minor.yy130); } - yymsp[-3].minor.yy130 = yylhsminor.yy130; + case 76: /* set_quantifier_opt ::= DISTINCT */ +{ PARSER_TRACE; yymsp[0].minor.yy209 = true; } break; - case 28: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy130 = yymsp[-4].minor.yy130;} - yy_destructor(yypParser,63,&yymsp[-3].minor); - yy_destructor(yypParser,65,&yymsp[-2].minor); - yy_destructor(yypParser,64,&yymsp[-1].minor); + case 77: /* set_quantifier_opt ::= ALL */ +{ PARSER_TRACE; yymsp[0].minor.yy209 = false; } break; - case 29: /* order_by_clause_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy30 = NULL; } + case 78: /* select_list ::= NK_STAR */ +{ PARSER_TRACE; yymsp[0].minor.yy192 = NULL; } break; - case 30: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ -{ PARSER_TRACE; yymsp[-2].minor.yy30 = yymsp[0].minor.yy30; } + case 79: /* select_list ::= select_sublist */ +{ PARSER_TRACE; yylhsminor.yy192 = yymsp[0].minor.yy192; } + yymsp[0].minor.yy192 = yylhsminor.yy192; break; - case 31: /* slimit_clause_opt ::= */ - case 34: /* limit_clause_opt ::= */ yytestcase(yyruleno==34); -{ yymsp[1].minor.yy130 = NULL; } + case 80: /* select_sublist ::= select_item */ + case 125: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==125); +{ PARSER_TRACE; yylhsminor.yy192 = createNodeList(pCxt, yymsp[0].minor.yy168); } + yymsp[0].minor.yy192 = yylhsminor.yy192; break; - case 32: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 35: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==35); -{ yymsp[-3].minor.yy130 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 81: /* select_sublist ::= select_sublist NK_COMMA select_item */ + case 126: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==126); +{ PARSER_TRACE; yylhsminor.yy192 = addNodeToList(pCxt, yymsp[-2].minor.yy192, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy192 = yylhsminor.yy192; break; - case 33: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 36: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==36); -{ yymsp[-3].minor.yy130 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } - break; - case 39: /* sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ -{ PARSER_TRACE; yylhsminor.yy130 = createOrderByExprNode(pCxt, yymsp[-2].minor.yy130, yymsp[-1].minor.yy108, yymsp[0].minor.yy68); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; - break; - case 40: /* ordering_specification_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy108 = ORDER_ASC; } - break; - case 41: /* ordering_specification_opt ::= ASC */ -{ PARSER_TRACE; yymsp[0].minor.yy108 = ORDER_ASC; } - break; - case 42: /* ordering_specification_opt ::= DESC */ -{ PARSER_TRACE; yymsp[0].minor.yy108 = ORDER_DESC; } - break; - case 43: /* null_ordering_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy68 = NULL_ORDER_DEFAULT; } - break; - case 44: /* null_ordering_opt ::= NULLS FIRST */ -{ PARSER_TRACE; yymsp[-1].minor.yy68 = NULL_ORDER_FIRST; } - break; - case 45: /* null_ordering_opt ::= NULLS LAST */ -{ PARSER_TRACE; yymsp[-1].minor.yy68 = NULL_ORDER_LAST; } - break; - case 46: /* value_function ::= NK_ID NK_LP value_expression NK_RP */ - case 48: /* value_expression_primary ::= NK_LP value_expression NK_RP */ yytestcase(yyruleno==48); + case 82: /* select_item ::= expression */ { -} - yy_destructor(yypParser,38,&yymsp[-1].minor); + PARSER_TRACE; + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); + yylhsminor.yy168 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy168), &t); + } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 47: /* value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ -{ -} - yy_destructor(yypParser,38,&yymsp[-3].minor); - yy_destructor(yypParser,38,&yymsp[-1].minor); + case 83: /* select_item ::= expression column_alias */ +{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168), &yymsp[0].minor.yy241); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; break; - case 49: /* value_expression_primary ::= nonparenthesized_value_expression_primary */ -{ yy_destructor(yypParser,40,&yymsp[0].minor); -{ -} -} + case 84: /* select_item ::= expression AS column_alias */ +{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), &yymsp[0].minor.yy241); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 50: /* nonparenthesized_value_expression_primary ::= literal */ -{ yy_destructor(yypParser,41,&yymsp[0].minor); -{ -} -} + case 85: /* select_item ::= table_name NK_DOT NK_STAR */ +{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 51: /* nonparenthesized_value_expression_primary ::= column_reference */ -{ yy_destructor(yypParser,42,&yymsp[0].minor); -{ -} -} + case 86: /* where_clause_opt ::= */ + case 90: /* twindow_clause_opt ::= */ yytestcase(yyruleno==90); + case 95: /* sliding_opt ::= */ yytestcase(yyruleno==95); + case 97: /* fill_opt ::= */ yytestcase(yyruleno==97); + case 107: /* having_clause_opt ::= */ yytestcase(yyruleno==107); + case 115: /* slimit_clause_opt ::= */ yytestcase(yyruleno==115); + case 119: /* limit_clause_opt ::= */ yytestcase(yyruleno==119); +{ PARSER_TRACE; yymsp[1].minor.yy168 = NULL; } break; - case 53: /* value_expression ::= common_value_expression */ -{ yy_destructor(yypParser,44,&yymsp[0].minor); -{ -} -} + case 88: /* partition_by_clause_opt ::= */ + case 105: /* group_by_clause_opt ::= */ yytestcase(yyruleno==105); + case 113: /* order_by_clause_opt ::= */ yytestcase(yyruleno==113); +{ PARSER_TRACE; yymsp[1].minor.yy192 = NULL; } break; - case 54: /* common_value_expression ::= numeric_value_expression */ -{ yy_destructor(yypParser,45,&yymsp[0].minor); -{ -} -} + case 89: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 106: /* group_by_clause_opt ::= GROUP BY expression_list */ yytestcase(yyruleno==106); + case 114: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==114); +{ PARSER_TRACE; yymsp[-2].minor.yy192 = yymsp[0].minor.yy192; } break; - case 55: /* numeric_value_expression ::= numeric_primary */ -{ yy_destructor(yypParser,46,&yymsp[0].minor); -{ -} -} + case 91: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy168 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy168), &yymsp[-1].minor.yy0); } break; - case 56: /* numeric_value_expression ::= NK_PLUS numeric_primary */ - case 57: /* numeric_value_expression ::= NK_MINUS numeric_primary */ yytestcase(yyruleno==57); -{ -} - yy_destructor(yypParser,46,&yymsp[0].minor); + case 92: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168)); } break; - case 58: /* numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ - case 59: /* numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ yytestcase(yyruleno==59); - case 60: /* numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ yytestcase(yyruleno==60); - case 61: /* numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ yytestcase(yyruleno==61); -{ yy_destructor(yypParser,45,&yymsp[-2].minor); -{ -} - yy_destructor(yypParser,45,&yymsp[0].minor); -} + case 93: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-5].minor.yy168 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy168, NULL, yymsp[-1].minor.yy168, yymsp[0].minor.yy168); } break; - case 62: /* numeric_primary ::= value_expression_primary */ -{ yy_destructor(yypParser,39,&yymsp[0].minor); -{ -} -} + case 94: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-7].minor.yy168 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy168, yymsp[-3].minor.yy168, yymsp[-1].minor.yy168, yymsp[0].minor.yy168); } break; - case 63: /* numeric_primary ::= value_function */ -{ yy_destructor(yypParser,37,&yymsp[0].minor); -{ -} -} + case 96: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy168 = yymsp[-1].minor.yy168; } break; - case 64: /* table_primary ::= derived_table */ -{ yy_destructor(yypParser,58,&yymsp[0].minor); -{ -} -} + case 98: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createFillNode(pCxt, yymsp[-1].minor.yy14, NULL); } break; - case 65: /* derived_table ::= table_subquery */ -{ yy_destructor(yypParser,59,&yymsp[0].minor); -{ -} -} + case 99: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy168 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy192)); } break; - case 66: /* subquery ::= NK_LR query_expression NK_RP */ -{ -} - yy_destructor(yypParser,36,&yymsp[-1].minor); + case 100: /* fill_mode ::= NONE */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NONE; } break; - case 67: /* table_subquery ::= subquery */ -{ yy_destructor(yypParser,60,&yymsp[0].minor); -{ -} -} + case 101: /* fill_mode ::= PREV */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_PREV; } + break; + case 102: /* fill_mode ::= NULL */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NULL; } + break; + case 103: /* fill_mode ::= LINEAR */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_LINEAR; } + break; + case 104: /* fill_mode ::= NEXT */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NEXT; } + break; + case 109: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ +{ + PARSER_TRACE; + yylhsminor.yy168 = addOrderByClause(pCxt, yymsp[-3].minor.yy168, yymsp[-2].minor.yy192); + yylhsminor.yy168 = addSlimitClause(pCxt, yylhsminor.yy168, yymsp[-1].minor.yy168); + yylhsminor.yy168 = addLimitClause(pCxt, yylhsminor.yy168, yymsp[0].minor.yy168); + } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 111: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ PARSER_TRACE; yylhsminor.yy168 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 116: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 120: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==120); +{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + break; + case 117: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 121: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==121); +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 118: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 122: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==122); +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + break; + case 123: /* subquery ::= NK_LP query_expression NK_RP */ +{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 127: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), yymsp[-1].minor.yy10, yymsp[0].minor.yy177); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 128: /* ordering_specification_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy10 = ORDER_ASC; } + break; + case 129: /* ordering_specification_opt ::= ASC */ +{ PARSER_TRACE; yymsp[0].minor.yy10 = ORDER_ASC; } + break; + case 130: /* ordering_specification_opt ::= DESC */ +{ PARSER_TRACE; yymsp[0].minor.yy10 = ORDER_DESC; } + break; + case 131: /* null_ordering_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy177 = NULL_ORDER_DEFAULT; } + break; + case 132: /* null_ordering_opt ::= NULLS FIRST */ +{ PARSER_TRACE; yymsp[-1].minor.yy177 = NULL_ORDER_FIRST; } + break; + case 133: /* null_ordering_opt ::= NULLS LAST */ +{ PARSER_TRACE; yymsp[-1].minor.yy177 = NULL_ORDER_LAST; } break; default: - /* (52) literal ::= NK_LITERAL */ yytestcase(yyruleno==52); break; /********** End reduce actions ************************************************/ }; @@ -1507,7 +2033,7 @@ static void yy_accept( /* Here code is inserted which will be executed whenever the ** parser accepts */ /*********** Begin %parse_accept code *****************************************/ - printf("parsing complete!\n" ); + PARSER_COMPLETE; /*********** End %parse_accept code *******************************************/ NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ NewParseCTX_STORE diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index 8902da9274..7ec3ee0da4 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -15,8 +15,12 @@ #include "parserImpl.h" -#include "ttoken.h" #include "astCreateContext.h" +#include "functionMgt.h" +#include "parserInt.h" +#include "tglobal.h" +#include "ttime.h" +#include "ttoken.h" typedef void* (*FMalloc)(size_t); typedef void (*FFree)(void*); @@ -24,9 +28,14 @@ typedef void (*FFree)(void*); extern void* NewParseAlloc(FMalloc); extern void NewParse(void*, int, SToken, void*); extern void NewParseFree(void*, FFree); +extern void NewParseTrace(FILE*, char*); -uint32_t toNewTokenId(uint32_t tokenId) { +static uint32_t toNewTokenId(uint32_t tokenId) { switch (tokenId) { + case TK_OR: + return NEW_TK_OR; + case TK_AND: + return NEW_TK_AND; case TK_UNION: return NEW_TK_UNION; case TK_ALL: @@ -39,28 +48,104 @@ uint32_t toNewTokenId(uint32_t tokenId) { return NEW_TK_NK_STAR; case TK_SLASH: return NEW_TK_NK_SLASH; + case TK_REM: + return NEW_TK_NK_REM; case TK_SHOW: return NEW_TK_SHOW; case TK_DATABASES: return NEW_TK_DATABASES; + case TK_INTEGER: + return NEW_TK_NK_INTEGER; + case TK_FLOAT: + return NEW_TK_NK_FLOAT; + case TK_STRING: + return NEW_TK_NK_STRING; + case TK_BOOL: + return NEW_TK_NK_BOOL; + case TK_TIMESTAMP: + return NEW_TK_TIMESTAMP; + case TK_VARIABLE: + return NEW_TK_NK_VARIABLE; + case TK_COMMA: + return NEW_TK_NK_COMMA; case TK_ID: return NEW_TK_NK_ID; case TK_LP: return NEW_TK_NK_LP; case TK_RP: return NEW_TK_NK_RP; - case TK_COMMA: - return NEW_TK_NK_COMMA; case TK_DOT: return NEW_TK_NK_DOT; + case TK_BETWEEN: + return NEW_TK_BETWEEN; + case TK_NOT: + return NEW_TK_NOT; + case TK_IS: + return NEW_TK_IS; + case TK_NULL: + return NEW_TK_NULL; + case TK_LT: + return NEW_TK_NK_LT; + case TK_GT: + return NEW_TK_NK_GT; + case TK_LE: + return NEW_TK_NK_LE; + case TK_GE: + return NEW_TK_NK_GE; + case TK_NE: + return NEW_TK_NK_NE; + case TK_EQ: + return NEW_TK_NK_EQ; + case TK_LIKE: + return NEW_TK_LIKE; + case TK_MATCH: + return NEW_TK_MATCH; + case TK_NMATCH: + return NEW_TK_NMATCH; + case TK_IN: + return NEW_TK_IN; case TK_SELECT: return NEW_TK_SELECT; case TK_DISTINCT: return NEW_TK_DISTINCT; + case TK_WHERE: + return NEW_TK_WHERE; case TK_AS: return NEW_TK_AS; case TK_FROM: return NEW_TK_FROM; + case TK_JOIN: + return NEW_TK_JOIN; + // case TK_ON: + // return NEW_TK_ON; + // case TK_INNER: + // return NEW_TK_INNER; + // case TK_PARTITION: + // return NEW_TK_PARTITION; + case TK_SESSION: + return NEW_TK_SESSION; + case TK_STATE_WINDOW: + return NEW_TK_STATE_WINDOW; + case TK_INTERVAL: + return NEW_TK_INTERVAL; + case TK_SLIDING: + return NEW_TK_SLIDING; + case TK_FILL: + return NEW_TK_FILL; + // case TK_VALUE: + // return NEW_TK_VALUE; + case TK_NONE: + return NEW_TK_NONE; + case TK_PREV: + return NEW_TK_PREV; + case TK_LINEAR: + return NEW_TK_LINEAR; + // case TK_NEXT: + // return NEW_TK_NEXT; + case TK_GROUP: + return NEW_TK_GROUP; + case TK_HAVING: + return NEW_TK_HAVING; case TK_ORDER: return NEW_TK_ORDER; case TK_BY: @@ -69,31 +154,44 @@ uint32_t toNewTokenId(uint32_t tokenId) { return NEW_TK_ASC; case TK_DESC: return NEW_TK_DESC; + case TK_SLIMIT: + return NEW_TK_SLIMIT; + case TK_SOFFSET: + return NEW_TK_SOFFSET; + case TK_LIMIT: + return NEW_TK_LIMIT; + case TK_OFFSET: + return NEW_TK_OFFSET; + case TK_SPACE: + break; + default: + printf("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!tokenId = %d\n", tokenId); } return tokenId; } -uint32_t getToken(const char* z, uint32_t* tokenId) { +static uint32_t getToken(const char* z, uint32_t* tokenId) { uint32_t n = tGetToken(z, tokenId); *tokenId = toNewTokenId(*tokenId); return n; } int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { - SAstCreateContext cxt = { .pQueryCxt = pParseCxt, .valid = true, .pRootNode = NULL }; + SAstCreateContext cxt; + createAstCreateContext(pParseCxt, &cxt); void *pParser = NewParseAlloc(malloc); int32_t i = 0; while (1) { SToken t0 = {0}; - printf("===========================\n"); + // printf("===========================\n"); if (cxt.pQueryCxt->pSql[i] == 0) { NewParse(pParser, 0, t0, &cxt); goto abort_parse; } - printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); + // printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); t0.n = getToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type); t0.z = (char *)(cxt.pQueryCxt->pSql + i); - printf("token %p : %d %d [%s]\n", &t0, t0.type, t0.n, t0.z); + // printf("token : %d %d [%s]\n", t0.type, t0.n, t0.z); i += t0.n; switch (t0.type) { @@ -123,6 +221,7 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { default: NewParse(pParser, t0.type, t0, &cxt); + // NewParseTrace(stdout, ""); if (!cxt.valid) { goto abort_parse; } @@ -130,7 +229,467 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { } abort_parse: + // printf("doParse completed.\n"); NewParseFree(pParser, free); + destroyAstCreateContext(&cxt); pQuery->pRoot = cxt.pRootNode; return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; } + +typedef enum ESqlClause { + SQL_CLAUSE_FROM = 1, + SQL_CLAUSE_WHERE +} ESqlClause; + +typedef struct STranslateContext { + SParseContext* pParseCxt; + FuncMgtHandle fmgt; + int32_t errCode; + SMsgBuf msgBuf; + SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode* + int32_t currLevel; + ESqlClause currClause; +} STranslateContext; + +static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); + +static char* getSyntaxErrFormat(int32_t errCode) { + switch (errCode) { + case TSDB_CODE_PAR_INVALID_COLUMN: + return "Invalid column name : %s"; + case TSDB_CODE_PAR_TABLE_NOT_EXIST: + return "Table does not exist : %s"; + case TSDB_CODE_PAR_AMBIGUOUS_COLUMN: + return "Column ambiguously defined : %s"; + case TSDB_CODE_PAR_WRONG_VALUE_TYPE: + return "Invalid value type : %s"; + case TSDB_CODE_PAR_FUNTION_PARA_NUM: + return "Invalid number of arguments : %s"; + case TSDB_CODE_PAR_FUNTION_PARA_TYPE: + return "Inconsistent datatypes : %s"; + case TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION: + return "There mustn't be aggregation"; + default: + return "Unknown error"; + } +} + +static int32_t generateSyntaxErrMsg(STranslateContext* pCxt, int32_t errCode, ...) { + va_list vArgList; + va_start(vArgList, errCode); + vsnprintf(pCxt->msgBuf.buf, pCxt->msgBuf.len, getSyntaxErrFormat(errCode), vArgList); + va_end(vArgList); + pCxt->errCode = errCode; + return errCode; +} + +static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { + size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); + if (currTotalLevel > pCxt->currLevel) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + taosArrayPush(pTables, &pTable); + } else { + do { + SArray* pTables = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + if (pCxt->currLevel == currTotalLevel) { + taosArrayPush(pTables, &pTable); + } + taosArrayPush(pCxt->pNsLevel, &pTables); + ++currTotalLevel; + } while (currTotalLevel <= pCxt->currLevel); + } + return TSDB_CODE_SUCCESS; +} + +static SName* toName(int32_t acctId, const SRealTableNode* pRealTable, SName* pName) { + pName->type = TSDB_TABLE_NAME_T; + pName->acctId = acctId; + strcpy(pName->dbname, pRealTable->table.dbName); + strcpy(pName->tname, pRealTable->table.tableName); + return pName; +} + +static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) { + int cmp = 0; + if ('\0' != pCol->dbName[0]) { + cmp = strcmp(pCol->dbName, pTable->dbName); + } else { + cmp = strcmp(currentDb, pTable->dbName); + } + if (0 == cmp) { + cmp = strcmp(pCol->tableAlias, pTable->tableAlias); + } + return (0 == cmp); +} + +static SNodeList* getProjectList(SNode* pNode) { + if (QUERY_NODE_SELECT_STMT == nodeType(pNode)) { + return ((SSelectStmt*)pNode)->pProjectionList; + } + return NULL; +} + +static void setColumnInfoBySchema(const STableNode* pTable, const SSchema* pColSchema, SColumnNode* pCol) { + strcpy(pCol->dbName, pTable->dbName); + strcpy(pCol->tableAlias, pTable->tableAlias); + strcpy(pCol->tableName, pTable->tableName); + strcpy(pCol->colName, pColSchema->name); + if ('\0' == pCol->node.aliasName[0]) { + strcpy(pCol->node.aliasName, pColSchema->name); + } + pCol->colId = pColSchema->colId; + // pCol->colType = pColSchema->type; + pCol->node.resType.type = pColSchema->type; + pCol->node.resType.bytes = pColSchema->bytes; +} + +static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SColumnNode* pCol) { + pCol->pProjectRef = (SNode*)pExpr; + pExpr->pAssociationList = nodesListAppend(pExpr->pAssociationList, (SNode*)pCol); + strcpy(pCol->tableAlias, pTable->tableAlias); + strcpy(pCol->colName, pExpr->aliasName); + pCol->node.resType = pExpr->resType; +} + +static int32_t createColumnNodeByTable(const STableNode* pTable, SNodeList* pList) { + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; + int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; + for (int32_t i = 0; i < nums; ++i) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + setColumnInfoBySchema(pTable, pMeta->schema + i, pCol); + nodesListAppend(pList, (SNode*)pCol); + } + } else { + SNodeList* pProjectList = getProjectList(((STempTableNode*)pTable)->pSubquery); + SNode* pNode; + FOREACH(pNode, pProjectList) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + setColumnInfoByExpr(pTable, (SExprNode*)pNode, pCol); + nodesListAppend(pList, (SNode*)pCol); + } + } +} + +static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { + bool found = false; + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; + int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; + for (int32_t i = 0; i < nums; ++i) { + if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { + setColumnInfoBySchema(pTable, pMeta->schema + i, pCol); + found = true; + break; + } + } + } else { + SNodeList* pProjectList = getProjectList(((STempTableNode*)pTable)->pSubquery); + SNode* pNode; + FOREACH(pNode, pProjectList) { + SExprNode* pExpr = (SExprNode*)pNode; + if (0 == strcmp(pCol->colName, pExpr->aliasName)) { + setColumnInfoByExpr(pTable, pExpr, pCol); + found = true; + break; + } + } + } + return found; +} + +static bool translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* pCol) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (belongTable(pCxt->pParseCxt->db, pCol, pTable)) { + if (findAndSetColumn(pCol, pTable)) { + break; + } + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); + return false; + } + } + return true; +} + +static bool translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* pCol) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + bool found = false; + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (findAndSetColumn(pCol, pTable)) { + if (found) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName); + return false; + } + found = true; + } + } + if (!found) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); + return false; + } + return true; +} + +static bool translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { + if ('\0' != pCol->tableAlias[0]) { + return translateColumnWithPrefix(pCxt, pCol); + } + return translateColumnWithoutPrefix(pCxt, pCol); +} + +static int32_t trimStringCopy(const char* src, int32_t len, char* dst) { + // delete escape character: \\, \', \" + char delim = src[0]; + int32_t cnt = 0; + int32_t j = 0; + for (uint32_t k = 1; k < len - 1; ++k) { + if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) { + dst[j] = src[k + 1]; + cnt++; + j++; + k++; + continue; + } + dst[j] = src[k]; + j++; + } + dst[j] = '\0'; + return j; +} + +static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) { + if (pVal->isDuration) { + char unit = 0; + if (parseAbsoluteDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + return false; + } + } else { + switch (pVal->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + pVal->datum.b = (0 == strcasecmp(pVal->literal, "true")); + break; + case TSDB_DATA_TYPE_BIGINT: { + char* endPtr = NULL; + pVal->datum.i = strtoull(pVal->literal, &endPtr, 10); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + char* endPtr = NULL; + pVal->datum.d = strtold(pVal->literal, &endPtr); + break; + } + case TSDB_DATA_TYPE_BINARY: { + int32_t n = strlen(pVal->literal); + pVal->datum.p = calloc(1, n); + trimStringCopy(pVal->literal, n, pVal->datum.p); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: { + int32_t n = strlen(pVal->literal); + char* tmp = calloc(1, n); + int32_t len = trimStringCopy(pVal->literal, n, tmp); + if (taosParseTime(tmp, &pVal->datum.u, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { + tfree(tmp); + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + return false; + } + tfree(tmp); + break; + } + default: + break; + } + } + return true; +} + +static bool translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { + SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; + SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; + if (nodesIsArithmeticOp(pOp)) { + if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || + TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + return false; + } + pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + return true; + } else if (nodesIsComparisonOp(pOp)) { + if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || + TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + return false; + } + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + return true; + } else { + // todo json operator + return true; + } + return true; +} + +static bool translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { + int32_t code = fmGetFuncResultType(pCxt->fmgt, pFunc); + if (TSDB_CODE_SUCCESS != code) { + generateSyntaxErrMsg(pCxt, code, pFunc->functionName); + return false; + } + if (fmIsAggFunc(pFunc->funcId) && (SQL_CLAUSE_FROM == pCxt->currClause || SQL_CLAUSE_WHERE == pCxt->currClause)) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION); + return false; + } + return true; +} + +static bool doTranslateExpr(SNode* pNode, void* pContext) { + STranslateContext* pCxt = (STranslateContext*)pContext; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + return translateColumn(pCxt, (SColumnNode*)pNode); + case QUERY_NODE_VALUE: + return translateValue(pCxt, (SValueNode*)pNode); + case QUERY_NODE_OPERATOR: + return translateOperator(pCxt, (SOperatorNode*)pNode); + case QUERY_NODE_FUNCTION: + return translateFunction(pCxt, (SFunctionNode*)pNode); + case QUERY_NODE_TEMP_TABLE: + return translateSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); + default: + break; + } + return true; +} + +static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) { + nodesWalkNodePostOrder(pNode, doTranslateExpr, pCxt); + return pCxt->errCode; +} + +static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { + nodesWalkListPostOrder(pList, doTranslateExpr, pCxt); + return pCxt->errCode; +} + +static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pTable)) { + case QUERY_NODE_REAL_TABLE: { + SRealTableNode* pRealTable = (SRealTableNode*)pTable; + SName name; + code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &(pCxt->pParseCxt->mgmtEpSet), + toName(pCxt->pParseCxt->acctId, pRealTable, &name), &(pRealTable->pMeta)); + if (TSDB_CODE_SUCCESS != code) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName); + } + code = addNamespace(pCxt, pRealTable); + break; + } + case QUERY_NODE_TEMP_TABLE: { + STempTableNode* pTempTable = (STempTableNode*)pTable; + code = translateSubquery(pCxt, pTempTable->pSubquery); + if (TSDB_CODE_SUCCESS == code) { + code = addNamespace(pCxt, pTempTable); + } + break; + } + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTable = (SJoinTableNode*)pTable; + code = translateTable(pCxt, pJoinTable->pLeft); + if (TSDB_CODE_SUCCESS == code) { + code = translateTable(pCxt, pJoinTable->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, pJoinTable->pOnCond); + } + break; + } + default: + break; + } + return code; +} + +static int32_t translateFrom(STranslateContext* pCxt, SNode* pTable) { + pCxt->currClause = SQL_CLAUSE_FROM; + return translateTable(pCxt, pTable); +} + +static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect, bool* pIsSelectStar) { + if (NULL == pSelect->pProjectionList) { // select * ... + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + pSelect->pProjectionList = nodesMakeList(); + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + createColumnNodeByTable(pTable, pSelect->pProjectionList); + } + *pIsSelectStar = true; + } else { + + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; + code = translateFrom(pCxt, pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, pSelect->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateExprList(pCxt, pSelect->pGroupByList); + } + bool isSelectStar = false; + if (TSDB_CODE_SUCCESS == code) { + code = translateStar(pCxt, pSelect, &isSelectStar); + } + if (TSDB_CODE_SUCCESS == code && !isSelectStar) { + code = translateExprList(pCxt, pSelect->pProjectionList); + } + // printf("%s:%d code = %d\n", __FUNCTION__, __LINE__, code); + return code; +} + +static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pNode)) { + case QUERY_NODE_SELECT_STMT: + code = translateSelect(pCxt, (SSelectStmt*)pNode); + break; + default: + break; + } + return code; +} + +static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { + ++(pCxt->currLevel); + ESqlClause currClause = pCxt->currClause; + int32_t code = translateQuery(pCxt, pNode); + --(pCxt->currLevel); + pCxt->currClause = currClause; + return code; +} + +int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { + STranslateContext cxt = { + .pParseCxt = pParseCxt, + .errCode = TSDB_CODE_SUCCESS, + .msgBuf = { .buf = pParseCxt->pMsg, .len = pParseCxt->msgLen }, + .pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES), + .currLevel = 0, + .currClause = 0 + }; + return translateQuery(&cxt, pQuery->pRoot); +} diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 6f63feb3c4..664f2a3ff2 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -3199,14 +3199,14 @@ static void yy_reduce( case 285: /* cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; case 286: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_TAG, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3217,7 +3217,7 @@ static void yy_reduce( toTSDBType(yymsp[0].minor.yy0.type); SArray* A = tListItemAppendToken(NULL, &yymsp[0].minor.yy0, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3231,7 +3231,7 @@ static void yy_reduce( toTSDBType(yymsp[0].minor.yy0.type); A = tListItemAppendToken(A, &yymsp[0].minor.yy0, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3250,7 +3250,7 @@ static void yy_reduce( case 290: /* cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3275,14 +3275,14 @@ static void yy_reduce( case 293: /* cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; case 294: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_TAG, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3293,7 +3293,7 @@ static void yy_reduce( toTSDBType(yymsp[0].minor.yy0.type); SArray* A = tListItemAppendToken(NULL, &yymsp[0].minor.yy0, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3307,7 +3307,7 @@ static void yy_reduce( toTSDBType(yymsp[0].minor.yy0.type); A = tListItemAppendToken(A, &yymsp[0].minor.yy0, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3326,7 +3326,7 @@ static void yy_reduce( case 298: /* cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index 5223c9f02a..c4bca54aa6 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -38,64 +38,350 @@ protected: } - int32_t run() { + bool run(int32_t parseCode = TSDB_CODE_SUCCESS, int32_t translateCode = TSDB_CODE_SUCCESS) { int32_t code = doParse(&cxt_, &query_); + // cout << "doParse return " << code << endl; if (code != TSDB_CODE_SUCCESS) { - cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - return code; + cout << "sql:[" << cxt_.pSql << "] code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + return (TSDB_CODE_SUCCESS != parseCode); + } + if (TSDB_CODE_SUCCESS != parseCode) { + return false; + } + code = doTranslate(&cxt_, &query_); + // cout << "doTranslate return " << code << endl; + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + return (TSDB_CODE_SUCCESS != translateCode); } - cout << nodeType(query_.pRoot) << endl; if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) { - // SNode* pWhereCond; - // SNodeList* pPartitionByList; // SNode - // SNode* pWindowClause; - // SNodeList* pGroupByList; // SGroupingSetNode - // SNodeList* pOrderByList; // SOrderByExprNode - // SLimitNode limit; - // SLimitNode slimit; - - SSelectStmt* select = (SSelectStmt*)query_.pRoot; - string sql("SELECT "); - if (select->isDistinct) { - sql.append("DISTINCT "); - } - if (nullptr == select->pProjectionList) { - sql.append("* "); - } else { - nodeListToSql(select->pProjectionList, sql); - } - sql.append("FROM "); - tableToSql(select->pFromTable, sql); - cout << sql << endl; + cout << "input sql : [" << cxt_.pSql << "]" << endl; + // string sql; + // selectToSql(query_.pRoot, sql); + // cout << "output sql : [" << sql << "]" << endl; + string str; + selectToStr(query_.pRoot, str); + cout << "translate str : \n" << str << endl; } - // char* pStr = NULL; - // int32_t len = 0; - // code = nodesNodeToString(query_.pRoot, &pStr, &len); - // if (code != TSDB_CODE_SUCCESS) { - // cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - // return code; - // } - // cout << "node tree:\n" << pStr << endl; - return TSDB_CODE_SUCCESS; + return (TSDB_CODE_SUCCESS == translateCode); } private: static const int max_err_len = 1024; static const int max_sql_len = 1024 * 1024; - void tableToSql(const SNode* node, string& sql) { + string dataTypeToStr(const SDataType& dt) { + switch (dt.type) { + case TSDB_DATA_TYPE_NULL: + return "NULL"; + case TSDB_DATA_TYPE_BOOL: + return "BOOL"; + case TSDB_DATA_TYPE_TINYINT: + return "TINYINT"; + case TSDB_DATA_TYPE_SMALLINT: + return "SMALLINT"; + case TSDB_DATA_TYPE_INT: + return "INT"; + case TSDB_DATA_TYPE_BIGINT: + return "BIGINT"; + case TSDB_DATA_TYPE_FLOAT: + return "FLOAT"; + case TSDB_DATA_TYPE_DOUBLE: + return "DOUBLE"; + case TSDB_DATA_TYPE_BINARY: + return "BINART(" + to_string(dt.bytes) + ")"; + case TSDB_DATA_TYPE_TIMESTAMP: + return "TIMESTAMP"; + case TSDB_DATA_TYPE_NCHAR: + return "NCHAR(" + to_string(dt.bytes) + ")"; + case TSDB_DATA_TYPE_UTINYINT: + return "UTINYINT"; + case TSDB_DATA_TYPE_USMALLINT: + return "USMALLINT"; + case TSDB_DATA_TYPE_UINT: + return "UINT"; + case TSDB_DATA_TYPE_UBIGINT: + return "UBIGINT"; + case TSDB_DATA_TYPE_VARCHAR: + return "VARCHAR(" + to_string(dt.bytes) + ")"; + case TSDB_DATA_TYPE_VARBINARY: + return "VARBINARY(" + to_string(dt.bytes) + ")"; + case TSDB_DATA_TYPE_JSON: + return "JSON"; + case TSDB_DATA_TYPE_DECIMAL: + return "DECIMAL(" + to_string(dt.precision) + ", " + to_string(dt.scale) + ")"; + case TSDB_DATA_TYPE_BLOB: + return "BLOB"; + default: + break; + } + return "Unknown Data Type " + to_string(dt.type); + } + + void valueNodeToStr(const SValueNode* pVal, string& str, bool isProject) { + switch (pVal->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + str.append("null"); + break; + case TSDB_DATA_TYPE_BOOL: + str.append(pVal->datum.b ? "true" : "false"); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + str.append(to_string(pVal->datum.i)); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + str.append(to_string(pVal->datum.d)); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + str.append(pVal->datum.p); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + str.append(to_string(pVal->datum.u)); + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + str.append(to_string(pVal->datum.u)); + break; + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + str.append("JSON or DECIMAL or BLOB"); + break; + default: + break; + } + str.append(" [" + dataTypeToStr(pVal->node.resType) + "]"); + if (isProject) { + str.append(" AS " + string(pVal->node.aliasName)); + } + } + + void nodeToStr(const SNode* node, string& str, bool isProject) { + if (nullptr == node) { + return; + } + + switch (nodeType(node)) { + case QUERY_NODE_COLUMN: { + SColumnNode* pCol = (SColumnNode*)node; + if ('\0' != pCol->dbName[0]) { + str.append(pCol->dbName); + str.append("."); + } + if ('\0' != pCol->tableAlias[0]) { + str.append(pCol->tableAlias); + str.append("."); + } + str.append(pCol->colName); + str.append(" [" + dataTypeToStr(pCol->node.resType) + "]"); + if (isProject) { + str.append(" AS " + string(pCol->node.aliasName)); + } + break; + } + case QUERY_NODE_VALUE: { + valueNodeToStr((SValueNode*)node, str, isProject); + break; + } + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOp = (SOperatorNode*)node; + nodeToStr(pOp->pLeft, str, false); + str.append(opTypeToStr(pOp->opType)); + nodeToStr(pOp->pRight, str, false); + str.append(" [" + dataTypeToStr(pOp->node.resType) + "]"); + if (isProject) { + str.append(" AS " + string(pOp->node.aliasName)); + } + break; + } + default: + break; + } + } + + void nodeListToStr(const SNodeList* nodelist, const string& prefix, string& str, bool isProject = false) { + SNode* node = nullptr; + FOREACH(node, nodelist) { + str.append(prefix); + nodeToStr(node, str, isProject); + str.append("\n"); + } + } + + void tableToStr(const SNode* node, const string& prefix, string& str) { const STableNode* table = (const STableNode*)node; - cout << "node : " << nodeType(node) << endl; switch (nodeType(node)) { case QUERY_NODE_REAL_TABLE: { SRealTableNode* realTable = (SRealTableNode*)table; - if ('\0' != realTable->dbName[0]) { - sql.append(realTable->dbName); + str.append(prefix); + if ('\0' != realTable->table.dbName[0]) { + str.append(realTable->table.dbName); + str.append("."); + } + str.append(realTable->table.tableName); + str.append(string(" ") + realTable->table.tableAlias); + break; + } + case QUERY_NODE_TEMP_TABLE: { + STempTableNode* tempTable = (STempTableNode*)table; + str.append(prefix + "(\n"); + selectToStr(tempTable->pSubquery, str, prefix + "\t"); + str.append("\n"); + str.append(prefix + ") "); + str.append(tempTable->table.tableAlias); + break; + } + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* joinTable = (SJoinTableNode*)table; + tableToStr(joinTable->pLeft, prefix, str); + str.append("\n" + prefix + "JOIN\n"); + tableToStr(joinTable->pRight, prefix, str); + if (nullptr != joinTable->pOnCond) { + str.append("\n" + prefix + "\tON "); + nodeToStr(joinTable->pOnCond, str, false); + } + break; + } + default: + break; + } + } + + void selectToStr(const SNode* node, string& str, const string& prefix = "") { + SSelectStmt* select = (SSelectStmt*)node; + str.append(prefix + "SELECT "); + if (select->isDistinct) { + str.append("DISTINCT"); + } + str.append("\n"); + nodeListToStr(select->pProjectionList, prefix + "\t", str, true); + str.append("\n" + prefix + "FROM\n"); + tableToStr(select->pFromTable, prefix + "\t", str); + } + + void selectToSql(const SNode* node, string& sql) { + SSelectStmt* select = (SSelectStmt*)node; + sql.append("SELECT "); + if (select->isDistinct) { + sql.append("DISTINCT "); + } + if (nullptr == select->pProjectionList) { + sql.append("* "); + } else { + nodeListToSql(select->pProjectionList, sql); + sql.append(" "); + } + sql.append("FROM "); + tableToSql(select->pFromTable, sql); + if (nullptr != select->pWhere) { + sql.append(" WHERE "); + nodeToSql(select->pWhere, sql); + } + } + + void tableToSql(const SNode* node, string& sql) { + const STableNode* table = (const STableNode*)node; + switch (nodeType(node)) { + case QUERY_NODE_REAL_TABLE: { + SRealTableNode* realTable = (SRealTableNode*)table; + if ('\0' != realTable->table.dbName[0]) { + sql.append(realTable->table.dbName); sql.append("."); } sql.append(realTable->table.tableName); break; } + case QUERY_NODE_TEMP_TABLE: { + STempTableNode* tempTable = (STempTableNode*)table; + sql.append("("); + selectToSql(tempTable->pSubquery, sql); + sql.append(") "); + sql.append(tempTable->table.tableAlias); + break; + } + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* joinTable = (SJoinTableNode*)table; + tableToSql(joinTable->pLeft, sql); + sql.append(" JOIN "); + tableToSql(joinTable->pRight, sql); + if (nullptr != joinTable->pOnCond) { + sql.append(" ON "); + nodeToSql(joinTable->pOnCond, sql); + } + break; + } + default: + break; + } + } + + string opTypeToStr(EOperatorType type) { + switch (type) { + case OP_TYPE_ADD: + return " + "; + case OP_TYPE_SUB: + return " - "; + case OP_TYPE_MULTI: + case OP_TYPE_DIV: + case OP_TYPE_MOD: + case OP_TYPE_GREATER_THAN: + case OP_TYPE_GREATER_EQUAL: + case OP_TYPE_LOWER_THAN: + case OP_TYPE_LOWER_EQUAL: + case OP_TYPE_EQUAL: + return " = "; + case OP_TYPE_NOT_EQUAL: + case OP_TYPE_IN: + case OP_TYPE_NOT_IN: + case OP_TYPE_LIKE: + case OP_TYPE_NOT_LIKE: + case OP_TYPE_MATCH: + case OP_TYPE_NMATCH: + case OP_TYPE_JSON_GET_VALUE: + case OP_TYPE_JSON_CONTAINS: + default: + break; + } + return " unknown operator "; + } + + void nodeToSql(const SNode* node, string& sql) { + if (nullptr == node) { + return; + } + + switch (nodeType(node)) { + case QUERY_NODE_COLUMN: { + SColumnNode* pCol = (SColumnNode*)node; + if ('\0' != pCol->dbName[0]) { + sql.append(pCol->dbName); + sql.append("."); + } + if ('\0' != pCol->tableAlias[0]) { + sql.append(pCol->tableAlias); + sql.append("."); + } + sql.append(pCol->colName); + break; + } + case QUERY_NODE_VALUE: + break; + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOp = (SOperatorNode*)node; + nodeToSql(pOp->pLeft, sql); + sql.append(opTypeToStr(pOp->opType)); + nodeToSql(pOp->pRight, sql); + break; + } default: break; } @@ -108,11 +394,8 @@ private: if (!firstNode) { sql.append(", "); } - switch (nodeType(node)) { - case QUERY_NODE_COLUMN: - sql.append(((SColumnNode*)node)->colName); - break; - } + firstNode = false; + nodeToSql(node, sql); } } @@ -131,13 +414,67 @@ private: SQuery query_; }; -// SELECT * FROM t1 -TEST_F(NewParserTest, selectStar) { +TEST_F(NewParserTest, selectSimple) { setDatabase("root", "test"); bind("SELECT * FROM t1"); - ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + ASSERT_TRUE(run()); bind("SELECT * FROM test.t1"); - ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + ASSERT_TRUE(run()); + + bind("SELECT ts, c1 FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT ts, t.c1 FROM (SELECT * FROM t1) t"); + ASSERT_TRUE(run()); + + bind("SELECT * FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); + ASSERT_TRUE(run()); +} + +TEST_F(NewParserTest, selectConstant) { + setDatabase("root", "test"); + + bind("SELECT 123, 20.4, 'abc', \"wxy\", TIMESTAMP '2022-02-09 17:30:20', true, false, 10s FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", TIMESTAMP '2022-02-09 17:30:20', true, false, 15s FROM t1"); + ASSERT_TRUE(run()); +} + +TEST_F(NewParserTest, selectExpression) { + setDatabase("root", "test"); + + bind("SELECT ts + 10s, c1 + 10, concat(c2, 'abc') FROM t1"); + ASSERT_TRUE(run()); +} + +TEST_F(NewParserTest, selectSyntaxError) { + setDatabase("root", "test"); + + bind("SELECTT * FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT *"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT *, * FROM test.t1"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT * FROM test.t1 t WHER"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); +} + +TEST_F(NewParserTest, selectSemanticError) { + setDatabase("root", "test"); + + bind("SELECT * FROM t10"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + + bind("SELECT c1, c3 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + + bind("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 38ce005094..3e14bfca09 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -45,11 +45,11 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 bMsg->header.vgId = htonl(bInput->vgId); - if (bInput->dbName) { - tstrncpy(bMsg->dbFname, bInput->dbName, tListLen(bMsg->dbFname)); + if (bInput->dbFName) { + tstrncpy(bMsg->dbFName, bInput->dbFName, tListLen(bMsg->dbFName)); } - tstrncpy(bMsg->tableFname, bInput->tableFullName, tListLen(bMsg->tableFname)); + tstrncpy(bMsg->tbName, bInput->tbName, tListLen(bMsg->tbName)); *msgLen = (int32_t)sizeof(*bMsg); return TSDB_CODE_SUCCESS; @@ -113,12 +113,19 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } - pOut->dbVgroup.vgVersion = pRsp->vgVersion; - pOut->dbVgroup.hashMethod = pRsp->hashMethod; - pOut->dbVgroup.dbId = pRsp->uid; - pOut->dbVgroup.vgInfo = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == pOut->dbVgroup.vgInfo) { - qError("hash init[%d] failed", pRsp->vgNum); + pOut->dbVgroup = calloc(1, sizeof(SDBVgroupInfo)); + if (NULL == pOut->dbVgroup) { + qError("calloc %d failed", (int32_t)sizeof(SDBVgroupInfo)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pOut->dbId = pRsp->uid; + pOut->dbVgroup->vgVersion = pRsp->vgVersion; + pOut->dbVgroup->hashMethod = pRsp->hashMethod; + pOut->dbVgroup->vgHash = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pOut->dbVgroup->vgHash) { + qError("taosHashInit %d failed", pRsp->vgNum); + tfree(pOut->dbVgroup); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -131,8 +138,8 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { pRsp->vgroupInfo[i].epset.eps[n].port = ntohs(pRsp->vgroupInfo[i].epset.eps[n].port); } - if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { - qError("hash push failed"); + if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { + qError("taosHashPut failed"); goto _return; } } @@ -142,14 +149,17 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return code; _return: + if (pOut) { - tfree(pOut->dbVgroup.vgInfo); + taosHashCleanup(pOut->dbVgroup->vgHash); + tfree(pOut->dbVgroup); } return code; } static int32_t queryConvertTableMetaMsg(STableMetaRsp* pMetaMsg) { + pMetaMsg->dbId = be64toh(pMetaMsg->dbId); pMetaMsg->numOfTags = ntohl(pMetaMsg->numOfTags); pMetaMsg->numOfColumns = ntohl(pMetaMsg->numOfColumns); pMetaMsg->sversion = ntohl(pMetaMsg->sversion); @@ -248,16 +258,15 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_VALUE; } + strcpy(pOut->dbFName, pMetaMsg->dbFName); + + pOut->dbId = pMetaMsg->dbId; + if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { SET_META_TYPE_BOTH_TABLE(pOut->metaType); - if (pMetaMsg->dbFname[0]) { - snprintf(pOut->ctbFname, sizeof(pOut->ctbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); - snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->stbFname); - } else { - memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname)); - memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname)); - } + strcpy(pOut->ctbName, pMetaMsg->tbName); + strcpy(pOut->tbName, pMetaMsg->stbName); pOut->ctbMeta.vgId = pMetaMsg->vgId; pOut->ctbMeta.tableType = pMetaMsg->tableType; @@ -268,11 +277,7 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { } else { SET_META_TYPE_TABLE(pOut->metaType); - if (pMetaMsg->dbFname[0]) { - snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); - } else { - memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname)); - } + strcpy(pOut->tbName, pMetaMsg->tbName); code = queryCreateTableMetaFromMsg(pMetaMsg, (pMetaMsg->tableType == TSDB_SUPER_TABLE), &pOut->tbMeta); } @@ -291,4 +296,4 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index f5e3ef9963..94c833307f 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -132,7 +132,7 @@ typedef struct SSchJob { SQueryProfileSummary summary; } SSchJob; -#define SCH_TASK_READY_TO_LUNCH(task) (atomic_load_32(&(task)->childReady) >= taosArrayGetSize((task)->children)) +#define SCH_TASK_READY_TO_LUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) #define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->type == QUERY_TYPE_SCAN) #define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 662df1896b..1b6d871274 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -773,14 +773,14 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i); pErrTask = par; - atomic_add_fetch_32(&par->childReady, 1); + int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); SDownstreamSource source = {.taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); - if (SCH_TASK_READY_TO_LUNCH(par)) { + if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) { SCH_ERR_RET(schLaunchTask(pJob, par)); } } diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index 37ee5194c8..784a864451 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -1,3 +1,4 @@ + aux_source_directory(src SYNC_SRC) add_library(sync ${SYNC_SRC}) diff --git a/source/libs/sync/inc/raft.h b/source/libs/sync/inc/raft.h deleted file mode 100644 index 129f0f4dbc..0000000000 --- a/source/libs/sync/inc/raft.h +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_RAFT_H -#define _TD_LIBS_SYNC_RAFT_H - -#include "sync.h" -#include "sync_type.h" -#include "thash.h" -#include "raft_message.h" -#include "sync_raft_impl.h" -#include "sync_raft_quorum.h" - -typedef struct RaftLeaderState { - -} RaftLeaderState; - -typedef struct RaftCandidateState { - /* true if in pre-vote phase */ - bool inPreVote; -} RaftCandidateState; - -typedef struct SSyncRaftIOMethods { - // send SSyncMessage to node - int (*send)(const SSyncMessage* pMsg, const SNodeInfo* pNode); -} SSyncRaftIOMethods; - -typedef int (*SyncRaftStepFp)(SSyncRaft* pRaft, const SSyncMessage* pMsg); -typedef void (*SyncRaftTickFp)(SSyncRaft* pRaft); - -struct SSyncRaft { - // owner sync node - SSyncNode* pNode; - - // hash map nodeId -> SNodeInfo* - SHashObj* nodeInfoMap; - - SyncNodeId selfId; - SyncGroupId selfGroupId; - - SSyncRaftIOMethods io; - - SSyncFSM fsm; - SSyncLogStore logStore; - SStateManager stateManager; - - union { - RaftLeaderState leaderState; - RaftCandidateState candidateState; - }; - - SyncTerm term; - SyncNodeId voteFor; - - SSyncRaftLog *log; - - uint64_t maxMsgSize; - uint64_t maxUncommittedSize; - SSyncRaftProgressTracker *tracker; - - ESyncState state; - - // isLearner is true if the local raft node is a learner. - bool isLearner; - - /** - * the leader id - **/ - SyncNodeId leaderId; - - /** - * leadTransferee is id of the leader transfer target when its value is not zero. - * Follow the procedure defined in raft thesis 3.10. - **/ - SyncNodeId leadTransferee; - - /** - * Only one conf change may be pending (in the log, but not yet - * applied) at a time. This is enforced via pendingConfIndex, which - * is set to a value >= the log index of the latest pending - * configuration change (if any). Config changes are only allowed to - * be proposed if the leader's applied index is greater than this - * value. - **/ - SyncIndex pendingConfigIndex; - - /** - * an estimate of the size of the uncommitted tail of the Raft log. Used to - * prevent unbounded log growth. Only maintained by the leader. Reset on - * term changes. - **/ - uint32_t uncommittedSize; - - /** - * number of ticks since it reached last electionTimeout when it is leader - * or candidate. - * number of ticks since it reached last electionTimeout or received a - * valid message from current leader when it is a follower. - **/ - uint16_t electionElapsed; - - /** - * number of ticks since it reached last heartbeatTimeout. - * only leader keeps heartbeatElapsed. - **/ - uint16_t heartbeatElapsed; - - bool preVote; - bool checkQuorum; - - int heartbeatTimeout; - int electionTimeout; - - /** - * randomizedElectionTimeout is a random number between - * [electiontimeout, 2 * electiontimeout - 1]. It gets reset - * when raft changes its state to follower or candidate. - **/ - int randomizedElectionTimeout; - bool disableProposalForwarding; - - // current tick count since start up - uint32_t currentTick; - - SyncRaftStepFp stepFp; - - SyncRaftTickFp tickFp; -}; - -int32_t syncRaftStart(SSyncRaft* pRaft, const SSyncInfo* pInfo); -int32_t syncRaftStep(SSyncRaft* pRaft, const SSyncMessage* pMsg); -int32_t syncRaftTick(SSyncRaft* pRaft); - -#endif /* _TD_LIBS_SYNC_RAFT_H */ \ No newline at end of file diff --git a/source/libs/sync/inc/raft_log.h b/source/libs/sync/inc/raft_log.h deleted file mode 100644 index 117ed42c2c..0000000000 --- a/source/libs/sync/inc/raft_log.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_RAFT_LOG_H -#define _TD_LIBS_SYNC_RAFT_LOG_H - -#include "sync.h" -#include "sync_type.h" - -typedef enum ESyncRaftEntryType { - SYNC_ENTRY_TYPE_LOG = 1, -} ESyncRaftEntryType; - -struct SSyncRaftEntry { - SyncTerm term; - - SyncIndex index; - - ESyncRaftEntryType type; - - SSyncBuffer buffer; -}; - -struct SSyncRaftLog { - SyncIndex uncommittedConfigIndex; - - SyncIndex commitIndex; - - SyncIndex appliedIndex; -}; - -SSyncRaftLog* syncRaftLogOpen(); - -SyncIndex syncRaftLogLastIndex(SSyncRaftLog* pLog); - -SyncIndex syncRaftLogSnapshotIndex(SSyncRaftLog* pLog); - -SyncTerm syncRaftLogLastTerm(SSyncRaftLog* pLog); - -void syncRaftLogAppliedTo(SSyncRaftLog* pLog, SyncIndex appliedIndex); - -bool syncRaftLogIsUptodate(SSyncRaftLog* pLog, SyncIndex index, SyncTerm term); - -int syncRaftLogNumOfPendingConf(SSyncRaftLog* pLog); - -bool syncRaftHasUnappliedLog(SSyncRaftLog* pLog); - -SyncTerm syncRaftLogTermOf(SSyncRaftLog* pLog, SyncIndex index); - -int syncRaftLogAppend(SSyncRaftLog* pLog, SSyncRaftEntry *pEntries, int n); - -int syncRaftLogAcquire(SSyncRaftLog* pLog, SyncIndex index, int maxMsgSize, - SSyncRaftEntry **ppEntries, int *n); - -void syncRaftLogRelease(SSyncRaftLog* pLog, SyncIndex index, - SSyncRaftEntry *pEntries, int n); - -bool syncRaftLogMatchTerm(); - -static FORCE_INLINE bool syncRaftLogIsCommitted(SSyncRaftLog* pLog, SyncIndex index) { - return pLog->commitIndex > index; -} - -#endif /* _TD_LIBS_SYNC_RAFT_LOG_H */ diff --git a/source/libs/sync/inc/raft_message.h b/source/libs/sync/inc/raft_message.h deleted file mode 100644 index 0d81511756..0000000000 --- a/source/libs/sync/inc/raft_message.h +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_RAFT_MESSAGE_H -#define _TD_LIBS_SYNC_RAFT_MESSAGE_H - -#include "sync.h" -#include "sync_type.h" - -/** - * below define message type which handled by Raft. - * - * internal message, which communicate between threads, start with RAFT_MSG_INTERNAL_*. - * internal message use pointer only and stack memory, need not to be decode/encode and free. - * - * outter message start with RAFT_MSG_*, which communicate between cluster peers, - * need to implement its decode/encode functions. - **/ -typedef enum ESyncRaftMessageType { - // client propose a cmd - RAFT_MSG_INTERNAL_PROP = 1, - - // node election timeout - RAFT_MSG_INTERNAL_ELECTION = 2, - - RAFT_MSG_VOTE = 3, - RAFT_MSG_VOTE_RESP = 4, - - RAFT_MSG_APPEND = 5, - RAFT_MSG_APPEND_RESP = 6, -} ESyncRaftMessageType; - -typedef struct RaftMsgInternal_Prop { - const SSyncBuffer *pBuf; - bool isWeak; - void* pData; -} RaftMsgInternal_Prop; - -typedef struct RaftMsgInternal_Election { - -} RaftMsgInternal_Election; - -typedef struct RaftMsg_Vote { - ESyncRaftElectionType cType; - SyncIndex lastIndex; - SyncTerm lastTerm; -} RaftMsg_Vote; - -typedef struct RaftMsg_VoteResp { - bool rejected; - ESyncRaftElectionType cType; -} RaftMsg_VoteResp; - -typedef struct RaftMsg_Append_Entries { - // index of log entry preceeding new ones - SyncIndex index; - - // term of entry at prevIndex - SyncTerm term; - - // leader's commit index. - SyncIndex commitIndex; - - // size of the log entries array - int nEntries; - - // log entries array - SSyncRaftEntry* entries; -} RaftMsg_Append_Entries; - -typedef struct RaftMsg_Append_Resp { - SyncIndex index; -} RaftMsg_Append_Resp; - -typedef struct SSyncMessage { - ESyncRaftMessageType msgType; - SyncTerm term; - SyncGroupId groupId; - SyncNodeId from; - - union { - RaftMsgInternal_Prop propose; - - RaftMsgInternal_Election election; - - RaftMsg_Vote vote; - RaftMsg_VoteResp voteResp; - - RaftMsg_Append_Entries appendEntries; - RaftMsg_Append_Resp appendResp; - }; -} SSyncMessage; - -static FORCE_INLINE SSyncMessage* syncInitPropMsg(SSyncMessage* pMsg, const SSyncBuffer* pBuf, void* pData, bool isWeak) { - *pMsg = (SSyncMessage) { - .msgType = RAFT_MSG_INTERNAL_PROP, - .term = 0, - .propose = (RaftMsgInternal_Prop) { - .isWeak = isWeak, - .pBuf = pBuf, - .pData = pData, - }, - }; - - return pMsg; -} - -static FORCE_INLINE SSyncMessage* syncInitElectionMsg(SSyncMessage* pMsg, SyncNodeId from) { - *pMsg = (SSyncMessage) { - .msgType = RAFT_MSG_INTERNAL_ELECTION, - .term = 0, - .from = from, - .election = (RaftMsgInternal_Election) { - - }, - }; - - return pMsg; -} - -static FORCE_INLINE SSyncMessage* syncNewVoteMsg(SyncGroupId groupId, SyncNodeId from, - SyncTerm term, ESyncRaftElectionType cType, - SyncIndex lastIndex, SyncTerm lastTerm) { - SSyncMessage* pMsg = (SSyncMessage*)malloc(sizeof(SSyncMessage)); - if (pMsg == NULL) { - return NULL; - } - *pMsg = (SSyncMessage) { - .groupId = groupId, - .from = from, - .term = term, - .msgType = RAFT_MSG_VOTE, - .vote = (RaftMsg_Vote) { - .cType = cType, - .lastIndex = lastIndex, - .lastTerm = lastTerm, - }, - }; - - return pMsg; -} - -static FORCE_INLINE SSyncMessage* syncNewVoteRespMsg(SyncGroupId groupId, SyncNodeId from, - ESyncRaftElectionType cType, bool rejected) { - SSyncMessage* pMsg = (SSyncMessage*)malloc(sizeof(SSyncMessage)); - if (pMsg == NULL) { - return NULL; - } - *pMsg = (SSyncMessage) { - .groupId = groupId, - .from = from, - .msgType = RAFT_MSG_VOTE_RESP, - .voteResp = (RaftMsg_VoteResp) { - .cType = cType, - .rejected = rejected, - }, - }; - - return pMsg; -} - -static FORCE_INLINE SSyncMessage* syncNewAppendMsg(SyncGroupId groupId, SyncNodeId from, - SyncTerm term, SyncIndex logIndex, SyncTerm logTerm, - SyncIndex commitIndex, int nEntries, SSyncRaftEntry* entries) { - SSyncMessage* pMsg = (SSyncMessage*)malloc(sizeof(SSyncMessage)); - if (pMsg == NULL) { - return NULL; - } - *pMsg = (SSyncMessage) { - .groupId = groupId, - .from = from, - .term = term, - .msgType = RAFT_MSG_APPEND, - .appendEntries = (RaftMsg_Append_Entries) { - .index = logIndex, - .term = logTerm, - .commitIndex = commitIndex, - .nEntries = nEntries, - .entries = entries, - }, - }; - - return pMsg; -} - -static FORCE_INLINE SSyncMessage* syncNewEmptyAppendRespMsg(SyncGroupId groupId, SyncNodeId from, SyncTerm term) { - SSyncMessage* pMsg = (SSyncMessage*)malloc(sizeof(SSyncMessage)); - if (pMsg == NULL) { - return NULL; - } - *pMsg = (SSyncMessage) { - .groupId = groupId, - .from = from, - .term = term, - .msgType = RAFT_MSG_APPEND_RESP, - .appendResp = (RaftMsg_Append_Resp) { - - }, - }; - - return pMsg; -} - -static FORCE_INLINE bool syncIsInternalMsg(ESyncRaftMessageType msgType) { - return msgType == RAFT_MSG_INTERNAL_PROP || - msgType == RAFT_MSG_INTERNAL_ELECTION; -} - -static FORCE_INLINE bool syncIsPreVoteRespMsg(const SSyncMessage* pMsg) { - return pMsg->msgType == RAFT_MSG_VOTE_RESP && pMsg->voteResp.cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION; -} - -static FORCE_INLINE bool syncIsPreVoteMsg(const SSyncMessage* pMsg) { - return pMsg->msgType == RAFT_MSG_VOTE && pMsg->voteResp.cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION; -} - -void syncFreeMessage(const SSyncMessage* pMsg); - -// message handlers -int syncRaftHandleElectionMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg); -int syncRaftHandleVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg); -int syncRaftHandleVoteRespMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg); -int syncRaftHandleAppendEntriesMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg); - -#endif /* _TD_LIBS_SYNC_RAFT_MESSAGE_H */ \ No newline at end of file diff --git a/source/libs/sync/inc/raft_replication.h b/source/libs/sync/inc/raft_replication.h deleted file mode 100644 index 180a2db61f..0000000000 --- a/source/libs/sync/inc/raft_replication.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TD_SYNC_RAFT_REPLICATION_H -#define TD_SYNC_RAFT_REPLICATION_H - -#include "sync.h" -#include "syncInt.h" -#include "sync_type.h" - -// syncRaftMaybeSendAppend sends an append RPC with new entries to the given peer, -// if necessary. Returns true if a message was sent. The sendIfEmpty -// argument controls whether messages with no entries will be sent -// ("empty" messages are useful to convey updated Commit indexes, but -// are undesirable when we're sending multiple messages in a batch). -bool syncRaftMaybeSendAppend(SSyncRaft* pRaft, SSyncRaftProgress* progress, bool sendIfEmpty); - -#endif /* TD_SYNC_RAFT_REPLICATION_H */ diff --git a/source/libs/sync/inc/raft_unstable_log.h b/source/libs/sync/inc/raft_unstable_log.h deleted file mode 100644 index 0748a425a1..0000000000 --- a/source/libs/sync/inc/raft_unstable_log.h +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TD_SYNC_RAFT_UNSTABLE_LOG_H -#define TD_SYNC_RAFT_UNSTABLE_LOG_H - -#include "sync_type.h" - -/* in-memory unstable raft log storage */ -struct SSyncRaftUnstableLog { -#if 0 - /* Circular buffer of log entries */ - RaftEntry *entries; - - /* size of Circular buffer */ - int size; - - /* Indexes of used slots [front, back) */ - int front, back; - - /* Index of first entry is offset + 1 */ - SyncIndex offset; - - /* meta data of snapshot */ - SSyncRaftUnstableLog snapshot; -#endif -}; - -/** - * return index of last in memory log, return 0 if log is empty - **/ -//SyncIndex syncRaftLogLastIndex(SSyncRaftUnstableLog* pLog); - -#if 0 -void raftLogInit(RaftLog* pLog); - -void raftLogClose(RaftLog* pLog); - -/** - * When startup populating log entrues loaded from disk, - * init raft memory log with snapshot index,term and log start idnex. - **/ -/* -void raftLogStart(RaftLog* pLog, - RaftSnapshotMeta snapshot, - SyncIndex startIndex); -*/ -/** - * Get the number of entries the log. - **/ -int raftLogNumEntries(const RaftLog* pLog); - - - -/** - * return last term of in memory log, return 0 if log is empty - **/ -SyncTerm raftLogLastTerm(RaftLog* pLog); - -/** - * return term of log with the given index, return 0 if the term of index cannot be found - * , errCode will save the error code. - **/ -SyncTerm raftLogTermOf(RaftLog* pLog, SyncIndex index, RaftCode* errCode); - -/** - * Get the last index of the most recent snapshot. Return 0 if there are no * - * snapshots. - **/ -SyncIndex raftLogSnapshotIndex(RaftLog* pLog); - -/* Append a new entry to the log. */ -int raftLogAppend(RaftLog* pLog, - SyncTerm term, - const SSyncBuffer *buf); - -/** - * acquire log from given index onwards. - **/ -/* -int raftLogAcquire(RaftLog* pLog, - SyncIndex index, - RaftEntry **ppEntries, - int *n); - -void raftLogRelease(RaftLog* pLog, - SyncIndex index, - RaftEntry *pEntries, - int n); -*/ -/* Delete all entries from the given index (included) onwards. */ -void raftLogTruncate(RaftLog* pLog, SyncIndex index); - -/** - * when taking a new snapshot, the function will update the last snapshot information and delete - * all entries up last_index - trailing (included). If the log contains no entry - * a last_index - trailing, then no entry will be deleted. - **/ -void raftLogSnapshot(RaftLog* pLog, SyncIndex index, SyncIndex trailing); - -#endif - -#endif /* TD_SYNC_RAFT_UNSTABLE_LOG_H */ \ No newline at end of file diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h deleted file mode 100644 index f99fb066ae..0000000000 --- a/source/libs/sync/inc/syncInt.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_INT_H -#define _TD_LIBS_SYNC_INT_H - -#include "thash.h" -#include "os.h" -#include "sync.h" -#include "sync_type.h" -#include "raft.h" -#include "tlog.h" - -#define TAOS_SYNC_MAX_WORKER 3 - -typedef struct SSyncWorker { - pthread_t thread; -} SSyncWorker; - -struct SSyncNode { - pthread_mutex_t mutex; - int32_t refCount; - SyncGroupId vgId; - SSyncRaft raft; - void* syncTimer; -}; - -typedef struct SSyncManager { - pthread_mutex_t mutex; - - // sync server rpc - void* serverRpc; - // rpc server hash table base on FQDN:port key - SHashObj* rpcServerTable; - - // sync client rpc - void* clientRpc; - - // worker threads - SSyncWorker worker[TAOS_SYNC_MAX_WORKER]; - - // vgroup hash table - SHashObj* vgroupTable; - - // timer manager - void* syncTimerManager; - -} SSyncManager; - -extern SSyncManager* gSyncManager; - -#define syncFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYNC FATAL ", 255, __VA_ARGS__); }} while(0) -#define syncError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("SYNC ERROR ", 255, __VA_ARGS__); }} while(0) -#define syncWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("SYNC WARN ", 255, __VA_ARGS__); }} while(0) -#define syncInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("SYNC ", 255, __VA_ARGS__); }} while(0) -#define syncDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYNC ", sDebugFlag, __VA_ARGS__); }} while(0) -#define syncTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYNC ", sDebugFlag, __VA_ARGS__); }} while(0) - -#endif /* _TD_LIBS_SYNC_INT_H */ \ No newline at end of file diff --git a/source/libs/sync/inc/sync_const.h b/source/libs/sync/inc/sync_const.h deleted file mode 100644 index b49c17f82e..0000000000 --- a/source/libs/sync/inc/sync_const.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_CONST_H -#define _TD_LIBS_SYNC_CONST_H - -#include "sync.h" - -static int kSyncRaftMaxInflghtMsgs = 20; - -static SyncIndex kMaxCommitIndex = UINT64_MAX; - -#endif /* _TD_LIBS_SYNC_CONST_H */ diff --git a/source/libs/sync/inc/sync_raft_config_change.h b/source/libs/sync/inc/sync_raft_config_change.h deleted file mode 100644 index 75a29f35e8..0000000000 --- a/source/libs/sync/inc/sync_raft_config_change.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TD_SYNC_RAFT_CONFIG_CHANGE_H -#define TD_SYNC_RAFT_CONFIG_CHANGE_H - -#include "sync_type.h" -#include "sync_raft_proto.h" - -/** - * Changer facilitates configuration changes. It exposes methods to handle - * simple and joint consensus while performing the proper validation that allows - * refusing invalid configuration changes before they affect the active - * configuration. - **/ -struct SSyncRaftChanger { - SSyncRaftProgressTracker* tracker; - SyncIndex lastIndex; -}; - -typedef int (*configChangeFp)(SSyncRaftChanger* changer, const SSyncConfChangeSingleArray* css, - SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); - -// Simple carries out a series of configuration changes that (in aggregate) -// mutates the incoming majority config Voters[0] by at most one. This method -// will return an error if that is not the case, if the resulting quorum is -// zero, or if the configuration is in a joint state (i.e. if there is an -// outgoing configuration). -int syncRaftChangerSimpleConfig(SSyncRaftChanger* changer, const SSyncConfChangeSingleArray* css, - SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); - -int syncRaftChangerEnterJoint(SSyncRaftChanger* changer, bool autoLeave, const SSyncConfChangeSingleArray* css, - SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); - -#endif /* TD_SYNC_RAFT_CONFIG_CHANGE_H */ diff --git a/source/libs/sync/inc/sync_raft_impl.h b/source/libs/sync/inc/sync_raft_impl.h deleted file mode 100644 index 1a6c13f65f..0000000000 --- a/source/libs/sync/inc/sync_raft_impl.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_RAFT_IMPL_H -#define _TD_LIBS_SYNC_RAFT_IMPL_H - -#include "sync.h" -#include "sync_type.h" -#include "raft_message.h" -#include "sync_raft_quorum.h" - -void syncRaftBecomeFollower(SSyncRaft* pRaft, SyncTerm term, SyncNodeId leaderId); -void syncRaftBecomePreCandidate(SSyncRaft* pRaft); -void syncRaftBecomeCandidate(SSyncRaft* pRaft); -void syncRaftBecomeLeader(SSyncRaft* pRaft); - -void syncRaftStartElection(SSyncRaft* pRaft, ESyncRaftElectionType cType); - -void syncRaftCampaign(SSyncRaft* pRaft, ESyncRaftElectionType cType); - -void syncRaftTriggerHeartbeat(SSyncRaft* pRaft); - -void syncRaftRandomizedElectionTimeout(SSyncRaft* pRaft); -bool syncRaftIsPromotable(SSyncRaft* pRaft); -bool syncRaftIsPastElectionTimeout(SSyncRaft* pRaft); -int syncRaftQuorum(SSyncRaft* pRaft); - -bool syncRaftMaybeCommit(SSyncRaft* pRaft); - -ESyncRaftVoteResult syncRaftPollVote(SSyncRaft* pRaft, SyncNodeId id, - bool preVote, bool accept, - int* rejectNum, int *granted); - -static FORCE_INLINE bool syncRaftIsEmptyServerState(const SSyncServerState* serverState) { - return serverState->commitIndex == 0 && - serverState->term == SYNC_NON_TERM && - serverState->voteFor == SYNC_NON_NODE_ID; -} - -void syncRaftLoadState(SSyncRaft* pRaft, const SSyncServerState* serverState); - -void syncRaftBroadcastAppend(SSyncRaft* pRaft); - -SNodeInfo* syncRaftGetNodeById(SSyncRaft *pRaft, SyncNodeId id); - -#endif /* _TD_LIBS_SYNC_RAFT_IMPL_H */ diff --git a/source/libs/sync/inc/sync_raft_inflights.h b/source/libs/sync/inc/sync_raft_inflights.h deleted file mode 100644 index 627bf9a26f..0000000000 --- a/source/libs/sync/inc/sync_raft_inflights.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TD_SYNC_RAFT_INFLIGHTS_H -#define TD_SYNC_RAFT_INFLIGHTS_H - -#include "sync.h" - -// Inflights limits the number of MsgApp (represented by the largest index -// contained within) sent to followers but not yet acknowledged by them. Callers -// use Full() to check whether more messages can be sent, call Add() whenever -// they are sending a new append, and release "quota" via FreeLE() whenever an -// ack is received. -typedef struct SSyncRaftInflights { - // the starting index in the buffer - int start; - - // number of inflights in the buffer - int count; - - // the size of the buffer - int size; - - // buffer contains the index of the last entry - // inside one message. - SyncIndex* buffer; -} SSyncRaftInflights; - -SSyncRaftInflights* syncRaftOpenInflights(int size); -void syncRaftCloseInflights(SSyncRaftInflights*); - -// reset frees all inflights. -static FORCE_INLINE void syncRaftInflightReset(SSyncRaftInflights* inflights) { - inflights->count = 0; - inflights->start = 0; -} - -// Full returns true if no more messages can be sent at the moment. -static FORCE_INLINE bool syncRaftInflightFull(SSyncRaftInflights* inflights) { - return inflights->count == inflights->size; -} - -// Add notifies the Inflights that a new message with the given index is being -// dispatched. Full() must be called prior to Add() to verify that there is room -// for one more message, and consecutive calls to add Add() must provide a -// monotonic sequence of indexes. -void syncRaftInflightAdd(SSyncRaftInflights* inflights, SyncIndex inflightIndex); - -// FreeLE frees the inflights smaller or equal to the given `to` flight. -void syncRaftInflightFreeLE(SSyncRaftInflights* inflights, SyncIndex toIndex); - -/** - * syncRaftInflightFreeFirstOne releases the first inflight. - * This is a no-op if nothing is inflight. - **/ -void syncRaftInflightFreeFirstOne(SSyncRaftInflights* inflights); - -#endif /* TD_SYNC_RAFT_INFLIGHTS_H */ \ No newline at end of file diff --git a/source/libs/sync/inc/sync_raft_node_map.h b/source/libs/sync/inc/sync_raft_node_map.h deleted file mode 100644 index b4cf04056d..0000000000 --- a/source/libs/sync/inc/sync_raft_node_map.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_RAFT_NODE_MAP_H -#define _TD_LIBS_SYNC_RAFT_NODE_MAP_H - -#include "thash.h" -#include "sync.h" -#include "sync_type.h" - -struct SSyncRaftNodeMap { - SHashObj* nodeIdMap; -}; - -void syncRaftInitNodeMap(SSyncRaftNodeMap* nodeMap); -void syncRaftFreeNodeMap(SSyncRaftNodeMap* nodeMap); - -void syncRaftClearNodeMap(SSyncRaftNodeMap* nodeMap); - -bool syncRaftIsInNodeMap(const SSyncRaftNodeMap* nodeMap, SyncNodeId nodeId); - -void syncRaftCopyNodeMap(SSyncRaftNodeMap* from, SSyncRaftNodeMap* to); - -void syncRaftUnionNodeMap(SSyncRaftNodeMap* nodeMap, SSyncRaftNodeMap* to); - -void syncRaftAddToNodeMap(SSyncRaftNodeMap* nodeMap, SyncNodeId nodeId); - -void syncRaftRemoveFromNodeMap(SSyncRaftNodeMap* nodeMap, SyncNodeId nodeId); - -int32_t syncRaftNodeMapSize(const SSyncRaftNodeMap* nodeMap); - -// return true if reach the end -bool syncRaftIterateNodeMap(const SSyncRaftNodeMap* nodeMap, SyncNodeId *pId); - -bool syncRaftIsAllNodeInProgressMap(SSyncRaftNodeMap* nodeMap, SSyncRaftProgressMap* progressMap); - -#endif /* _TD_LIBS_SYNC_RAFT_NODE_MAP_H */ \ No newline at end of file diff --git a/source/libs/sync/inc/sync_raft_progress.h b/source/libs/sync/inc/sync_raft_progress.h deleted file mode 100644 index 32c21281cd..0000000000 --- a/source/libs/sync/inc/sync_raft_progress.h +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TD_SYNC_RAFT_PROGRESS_H -#define TD_SYNC_RAFT_PROGRESS_H - -#include "sync_type.h" -#include "sync_raft_inflights.h" -#include "thash.h" - -/** - * State defines how the leader should interact with the follower. - * - * When in PROGRESS_STATE_PROBE, leader sends at most one replication message - * per heartbeat interval. It also probes actual progress of the follower. - * - * When in PROGRESS_STATE_REPLICATE, leader optimistically increases next - * to the latest entry sent after sending replication message. This is - * an optimized state for fast replicating log entries to the follower. - * - * When in PROGRESS_STATE_SNAPSHOT, leader should have sent out snapshot - * before and stops sending any replication message. - * - * PROGRESS_STATE_PROBE is the initial state. - **/ -typedef enum ESyncRaftProgressState { - /** - * StateProbe indicates a follower whose last index isn't known. Such a - * follower is "probed" (i.e. an append sent periodically) to narrow down - * its last index. In the ideal (and common) case, only one round of probing - * is necessary as the follower will react with a hint. Followers that are - * probed over extended periods of time are often offline. - **/ - PROGRESS_STATE_PROBE = 0, - - /** - * StateReplicate is the state steady in which a follower eagerly receives - * log entries to append to its log. - **/ - PROGRESS_STATE_REPLICATE, - - /** - * StateSnapshot indicates a follower that needs log entries not available - * from the leader's Raft log. Such a follower needs a full snapshot to - * return to StateReplicate. - **/ - PROGRESS_STATE_SNAPSHOT, -} ESyncRaftProgressState; - -static const char* kProgressStateString[] = { - "Probe", - "Replicate", - "Snapshot", -}; - -// Progress represents a follower’s progress in the view of the leader. Leader -// maintains progresses of all followers, and sends entries to the follower -// based on its progress. -// -// NB(tbg): Progress is basically a state machine whose transitions are mostly -// strewn around `*raft.raft`. Additionally, some fields are only used when in a -// certain State. All of this isn't ideal. -struct SSyncRaftProgress { - SyncGroupId groupId; - - SyncNodeId id; - - int16_t refCount; - - SyncIndex nextIndex; - - SyncIndex matchIndex; - - // State defines how the leader should interact with the follower. - // - // When in StateProbe, leader sends at most one replication message - // per heartbeat interval. It also probes actual progress of the follower. - // - // When in StateReplicate, leader optimistically increases next - // to the latest entry sent after sending replication message. This is - // an optimized state for fast replicating log entries to the follower. - // - // When in StateSnapshot, leader should have sent out snapshot - // before and stops sending any replication message. - ESyncRaftProgressState state; - - // PendingSnapshot is used in StateSnapshot. - // If there is a pending snapshot, the pendingSnapshot will be set to the - // index of the snapshot. If pendingSnapshot is set, the replication process of - // this Progress will be paused. raft will not resend snapshot until the pending one - // is reported to be failed. - SyncIndex pendingSnapshotIndex; - - // RecentActive is true if the progress is recently active. Receiving any messages - // from the corresponding follower indicates the progress is active. - // RecentActive can be reset to false after an election timeout. - // - // TODO(tbg): the leader should always have this set to true. - bool recentActive; - - // ProbeSent is used while this follower is in StateProbe. When ProbeSent is - // true, raft should pause sending replication message to this peer until - // ProbeSent is reset. See ProbeAcked() and IsPaused(). - bool probeSent; - - // Inflights is a sliding window for the inflight messages. - // Each inflight message contains one or more log entries. - // The max number of entries per message is defined in raft config as MaxSizePerMsg. - // Thus inflight effectively limits both the number of inflight messages - // and the bandwidth each Progress can use. - // When inflights is Full, no more message should be sent. - // When a leader sends out a message, the index of the last - // entry should be added to inflights. The index MUST be added - // into inflights in order. - // When a leader receives a reply, the previous inflights should - // be freed by calling inflights.FreeLE with the index of the last - // received entry. - SSyncRaftInflights* inflights; - - // IsLearner is true if this progress is tracked for a learner. - bool isLearner; -}; - -struct SSyncRaftProgressMap { - // map nodeId -> SSyncRaftProgress* - SHashObj* progressMap; -}; - -static FORCE_INLINE const char* syncRaftProgressStateString(const SSyncRaftProgress* progress) { - return kProgressStateString[progress->state]; -} - -void syncRaftResetProgress(SSyncRaft* pRaft, SSyncRaftProgress* progress); - -// BecomeProbe transitions into StateProbe. Next is reset to Match+1 or, -// optionally and if larger, the index of the pending snapshot. -void syncRaftProgressBecomeProbe(SSyncRaftProgress* progress); - -// BecomeReplicate transitions into StateReplicate, resetting Next to Match+1. -void syncRaftProgressBecomeReplicate(SSyncRaftProgress* progress); - -// MaybeUpdate is called when an MsgAppResp arrives from the follower, with the -// index acked by it. The method returns false if the given n index comes from -// an outdated message. Otherwise it updates the progress and returns true. -bool syncRaftProgressMaybeUpdate(SSyncRaftProgress* progress, SyncIndex lastIndex); - -// OptimisticUpdate signals that appends all the way up to and including index n -// are in-flight. As a result, Next is increased to n+1. -static FORCE_INLINE void syncRaftProgressOptimisticNextIndex(SSyncRaftProgress* progress, SyncIndex nextIndex) { - progress->nextIndex = nextIndex + 1; -} - -// MaybeDecrTo adjusts the Progress to the receipt of a MsgApp rejection. The -// arguments are the index of the append message rejected by the follower, and -// the hint that we want to decrease to. -// -// Rejections can happen spuriously as messages are sent out of order or -// duplicated. In such cases, the rejection pertains to an index that the -// Progress already knows were previously acknowledged, and false is returned -// without changing the Progress. -// -// If the rejection is genuine, Next is lowered sensibly, and the Progress is -// cleared for sending log entries. -bool syncRaftProgressMaybeDecrTo(SSyncRaftProgress* progress, - SyncIndex rejected, SyncIndex matchHint); - -// IsPaused returns whether sending log entries to this node has been throttled. -// This is done when a node has rejected recent MsgApps, is currently waiting -// for a snapshot, or has reached the MaxInflightMsgs limit. In normal -// operation, this is false. A throttled node will be contacted less frequently -// until it has reached a state in which it's able to accept a steady stream of -// log entries again. -bool syncRaftProgressIsPaused(SSyncRaftProgress* progress); - -static FORCE_INLINE SyncIndex syncRaftProgressNextIndex(SSyncRaftProgress* progress) { - return progress->nextIndex; -} - -static FORCE_INLINE ESyncRaftProgressState syncRaftProgressInReplicate(SSyncRaftProgress* progress) { - return progress->state == PROGRESS_STATE_REPLICATE; -} - -static FORCE_INLINE ESyncRaftProgressState syncRaftProgressInSnapshot(SSyncRaftProgress* progress) { - return progress->state == PROGRESS_STATE_SNAPSHOT; -} - -static FORCE_INLINE ESyncRaftProgressState syncRaftProgressInProbe(SSyncRaftProgress* progress) { - return progress->state == PROGRESS_STATE_PROBE; -} - -static FORCE_INLINE bool syncRaftProgressRecentActive(SSyncRaftProgress* progress) { - return progress->recentActive; -} - -void syncRaftInitProgressMap(SSyncRaftProgressMap* progressMap); -void syncRaftFreeProgressMap(SSyncRaftProgressMap* progressMap); - -void syncRaftClearProgressMap(SSyncRaftProgressMap* progressMap); -void syncRaftCopyProgressMap(SSyncRaftProgressMap* from, SSyncRaftProgressMap* to); - -SSyncRaftProgress* syncRaftFindProgressByNodeId(const SSyncRaftProgressMap* progressMap, SyncNodeId id); - -int syncRaftAddToProgressMap(SSyncRaftProgressMap* progressMap, SSyncRaftProgress* progress); - -void syncRaftRemoveFromProgressMap(SSyncRaftProgressMap* progressMap, SyncNodeId id); - -bool syncRaftIsInProgressMap(SSyncRaftProgressMap* progressMap, SyncNodeId id); - -/** - * return true if progress's log is up-todate - **/ -bool syncRaftProgressIsUptodate(SSyncRaft* pRaft, SSyncRaftProgress* progress); - -// BecomeSnapshot moves the Progress to StateSnapshot with the specified pending -// snapshot index. -void syncRaftProgressBecomeSnapshot(SSyncRaftProgress* progress, SyncIndex snapshotIndex); - -void syncRaftCopyProgress(const SSyncRaftProgress* from, SSyncRaftProgress* to); - -// return true if reach the end -bool syncRaftIterateProgressMap(const SSyncRaftProgressMap* progressMap, SSyncRaftProgress *pProgress); - -bool syncRaftVisitProgressMap(SSyncRaftProgressMap* progressMap, visitProgressFp fp, void* arg); - -#if 0 - -void syncRaftProgressAbortSnapshot(SSyncRaft* pRaft, int i); - - - -SyncIndex syncRaftProgressMatchIndex(SSyncRaft* pRaft, int i); - -void syncRaftProgressUpdateLastSend(SSyncRaft* pRaft, int i); - -void syncRaftProgressUpdateSnapshotLastSend(SSyncRaft* pRaft, int i); - -bool syncRaftProgressResetRecentRecv(SSyncRaft* pRaft, int i); - -void syncRaftProgressMarkRecentRecv(SSyncRaft* pRaft, int i); - - - -void syncRaftProgressAbortSnapshot(SSyncRaft* pRaft, int i); - -#endif - -#endif /* TD_SYNC_RAFT_PROGRESS_H */ \ No newline at end of file diff --git a/source/libs/sync/inc/sync_raft_progress_tracker.h b/source/libs/sync/inc/sync_raft_progress_tracker.h deleted file mode 100644 index 0a3c7dd6fc..0000000000 --- a/source/libs/sync/inc/sync_raft_progress_tracker.h +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_RAFT_PROGRESS_TRACKER_H -#define _TD_LIBS_SYNC_RAFT_PROGRESS_TRACKER_H - -#include "sync_type.h" -#include "sync_raft_quorum.h" -#include "sync_raft_quorum_joint.h" -#include "sync_raft_progress.h" -#include "sync_raft_proto.h" -#include "thash.h" - -// Config reflects the configuration tracked in a ProgressTracker. -struct SSyncRaftProgressTrackerConfig { - SSyncRaftQuorumJointConfig voters; - - // autoLeave is true if the configuration is joint and a transition to the - // incoming configuration should be carried out automatically by Raft when - // this is possible. If false, the configuration will be joint until the - // application initiates the transition manually. - bool autoLeave; - - // Learners is a set of IDs corresponding to the learners active in the - // current configuration. - // - // Invariant: Learners and Voters does not intersect, i.e. if a peer is in - // either half of the joint config, it can't be a learner; if it is a - // learner it can't be in either half of the joint config. This invariant - // simplifies the implementation since it allows peers to have clarity about - // its current role without taking into account joint consensus. - SSyncRaftNodeMap learners; - - // When we turn a voter into a learner during a joint consensus transition, - // we cannot add the learner directly when entering the joint state. This is - // because this would violate the invariant that the intersection of - // voters and learners is empty. For example, assume a Voter is removed and - // immediately re-added as a learner (or in other words, it is demoted): - // - // Initially, the configuration will be - // - // voters: {1 2 3} - // learners: {} - // - // and we want to demote 3. Entering the joint configuration, we naively get - // - // voters: {1 2} & {1 2 3} - // learners: {3} - // - // but this violates the invariant (3 is both voter and learner). Instead, - // we get - // - // voters: {1 2} & {1 2 3} - // learners: {} - // next_learners: {3} - // - // Where 3 is now still purely a voter, but we are remembering the intention - // to make it a learner upon transitioning into the final configuration: - // - // voters: {1 2} - // learners: {3} - // next_learners: {} - // - // Note that next_learners is not used while adding a learner that is not - // also a voter in the joint config. In this case, the learner is added - // right away when entering the joint configuration, so that it is caught up - // as soon as possible. - SSyncRaftNodeMap learnersNext; -}; - -struct SSyncRaftProgressTracker { - SSyncRaftProgressTrackerConfig config; - - SSyncRaftProgressMap progressMap; - - // nodeid -> ESyncRaftVoteType map - SHashObj* votesMap; - - int maxInflightMsgs; - - SSyncRaft* pRaft; -}; - -SSyncRaftProgressTracker* syncRaftOpenProgressTracker(SSyncRaft* pRaft); - -void syncRaftInitTrackConfig(SSyncRaftProgressTrackerConfig* config); -void syncRaftFreeTrackConfig(SSyncRaftProgressTrackerConfig* config); - -void syncRaftFreeTrackConfig(SSyncRaftProgressTrackerConfig* config); - -// ResetVotes prepares for a new round of vote counting via recordVote. -void syncRaftResetVotes(SSyncRaftProgressTracker*); - -void syncRaftProgressVisit(SSyncRaftProgressTracker*, visitProgressFp visit, void* arg); - -// RecordVote records that the node with the given id voted for this Raft -// instance if v == true (and declined it otherwise). -void syncRaftRecordVote(SSyncRaftProgressTracker* tracker, SyncNodeId id, bool grant); - -void syncRaftCopyTrackerConfig(const SSyncRaftProgressTrackerConfig* from, SSyncRaftProgressTrackerConfig* to); - -int syncRaftCheckTrackerConfigInProgress(SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); - -// TallyVotes returns the number of granted and rejected Votes, and whether the -// election outcome is known. -ESyncRaftVoteResult syncRaftTallyVotes(SSyncRaftProgressTracker* tracker, int* rejected, int *granted); - -void syncRaftConfigState(SSyncRaftProgressTracker* tracker, SSyncConfigState* cs); - -// Committed returns the largest log index known to be committed based on what -// the voting members of the group have acknowledged. -SyncIndex syncRaftCommittedIndex(SSyncRaftProgressTracker* tracker); - -// QuorumActive returns true if the quorum is active from the view of the local -// raft state machine. Otherwise, it returns false. -bool syncRaftQuorumActive(SSyncRaftProgressTracker* tracker); - -bool syncRaftIsInNodeMap(const SSyncRaftNodeMap* nodeMap, SyncNodeId nodeId); - -#endif /* _TD_LIBS_SYNC_RAFT_PROGRESS_TRACKER_H */ diff --git a/source/libs/sync/inc/sync_raft_proto.h b/source/libs/sync/inc/sync_raft_proto.h deleted file mode 100644 index 29371e328d..0000000000 --- a/source/libs/sync/inc/sync_raft_proto.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TD_SYNC_RAFT_PROTO_H -#define TD_SYNC_RAFT_PROTO_H - -#include "sync_type.h" -#include "sync_raft_node_map.h" - -typedef enum ESyncRaftConfChangeType { - SYNC_RAFT_Conf_AddNode = 0, - SYNC_RAFT_Conf_RemoveNode = 1, - SYNC_RAFT_Conf_UpdateNode = 2, - SYNC_RAFT_Conf_AddLearnerNode = 3, -} ESyncRaftConfChangeType; - -// ConfChangeSingle is an individual configuration change operation. Multiple -// such operations can be carried out atomically via a ConfChangeV2. -typedef struct SSyncConfChangeSingle { - ESyncRaftConfChangeType type; - SyncNodeId nodeId; -} SSyncConfChangeSingle; - -typedef struct SSyncConfChangeSingleArray { - int n; - SSyncConfChangeSingle* changes; -} SSyncConfChangeSingleArray; - -typedef struct SSyncConfigState { - // The voters in the incoming config. (If the configuration is not joint, - // then the outgoing config is empty). - SSyncRaftNodeMap voters; - - // The learners in the incoming config. - SSyncRaftNodeMap learners; - - // The voters in the outgoing config. - SSyncRaftNodeMap votersOutgoing; - - // The nodes that will become learners when the outgoing config is removed. - // These nodes are necessarily currently in nodes_joint (or they would have - // been added to the incoming config right away). - SSyncRaftNodeMap learnersNext; - - // If set, the config is joint and Raft will automatically transition into - // the final config (i.e. remove the outgoing config) when this is safe. - bool autoLeave; -} SSyncConfigState; - -static FORCE_INLINE bool syncRaftConfArrayIsEmpty(const SSyncConfChangeSingleArray* ary) { - return ary->n == 0; -} - -static FORCE_INLINE void syncRaftInitConfArray(SSyncConfChangeSingleArray* ary) { - *ary = (SSyncConfChangeSingleArray) { - .changes = NULL, - .n = 0, - }; -} - -static FORCE_INLINE void syncRaftFreeConfArray(SSyncConfChangeSingleArray* ary) { - if (ary->changes != NULL) free(ary->changes); -} - -#endif /* TD_SYNC_RAFT_PROTO_H */ diff --git a/source/libs/sync/inc/sync_raft_quorum.h b/source/libs/sync/inc/sync_raft_quorum.h deleted file mode 100644 index 16ac1cd029..0000000000 --- a/source/libs/sync/inc/sync_raft_quorum.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TD_SYNC_RAFT_QUORUM_H -#define TD_SYNC_RAFT_QUORUM_H - -/** - * ESyncRaftVoteResult indicates the outcome of a vote. - **/ -typedef enum { - /** - * SYNC_RAFT_VOTE_PENDING indicates that the decision of the vote depends on future - * votes, i.e. neither "yes" or "no" has reached quorum yet. - **/ - SYNC_RAFT_VOTE_PENDING = 1, - - /** - * SYNC_RAFT_VOTE_LOST indicates that the quorum has voted "no". - **/ - SYNC_RAFT_VOTE_LOST = 2, - - /** - * SYNC_RAFT_VOTE_WON indicates that the quorum has voted "yes". - **/ - SYNC_RAFT_VOTE_WON = 3, -} ESyncRaftVoteResult; - -#endif /* TD_SYNC_RAFT_QUORUM_H */ \ No newline at end of file diff --git a/source/libs/sync/inc/sync_raft_quorum_joint.h b/source/libs/sync/inc/sync_raft_quorum_joint.h deleted file mode 100644 index 9d5f10ab51..0000000000 --- a/source/libs/sync/inc/sync_raft_quorum_joint.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_RAFT_QUORUM_JOINT_H -#define _TD_LIBS_SYNC_RAFT_QUORUM_JOINT_H - -#include "taosdef.h" -#include "sync.h" -#include "sync_type.h" -#include "sync_raft_node_map.h" -#include "thash.h" - -// JointConfig is a configuration of two groups of (possibly overlapping) -// majority configurations. Decisions require the support of both majorities. -typedef struct SSyncRaftQuorumJointConfig { - SSyncRaftNodeMap outgoing; - SSyncRaftNodeMap incoming; -} SSyncRaftQuorumJointConfig; - -// IDs returns a newly initialized map representing the set of voters present -// in the joint configuration. -void syncRaftJointConfigIDs(SSyncRaftQuorumJointConfig* config, SSyncRaftNodeMap* nodeMap); - -// CommittedIndex returns the largest committed index for the given joint -// quorum. An index is jointly committed if it is committed in both constituent -// majorities. -SyncIndex syncRaftJointConfigCommittedIndex(const SSyncRaftQuorumJointConfig* config, matchAckIndexerFp indexer, void* arg); - -// VoteResult takes a mapping of voters to yes/no (true/false) votes and returns -// a result indicating whether the vote is pending, lost, or won. A joint quorum -// requires both majority quorums to vote in favor. -ESyncRaftVoteType syncRaftVoteResult(SSyncRaftQuorumJointConfig* config, SHashObj* votesMap); - -void syncRaftInitQuorumJointConfig(SSyncRaftQuorumJointConfig* config); - -static FORCE_INLINE bool syncRaftJointConfigInOutgoing(const SSyncRaftQuorumJointConfig* config, SyncNodeId id) { - return syncRaftIsInNodeMap(&config->outgoing, id); -} - -static FORCE_INLINE bool syncRaftJointConfigInIncoming(const SSyncRaftQuorumJointConfig* config, SyncNodeId id) { - return syncRaftIsInNodeMap(&config->incoming, id); -} - -void syncRaftJointConfigAddToIncoming(SSyncRaftQuorumJointConfig* config, SyncNodeId id); - -void syncRaftJointConfigRemoveFromIncoming(SSyncRaftQuorumJointConfig* config, SyncNodeId id); - -static FORCE_INLINE const SSyncRaftNodeMap* syncRaftJointConfigIncoming(const SSyncRaftQuorumJointConfig* config) { - return &config->incoming; -} - -static FORCE_INLINE const SSyncRaftNodeMap* syncRaftJointConfigOutgoing(const SSyncRaftQuorumJointConfig* config) { - return &config->outgoing; -} - -static FORCE_INLINE void syncRaftJointConfigClearOutgoing(SSyncRaftQuorumJointConfig* config) { - syncRaftClearNodeMap(&config->outgoing); -} - -static FORCE_INLINE bool syncRaftJointConfigIsIncomingEmpty(const SSyncRaftQuorumJointConfig* config) { - return syncRaftNodeMapSize(&config->incoming) == 0; -} - -static FORCE_INLINE bool syncRaftJointConfigIsOutgoingEmpty(const SSyncRaftQuorumJointConfig* config) { - return syncRaftNodeMapSize(&config->outgoing) == 0; -} - -static FORCE_INLINE bool syncRaftJointConfigIsInOutgoing(const SSyncRaftQuorumJointConfig* config, SyncNodeId id) { - return syncRaftIsInNodeMap(&config->outgoing, id); -} - -#endif /* _TD_LIBS_SYNC_RAFT_QUORUM_JOINT_H */ diff --git a/source/libs/sync/inc/sync_raft_quorum_majority.h b/source/libs/sync/inc/sync_raft_quorum_majority.h deleted file mode 100644 index 399bd71db8..0000000000 --- a/source/libs/sync/inc/sync_raft_quorum_majority.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_RAFT_QUORUM_MAJORITY_H -#define _TD_LIBS_SYNC_RAFT_QUORUM_MAJORITY_H - -#include "sync.h" -#include "sync_type.h" -#include "sync_raft_quorum.h" -#include "thash.h" - -/** - * syncRaftMajorityVoteResult takes a mapping of voters to yes/no (true/false) votes and returns - * a result indicating whether the vote is pending (i.e. neither a quorum of - * yes/no has been reached), won (a quorum of yes has been reached), or lost (a - * quorum of no has been reached). - **/ -ESyncRaftVoteResult syncRaftMajorityVoteResult(SSyncRaftNodeMap* config, SHashObj* votesMap); - -// CommittedIndex computes the committed index from those supplied via the -// provided AckedIndexer (for the active config). -SyncIndex syncRaftMajorityConfigCommittedIndex(const SSyncRaftNodeMap* config, matchAckIndexerFp indexer, void* arg); - -#endif /* _TD_LIBS_SYNC_RAFT_QUORUM_MAJORITY_H */ diff --git a/source/libs/sync/inc/sync_raft_restore.h b/source/libs/sync/inc/sync_raft_restore.h deleted file mode 100644 index df4448cab8..0000000000 --- a/source/libs/sync/inc/sync_raft_restore.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TD_SYNC_RAFT_RESTORE_H -#define TD_SYNC_RAFT_RESTORE_H - -#include "sync_type.h" -#include "sync_raft_proto.h" - -// syncRaftRestoreConfig takes a Changer (which must represent an empty configuration), and -// runs a sequence of changes enacting the configuration described in the -// ConfState. -// -// TODO(tbg) it's silly that this takes a Changer. Unravel this by making sure -// the Changer only needs a ProgressMap (not a whole Tracker) at which point -// this can just take LastIndex and MaxInflight directly instead and cook up -// the results from that alone. -int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs, - SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); - -#endif /* TD_SYNC_RAFT_RESTORE_H */ diff --git a/source/libs/sync/inc/sync_type.h b/source/libs/sync/inc/sync_type.h deleted file mode 100644 index c5c4cc3a76..0000000000 --- a/source/libs/sync/inc/sync_type.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_TYPE_H -#define _TD_LIBS_SYNC_TYPE_H - -#include -#include "sync.h" -#include "osMath.h" - -#define SYNC_NON_NODE_ID -1 -#define SYNC_NON_TERM 0 - -typedef int32_t SyncTime; -typedef uint32_t SyncTick; - -typedef struct SSyncRaft SSyncRaft; - -typedef struct SSyncRaftProgress SSyncRaftProgress; -typedef struct SSyncRaftProgressMap SSyncRaftProgressMap; -typedef struct SSyncRaftProgressTrackerConfig SSyncRaftProgressTrackerConfig; - -typedef struct SSyncRaftNodeMap SSyncRaftNodeMap; - -typedef struct SSyncRaftProgressTracker SSyncRaftProgressTracker; - -typedef struct SSyncRaftChanger SSyncRaftChanger; - -typedef struct SSyncRaftLog SSyncRaftLog; - -typedef struct SSyncRaftEntry SSyncRaftEntry; - -#if 0 -#ifndef TMIN -#define TMIN(x, y) (((x) < (y)) ? (x) : (y)) -#endif - -#ifndef TMAX -#define TMAX(x, y) (((x) > (y)) ? (x) : (y)) -#endif -#endif - - -typedef struct SSyncServerState { - SyncNodeId voteFor; - SyncTerm term; - SyncIndex commitIndex; -} SSyncServerState; - -typedef struct SSyncClusterConfig { - // Log index number of current cluster config. - SyncIndex index; - - // Log index number of previous cluster config. - SyncIndex prevIndex; - - // current cluster - const SSyncCluster* cluster; -} SSyncClusterConfig; - -typedef enum { - SYNC_RAFT_CAMPAIGN_PRE_ELECTION = 0, - SYNC_RAFT_CAMPAIGN_ELECTION = 1, - SYNC_RAFT_CAMPAIGN_TRANSFER = 2, -} ESyncRaftElectionType; - -typedef enum { - // grant the vote request - SYNC_RAFT_VOTE_RESP_GRANT = 1, - - // reject the vote request - SYNC_RAFT_VOTE_RESP_REJECT = 2, -} ESyncRaftVoteType; - -typedef void (*visitProgressFp)(SSyncRaftProgress* progress, void* arg); - -typedef void (*matchAckIndexerFp)(SyncNodeId id, void* arg, SyncIndex* index); - -#endif /* _TD_LIBS_SYNC_TYPE_H */ diff --git a/source/libs/sync/src/raft.c b/source/libs/sync/src/raft.c deleted file mode 100644 index 72b0d268a8..0000000000 --- a/source/libs/sync/src/raft.c +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "raft.h" -#include "sync_raft_impl.h" -#include "raft_log.h" -#include "sync_raft_restore.h" -#include "raft_replication.h" -#include "sync_raft_config_change.h" -#include "sync_raft_progress_tracker.h" -#include "syncInt.h" - -#define RAFT_READ_LOG_MAX_NUM 100 - -static int deserializeServerStateFromBuffer(SSyncServerState* server, const char* buffer, int n); -static int deserializeClusterStateFromBuffer(SSyncConfigState* cluster, const char* buffer, int n); - -static void switchToConfig(SSyncRaft* pRaft, const SSyncRaftProgressTrackerConfig* config, - const SSyncRaftProgressMap* progressMap, SSyncConfigState* cs); - -static void abortLeaderTransfer(SSyncRaft* pRaft); - -static bool preHandleMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg); -static bool preHandleNewTermMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg); -static bool preHandleOldTermMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg); - -int32_t syncRaftStart(SSyncRaft* pRaft, const SSyncInfo* pInfo) { - SSyncNode* pNode = pRaft->pNode; - SSyncServerState serverState; - SSyncConfigState confState; - SStateManager* stateManager; - SSyncLogStore* logStore; - SSyncFSM* fsm; - SSyncBuffer buffer[RAFT_READ_LOG_MAX_NUM]; - int nBuf, limit, i; - char* buf; - int n; - SSyncRaftChanger changer; - - memset(pRaft, 0, sizeof(SSyncRaft)); - - memcpy(&pRaft->fsm, &pInfo->fsm, sizeof(SSyncFSM)); - memcpy(&pRaft->logStore, &pInfo->logStore, sizeof(SSyncLogStore)); - memcpy(&pRaft->stateManager, &pInfo->stateManager, sizeof(SStateManager)); - - stateManager = &(pRaft->stateManager); - logStore = &(pRaft->logStore); - fsm = &(pRaft->fsm); - - pRaft->nodeInfoMap = taosHashInit(TSDB_MAX_REPLICA, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (pRaft->nodeInfoMap == NULL) { - return -1; - } - - // init progress tracker - pRaft->tracker = syncRaftOpenProgressTracker(pRaft); - if (pRaft->tracker == NULL) { - return -1; - } - - // open raft log - if ((pRaft->log = syncRaftLogOpen()) == NULL) { - return -1; - } - // read server state - if (stateManager->readServerState(stateManager, &buf, &n) != 0) { - syncError("readServerState for vgid %d fail", pInfo->vgId); - return -1; - } - if (deserializeServerStateFromBuffer(&serverState, buf, n) != 0) { - syncError("deserializeServerStateFromBuffer for vgid %d fail", pInfo->vgId); - return -1; - } - free(buf); - //assert(initIndex <= serverState.commitIndex); - - // read config state - if (stateManager->readClusterState(stateManager, &buf, &n) != 0) { - syncError("readClusterState for vgid %d fail", pInfo->vgId); - return -1; - } - if (deserializeClusterStateFromBuffer(&confState, buf, n) != 0) { - syncError("deserializeClusterStateFromBuffer for vgid %d fail", pInfo->vgId); - return -1; - } - free(buf); - - changer = (SSyncRaftChanger) { - .tracker = pRaft->tracker, - .lastIndex = syncRaftLogLastIndex(pRaft->log), - }; - SSyncRaftProgressTrackerConfig config; - SSyncRaftProgressMap progressMap; - - if (syncRaftRestoreConfig(&changer, &confState, &config, &progressMap) < 0) { - syncError("syncRaftRestoreConfig for vgid %d fail", pInfo->vgId); - return -1; - } - - // save restored config and progress map to tracker - syncRaftCopyProgressMap(&progressMap, &pRaft->tracker->progressMap); - syncRaftCopyTrackerConfig(&config, &pRaft->tracker->config); - - // free progress map and config - syncRaftFreeProgressMap(&progressMap); - syncRaftFreeTrackConfig(&config); - - if (!syncRaftIsEmptyServerState(&serverState)) { - syncRaftLoadState(pRaft, &serverState); - } - - if (pInfo->appliedIndex > 0) { - syncRaftLogAppliedTo(pRaft->log, pInfo->appliedIndex); - } - - syncRaftBecomeFollower(pRaft, pRaft->term, SYNC_NON_NODE_ID); - - syncInfo("[%d:%d] restore vgid %d state: snapshot index success", - pRaft->selfGroupId, pRaft->selfId, pInfo->vgId); - return 0; -} - -int32_t syncRaftStep(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - syncDebug("from %d, type:%d, term:%" PRId64 ", state:%d", - pMsg->from, pMsg->msgType, pMsg->term, pRaft->state); - - if (preHandleMessage(pRaft, pMsg)) { - syncFreeMessage(pMsg); - return 0; - } - - ESyncRaftMessageType msgType = pMsg->msgType; - if (msgType == RAFT_MSG_INTERNAL_ELECTION) { - syncRaftHandleElectionMessage(pRaft, pMsg); - } else if (msgType == RAFT_MSG_VOTE) { - syncRaftHandleVoteMessage(pRaft, pMsg); - } else { - pRaft->stepFp(pRaft, pMsg); - } - - syncFreeMessage(pMsg); - return 0; -} - -int32_t syncRaftTick(SSyncRaft* pRaft) { - pRaft->currentTick += 1; - pRaft->tickFp(pRaft); - return 0; -} - -static int deserializeServerStateFromBuffer(SSyncServerState* server, const char* buffer, int n) { - return 0; -} - -static int deserializeClusterStateFromBuffer(SSyncConfigState* cluster, const char* buffer, int n) { - return 0; -} - -static void visitProgressMaybeSendAppend(SSyncRaftProgress* progress, void* arg) { - syncRaftMaybeSendAppend(arg, progress, false); -} - -// switchToConfig reconfigures this node to use the provided configuration. It -// updates the in-memory state and, when necessary, carries out additional -// actions such as reacting to the removal of nodes or changed quorum -// requirements. -// -// The inputs usually result from restoring a ConfState or applying a ConfChange. -static void switchToConfig(SSyncRaft* pRaft, const SSyncRaftProgressTrackerConfig* config, - const SSyncRaftProgressMap* progressMap, SSyncConfigState* cs) { - SyncNodeId selfId = pRaft->selfId; - int i; - bool exist; - SSyncRaftProgress* progress = NULL; - - syncRaftConfigState(pRaft->tracker, cs); - progress = syncRaftFindProgressByNodeId(&pRaft->tracker->progressMap, selfId); - exist = (progress != NULL); - - // Update whether the node itself is a learner, resetting to false when the - // node is removed. - if (exist) { - pRaft->isLearner = progress->isLearner; - } else { - pRaft->isLearner = false; - } - - if ((!exist || pRaft->isLearner) && pRaft->state == TAOS_SYNC_STATE_LEADER) { - // This node is leader and was removed or demoted. We prevent demotions - // at the time writing but hypothetically we handle them the same way as - // removing the leader: stepping down into the next Term. - // - // TODO(tbg): step down (for sanity) and ask follower with largest Match - // to TimeoutNow (to avoid interruption). This might still drop some - // proposals but it's better than nothing. - // - // TODO(tbg): test this branch. It is untested at the time of writing. - return; - } - - // The remaining steps only make sense if this node is the leader and there - // are other nodes. - if (pRaft->state != TAOS_SYNC_STATE_LEADER || syncRaftNodeMapSize(&cs->voters) == 0) { - return; - } - - if (syncRaftMaybeCommit(pRaft)) { - // If the configuration change means that more entries are committed now, - // broadcast/append to everyone in the updated config. - syncRaftBroadcastAppend(pRaft); - } else { - // Otherwise, still probe the newly added replicas; there's no reason to - // let them wait out a heartbeat interval (or the next incoming - // proposal). - syncRaftProgressVisit(pRaft->tracker, visitProgressMaybeSendAppend, pRaft); - - // If the the leadTransferee was removed or demoted, abort the leadership transfer. - SyncNodeId leadTransferee = pRaft->leadTransferee; - if (leadTransferee != SYNC_NON_NODE_ID) { - if (!syncRaftIsInNodeMap(&pRaft->tracker->config.voters.incoming, leadTransferee) && - !syncRaftIsInNodeMap(&pRaft->tracker->config.voters.outgoing, leadTransferee)) { - abortLeaderTransfer(pRaft); - } - } - } -} - -static void abortLeaderTransfer(SSyncRaft* pRaft) { - pRaft->leadTransferee = SYNC_NON_NODE_ID; -} - -/** - * pre-handle message, return true means no need to continue - * Handle the message term, which may result in our stepping down to a follower. - **/ -static bool preHandleMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - // local message? - if (pMsg->term == 0) { - return false; - } - - if (pMsg->term > pRaft->term) { - return preHandleNewTermMessage(pRaft, pMsg); - } else if (pMsg->term < pRaft->term) { - return preHandleOldTermMessage(pRaft, pMsg); - } - - return false; -} - -static bool preHandleNewTermMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - SyncNodeId leaderId = pMsg->from; - ESyncRaftMessageType msgType = pMsg->msgType; - - if (msgType == RAFT_MSG_VOTE) { - // TODO - leaderId = SYNC_NON_NODE_ID; - } - - if (syncIsPreVoteMsg(pMsg)) { - // Never change our term in response to a PreVote - } else if (syncIsPreVoteRespMsg(pMsg) && !pMsg->voteResp.rejected) { - /** - * We send pre-vote requests with a term in our future. If the - * pre-vote is granted, we will increment our term when we get a - * quorum. If it is not, the term comes from the node that - * rejected our vote so we should become a follower at the new - * term. - **/ - } else { - syncInfo("[%d:%d] [term:%" PRId64 "] received a %d message with higher term from %d [term:%" PRId64 "]", - pRaft->selfGroupId, pRaft->selfId, pRaft->term, msgType, pMsg->from, pMsg->term); - syncRaftBecomeFollower(pRaft, pMsg->term, leaderId); - } - - return false; -} - -static bool preHandleOldTermMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - if (pRaft->checkQuorum && pMsg->msgType == RAFT_MSG_APPEND) { - /** - * We have received messages from a leader at a lower term. It is possible - * that these messages were simply delayed in the network, but this could - * also mean that this node has advanced its term number during a network - * partition, and it is now unable to either win an election or to rejoin - * the majority on the old term. If checkQuorum is false, this will be - * handled by incrementing term numbers in response to MsgVote with a - * higher term, but if checkQuorum is true we may not advance the term on - * MsgVote and must generate other messages to advance the term. The net - * result of these two features is to minimize the disruption caused by - * nodes that have been removed from the cluster's configuration: a - * removed node will send MsgVotes (or MsgPreVotes) which will be ignored, - * but it will not receive MsgApp or MsgHeartbeat, so it will not create - * disruptive term increases - **/ - SNodeInfo* pNode = syncRaftGetNodeById(pRaft, pMsg->from); - if (pNode == NULL) { - return true; - } - SSyncMessage* msg = syncNewEmptyAppendRespMsg(pRaft->selfGroupId, pRaft->selfId, pRaft->term); - if (msg == NULL) { - return true; - } - - pRaft->io.send(msg, pNode); - } else { - // ignore other cases - syncInfo("[%d:%d] [term:%" PRId64 "] ignored a %d message with lower term from %d [term:%" PRId64 "]", - pRaft->selfGroupId, pRaft->selfId, pRaft->term, pMsg->msgType, pMsg->from, pMsg->term); - } - - return true; -} \ No newline at end of file diff --git a/source/libs/sync/src/raft_handle_append_entries_message.c b/source/libs/sync/src/raft_handle_append_entries_message.c deleted file mode 100644 index 92ebfe75f5..0000000000 --- a/source/libs/sync/src/raft_handle_append_entries_message.c +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncInt.h" -#include "raft.h" -#include "raft_log.h" -#include "sync_raft_impl.h" -#include "raft_message.h" - -int syncRaftHandleAppendEntriesMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - const RaftMsg_Append_Entries *appendEntries = &(pMsg->appendEntries); - - SNodeInfo* pNode = syncRaftGetNodeById(pRaft, pMsg->from); - if (pNode == NULL) { - return 0; - } - - SSyncMessage* pRespMsg = syncNewEmptyAppendRespMsg(pRaft->selfGroupId, pRaft->selfId, pRaft->term); - if (pRespMsg == NULL) { - return 0; - } - - RaftMsg_Append_Resp *appendResp = &(pRespMsg->appendResp); - // ignore committed logs - if (syncRaftLogIsCommitted(pRaft->log, appendEntries->index)) { - appendResp->index = pRaft->log->commitIndex; - goto out; - } - - syncInfo("[%d:%d] recv append from %d index %" PRId64"", - pRaft->selfGroupId, pRaft->selfId, pMsg->from, appendEntries->index); - -out: - pRaft->io.send(pRespMsg, pNode); - return 0; -} \ No newline at end of file diff --git a/source/libs/sync/src/raft_handle_election_message.c b/source/libs/sync/src/raft_handle_election_message.c deleted file mode 100644 index a58c8ba5cf..0000000000 --- a/source/libs/sync/src/raft_handle_election_message.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncInt.h" -#include "raft.h" -#include "raft_log.h" -#include "raft_message.h" - -int syncRaftHandleElectionMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - if (pRaft->preVote) { - syncRaftStartElection(pRaft, SYNC_RAFT_CAMPAIGN_PRE_ELECTION); - } else { - syncRaftStartElection(pRaft, SYNC_RAFT_CAMPAIGN_ELECTION); - } - - return 0; -} diff --git a/source/libs/sync/src/raft_handle_vote_message.c b/source/libs/sync/src/raft_handle_vote_message.c deleted file mode 100644 index 4d940732dc..0000000000 --- a/source/libs/sync/src/raft_handle_vote_message.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncInt.h" -#include "raft.h" -#include "sync_raft_impl.h" -#include "raft_log.h" -#include "raft_message.h" - -static bool canGrantVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg); - -int syncRaftHandleVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - SSyncMessage* pRespMsg; - SNodeInfo* pNode = syncRaftGetNodeById(pRaft, pMsg->from); - if (pNode == NULL) { - return 0; - } - - bool grant; - SyncIndex lastIndex = syncRaftLogLastIndex(pRaft->log); - SyncTerm lastTerm = syncRaftLogLastTerm(pRaft->log); - - grant = canGrantVoteMessage(pRaft, pMsg); - pRespMsg = syncNewVoteRespMsg(pRaft->selfGroupId, pRaft->selfId, pMsg->vote.cType, !grant); - if (pRespMsg == NULL) { - return 0; - } - syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 ", vote: %d] %s for %d" - "[logterm: %" PRId64 ", index: %" PRId64 "] at term %" PRId64 "", - pRaft->selfGroupId, pRaft->selfId, lastTerm, lastIndex, pRaft->voteFor, - grant ? "grant" : "reject", - pMsg->from, pMsg->vote.lastTerm, pMsg->vote.lastIndex, pRaft->term); - - pRaft->io.send(pRespMsg, pNode); - return 0; -} - -static bool canGrantVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - bool canVote = - // We can vote if this is a repeat of a vote we've already cast... - pRaft->voteFor == pMsg->from || - // ...we haven't voted and we don't think there's a leader yet in this term... - (pRaft->voteFor == SYNC_NON_NODE_ID && pRaft->leaderId == SYNC_NON_NODE_ID) || - // ...or this is a PreVote for a future term... - (pMsg->vote.cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION && pMsg->term > pRaft->term); - - // ...and we believe the candidate is up to date. - return canVote && syncRaftLogIsUptodate(pRaft->log, pMsg->vote.lastIndex, pMsg->vote.lastTerm); -} \ No newline at end of file diff --git a/source/libs/sync/src/raft_handle_vote_resp_message.c b/source/libs/sync/src/raft_handle_vote_resp_message.c deleted file mode 100644 index 87a5cfcd15..0000000000 --- a/source/libs/sync/src/raft_handle_vote_resp_message.c +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncInt.h" -#include "raft.h" -#include "sync_raft_impl.h" -#include "raft_message.h" - -int syncRaftHandleVoteRespMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - int granted, rejected; - int quorum; - int voterIndex; - - assert(pRaft->state == TAOS_SYNC_STATE_CANDIDATE); - - SNodeInfo* pNode = syncRaftGetNodeById(pRaft, pMsg->from); - if (pNode == NULL) { - syncError("[%d:%d] recv vote resp from unknown server %d", pRaft->selfGroupId, pRaft->selfId, pMsg->from); - return 0; - } - - if (pRaft->state != TAOS_SYNC_STATE_CANDIDATE) { - syncError("[%d:%d] is not candidate, ignore vote resp", pRaft->selfGroupId, pRaft->selfId); - return 0; - } - - ESyncRaftVoteResult result = syncRaftPollVote(pRaft, pMsg->from, - pMsg->voteResp.cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION, - !pMsg->voteResp.rejected, &rejected, &granted); - - syncInfo("[%d:%d] [quorum:%d] has received %d votes and %d vote rejections", - pRaft->selfGroupId, pRaft->selfId, quorum, granted, rejected); - - if (result == SYNC_RAFT_VOTE_WON) { - if (pRaft->candidateState.inPreVote) { - syncRaftCampaign(pRaft, SYNC_RAFT_CAMPAIGN_ELECTION); - } else { - syncRaftBecomeLeader(pRaft); - syncRaftBroadcastAppend(pRaft); - } - } else if (result == SYNC_RAFT_VOTE_LOST) { - // pb.MsgPreVoteResp contains future term of pre-candidate - // m.Term > r.Term; reuse r.Term - syncRaftBecomeFollower(pRaft, pRaft->term, SYNC_NON_NODE_ID); - } - - return 0; -} \ No newline at end of file diff --git a/source/libs/sync/src/raft_log.c b/source/libs/sync/src/raft_log.c deleted file mode 100644 index b6e6d292e8..0000000000 --- a/source/libs/sync/src/raft_log.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "raft_log.h" - -SSyncRaftLog* syncRaftLogOpen() { - return NULL; -} - -SyncIndex syncRaftLogLastIndex(SSyncRaftLog* pLog) { - return 0; -} - -SyncIndex syncRaftLogSnapshotIndex(SSyncRaftLog* pLog) { - return 0; -} - -SyncTerm syncRaftLogLastTerm(SSyncRaftLog* pLog) { - return 0; -} - -void syncRaftLogAppliedTo(SSyncRaftLog* pLog, SyncIndex appliedIndex) { - -} - -bool syncRaftLogIsUptodate(SSyncRaftLog* pLog, SyncIndex index, SyncTerm term) { - return true; -} - -int syncRaftLogNumOfPendingConf(SSyncRaftLog* pLog) { - return 0; -} - -bool syncRaftHasUnappliedLog(SSyncRaftLog* pLog) { - return pLog->commitIndex > pLog->appliedIndex; -} - -SyncTerm syncRaftLogTermOf(SSyncRaftLog* pLog, SyncIndex index) { - return SYNC_NON_TERM; -} - -int syncRaftLogAppend(SSyncRaftLog* pLog, SSyncRaftEntry *pEntries, int n) { - -} - -int syncRaftLogAcquire(SSyncRaftLog* pLog, SyncIndex index, int maxMsgSize, - SSyncRaftEntry **ppEntries, int *n) { - return 0; -} - -void syncRaftLogRelease(SSyncRaftLog* pLog, SyncIndex index, - SSyncRaftEntry *pEntries, int n) { - return; -} \ No newline at end of file diff --git a/source/libs/sync/src/raft_message.c b/source/libs/sync/src/raft_message.c deleted file mode 100644 index e706127f29..0000000000 --- a/source/libs/sync/src/raft_message.c +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "raft_message.h" - -void syncFreeMessage(const SSyncMessage* pMsg) { - if (!syncIsInternalMsg(pMsg->msgType)) { - free((SSyncMessage*)pMsg); - } -} \ No newline at end of file diff --git a/source/libs/sync/src/raft_replication.c b/source/libs/sync/src/raft_replication.c deleted file mode 100644 index c8c2d2c379..0000000000 --- a/source/libs/sync/src/raft_replication.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "raft.h" -#include "raft_log.h" -#include "sync_raft_progress.h" -#include "syncInt.h" -#include "raft_replication.h" - -static bool sendSnapshot(SSyncRaft* pRaft, SSyncRaftProgress* progress); -static bool sendAppendEntries(SSyncRaft* pRaft, SSyncRaftProgress* progress, - SyncIndex prevIndex, SyncTerm prevTerm, - SSyncRaftEntry *entries, int nEntry); - -// maybeSendAppend sends an append RPC with new entries to the given peer, -// if necessary. Returns true if a message was sent. The sendIfEmpty -// argument controls whether messages with no entries will be sent -// ("empty" messages are useful to convey updated Commit indexes, but -// are undesirable when we're sending multiple messages in a batch). -bool syncRaftMaybeSendAppend(SSyncRaft* pRaft, SSyncRaftProgress* progress, bool sendIfEmpty) { - assert(pRaft->state == TAOS_SYNC_STATE_LEADER); - SyncNodeId nodeId = progress->id; - - if (syncRaftProgressIsPaused(progress)) { - syncInfo("node [%d:%d] paused", pRaft->selfGroupId, nodeId); - return false; - } - - SyncIndex nextIndex = syncRaftProgressNextIndex(progress); - SSyncRaftEntry *entries; - int nEntry; - SyncIndex prevIndex; - SyncTerm prevTerm; - - prevIndex = nextIndex - 1; - prevTerm = syncRaftLogTermOf(pRaft->log, prevIndex); - int ret = syncRaftLogAcquire(pRaft->log, nextIndex, pRaft->maxMsgSize, &entries, &nEntry); - - if (nEntry == 0 && !sendIfEmpty) { - return false; - } - - if (ret != 0 || prevTerm == SYNC_NON_TERM) { - return sendSnapshot(pRaft, progress); - } - - return sendAppendEntries(pRaft, progress, prevIndex, prevTerm, entries, nEntry); -} - -static bool sendSnapshot(SSyncRaft* pRaft, SSyncRaftProgress* progress) { - if (!syncRaftProgressRecentActive(progress)) { - return false; - } - return true; -} - -static bool sendAppendEntries(SSyncRaft* pRaft, SSyncRaftProgress* progress, - SyncIndex prevIndex, SyncTerm prevTerm, - SSyncRaftEntry *entries, int nEntry) { - SNodeInfo* pNode = syncRaftGetNodeById(pRaft, progress->id); - if (pNode == NULL) { - return false; - } - SyncIndex lastIndex; - SyncTerm logTerm = prevTerm; - - SSyncMessage* msg = syncNewAppendMsg(pRaft->selfGroupId, pRaft->selfId, pRaft->term, - prevIndex, prevTerm, pRaft->log->commitIndex, - nEntry, entries); - - if (msg == NULL) { - goto err_release_log; - } - - if (nEntry != 0) { - switch (progress->state) { - // optimistically increase the next when in StateReplicate - case PROGRESS_STATE_REPLICATE: - lastIndex = entries[nEntry - 1].index; - syncRaftProgressOptimisticNextIndex(progress, lastIndex); - syncRaftInflightAdd(progress->inflights, lastIndex); - break; - case PROGRESS_STATE_PROBE: - progress->probeSent = true; - break; - default: - syncFatal("[%d:%d] is sending append in unhandled state %s", - pRaft->selfGroupId, pRaft->selfId, syncRaftProgressStateString(progress)); - break; - } - } - pRaft->io.send(msg, pNode); - return true; - -err_release_log: - syncRaftLogRelease(pRaft->log, prevIndex + 1, entries, nEntry); - return false; -} diff --git a/source/libs/sync/src/raft_unstable_log.c b/source/libs/sync/src/raft_unstable_log.c deleted file mode 100644 index e798e20662..0000000000 --- a/source/libs/sync/src/raft_unstable_log.c +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sync.h" -#include "raft_unstable_log.h" - -/* -SyncIndex syncRaftLogLastIndex(SSyncRaftUnstableLog* pLog) { - return 0; -} -*/ \ No newline at end of file diff --git a/source/libs/sync/src/sync.c b/source/libs/sync/src/sync.c index 321b03d2ee..7ded53b6e6 100644 --- a/source/libs/sync/src/sync.c +++ b/source/libs/sync/src/sync.c @@ -1,302 +1 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncInt.h" -#include "trpc.h" -#include "ttimer.h" - -SSyncManager* gSyncManager = NULL; - -#define SYNC_TICK_TIMER 50 -#define SYNC_ACTIVITY_TIMER 5 -#define SYNC_SERVER_WORKER 2 - -static void syncProcessRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); -static void syncProcessReqMsg(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); - -static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyncCfg); -static int syncInitRpcClient(SSyncManager* syncManager); -static int syncOpenWorkerPool(SSyncManager* syncManager); -static int syncCloseWorkerPool(SSyncManager* syncManager); -static void *syncWorkerMain(void *argv); -static void syncNodeTick(void *param, void *tmrId); - -int32_t syncInit() { - if (gSyncManager != NULL) { - return 0; - } - - gSyncManager = (SSyncManager*)calloc(sizeof(SSyncManager), 0); - if (gSyncManager == NULL) { - syncError("malloc SSyncManager fail"); - return -1; - } - - pthread_mutex_init(&gSyncManager->mutex, NULL); - - // init client rpc - if (syncInitRpcClient(gSyncManager) != 0) { - syncCleanUp(); - return -1; - } - - // init sync timer manager - gSyncManager->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); - if (gSyncManager->syncTimerManager == NULL) { - syncCleanUp(); - return -1; - } - - // init worker pool - if (syncOpenWorkerPool(gSyncManager) != 0) { - syncCleanUp(); - return -1; - } - - // init vgroup hash table - gSyncManager->vgroupTable = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (gSyncManager->vgroupTable == NULL) { - syncCleanUp(); - return -1; - } - return 0; -} - -void syncCleanUp() { - if (gSyncManager == NULL) { - return; - } - pthread_mutex_lock(&gSyncManager->mutex); - if (gSyncManager->vgroupTable) { - taosHashCleanup(gSyncManager->vgroupTable); - } - if (gSyncManager->clientRpc) { - rpcClose(gSyncManager->clientRpc); - syncInfo("sync inter-sync rpc client is closed"); - } - if (gSyncManager->syncTimerManager) { - taosTmrCleanUp(gSyncManager->syncTimerManager); - } - syncCloseWorkerPool(gSyncManager); - pthread_mutex_unlock(&gSyncManager->mutex); - pthread_mutex_destroy(&gSyncManager->mutex); - free(gSyncManager); - gSyncManager = NULL; -} - -SSyncNode* syncStart(const SSyncInfo* pInfo) { - pthread_mutex_lock(&gSyncManager->mutex); - - SSyncNode **ppNode = taosHashGet(gSyncManager->vgroupTable, &pInfo->vgId, sizeof(SyncGroupId*)); - if (ppNode != NULL) { - syncInfo("vgroup %d already exist", pInfo->vgId); - pthread_mutex_unlock(&gSyncManager->mutex); - return *ppNode; - } - - // init rpc server - if (syncInitRpcServer(gSyncManager, &pInfo->syncCfg) != 0) { - pthread_mutex_unlock(&gSyncManager->mutex); - return NULL; - } - - SSyncNode *pNode = (SSyncNode*)malloc(sizeof(SSyncNode)); - if (pNode == NULL) { - syncError("malloc vgroup %d node fail", pInfo->vgId); - pthread_mutex_unlock(&gSyncManager->mutex); - return NULL; - } - - pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)((int64_t)pInfo->vgId), gSyncManager->syncTimerManager); - - // start raft - pNode->raft.pNode = pNode; - if (syncRaftStart(&pNode->raft, pInfo) != 0) { - syncError("raft start at %d node fail", pInfo->vgId); - pthread_mutex_unlock(&gSyncManager->mutex); - return NULL; - } - - pthread_mutex_init(&pNode->mutex, NULL); - - taosHashPut(gSyncManager->vgroupTable, &pInfo->vgId, sizeof(SyncGroupId), &pNode, sizeof(SSyncNode *)); - - pthread_mutex_unlock(&gSyncManager->mutex); - return NULL; -} - -void syncStop(const SSyncNode* pNode) { - pthread_mutex_lock(&gSyncManager->mutex); - - SSyncNode **ppNode = taosHashGet(gSyncManager->vgroupTable, &pNode->vgId, sizeof(SyncGroupId*)); - if (ppNode == NULL) { - syncInfo("vgroup %d not exist", pNode->vgId); - pthread_mutex_unlock(&gSyncManager->mutex); - return; - } - assert(*ppNode == pNode); - taosTmrStop(pNode->syncTimer); - - taosHashRemove(gSyncManager->vgroupTable, &pNode->vgId, sizeof(SyncGroupId)); - pthread_mutex_unlock(&gSyncManager->mutex); - - pthread_mutex_destroy(&((*ppNode)->mutex)); - free(*ppNode); -} - -int32_t syncPropose(SSyncNode* syncNode, const SSyncBuffer* pBuf, void* pData, bool isWeak) { - SSyncMessage msg; - - pthread_mutex_lock(&syncNode->mutex); - int32_t ret = syncRaftStep(&syncNode->raft, syncInitPropMsg(&msg, pBuf, pData, isWeak)); - pthread_mutex_unlock(&syncNode->mutex); - return ret; -} - -void syncReconfig(const SSyncNode* pNode, const SSyncCluster* pCfg) {} - -int32_t syncAddNode(SSyncNode syncNode, const SNodeInfo *pNode) { - return 0; -} - -int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode) { - return 0; -} - -// process rpc rsp message from other sync server -static void syncProcessRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { - -} - -// process rpc message from other sync server -static void syncProcessReqMsg(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { - -} - -static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyncCfg) { - if (gSyncManager->rpcServerTable == NULL) { - gSyncManager->rpcServerTable = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (gSyncManager->rpcServerTable == NULL) { - syncError("init sync rpc server hash table error"); - return -1; - } - } - assert(pSyncCfg->selfIndex < pSyncCfg->replica && pSyncCfg->selfIndex >= 0); - const SNodeInfo* pNode = &(pSyncCfg->nodeInfo[pSyncCfg->replica]); - char buffer[156] = {'\0'}; - snprintf(buffer, sizeof(buffer), "%s:%d", &(pNode->nodeFqdn[0]), pNode->nodePort); - size_t len = strlen(buffer); - void** ppRpcServer = taosHashGet(gSyncManager->rpcServerTable, buffer, len); - if (ppRpcServer != NULL) { - // already inited - syncInfo("sync rpc server for %s already exist", buffer); - return 0; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = pNode->nodePort; - rpcInit.label = "sync-server"; - rpcInit.numOfThreads = SYNC_SERVER_WORKER; - rpcInit.cfp = syncProcessReqMsg; - rpcInit.sessions = TSDB_MAX_VNODES << 4; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = SYNC_ACTIVITY_TIMER * 1000; - - void* rpcServer = rpcOpen(&rpcInit); - if (rpcServer == NULL) { - syncInfo("rpcOpen for sync rpc server for %s fail", buffer); - return -1; - } - - taosHashPut(gSyncManager->rpcServerTable, buffer, strlen(buffer), rpcServer, len); - syncInfo("sync rpc server for %s init success", buffer); - - return 0; -} - -static int syncInitRpcClient(SSyncManager* syncManager) { - char secret[TSDB_PASSWORD_LEN] = "secret"; - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "sync-client"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = syncProcessRsp; - rpcInit.sessions = TSDB_MAX_VNODES << 4; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = SYNC_ACTIVITY_TIMER * 1000; - rpcInit.user = "t"; - rpcInit.ckey = "key"; - rpcInit.secret = secret; - - syncManager->clientRpc = rpcOpen(&rpcInit); - if (syncManager->clientRpc == NULL) { - syncError("failed to init sync rpc client"); - return -1; - } - - syncInfo("sync inter-sync rpc client is initialized"); - return 0; -} - -static int syncOpenWorkerPool(SSyncManager* syncManager) { - int i; - pthread_attr_t thattr; - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - - for (i = 0; i < TAOS_SYNC_MAX_WORKER; ++i) { - SSyncWorker* pWorker = &(syncManager->worker[i]); - - if (pthread_create(&(pWorker->thread), &thattr, (void *)syncWorkerMain, pWorker) != 0) { - syncError("failed to create sync worker since %s", strerror(errno)); - - return -1; - } - } - - pthread_attr_destroy(&thattr); - - return 0; -} - -static int syncCloseWorkerPool(SSyncManager* syncManager) { - return 0; -} - -static void *syncWorkerMain(void *argv) { - SSyncWorker* pWorker = (SSyncWorker *)argv; - - taosBlockSIGPIPE(); - setThreadName("syncWorker"); - - return NULL; -} - -static void syncNodeTick(void *param, void *tmrId) { - SyncGroupId vgId = (SyncGroupId)((int64_t)param); - SSyncNode **ppNode = taosHashGet(gSyncManager->vgroupTable, &vgId, sizeof(SyncGroupId*)); - if (ppNode == NULL) { - return; - } - SSyncNode *pNode = *ppNode; - - pthread_mutex_lock(&pNode->mutex); - syncRaftTick(&pNode->raft); - pthread_mutex_unlock(&pNode->mutex); - - pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)(int64_t)pNode->vgId, gSyncManager->syncTimerManager); -} \ No newline at end of file +#include "sync.h" \ No newline at end of file diff --git a/source/libs/sync/src/sync_raft_config_change.c b/source/libs/sync/src/sync_raft_config_change.c deleted file mode 100644 index de790b5876..0000000000 --- a/source/libs/sync/src/sync_raft_config_change.c +++ /dev/null @@ -1,409 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "raft.h" -#include "syncInt.h" -#include "sync_raft_config_change.h" -#include "sync_raft_progress.h" -#include "sync_raft_progress_tracker.h" -#include "sync_raft_quorum_joint.h" - -static int checkAndCopy(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); -static int checkAndReturn(SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); -static int checkInvariants(SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); -static int checkInvariants(SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); -static bool hasJointConfig(const SSyncRaftProgressTrackerConfig* config); -static int applyConfig(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, const SSyncConfChangeSingleArray* css); - -static int symDiff(const SSyncRaftNodeMap* l, const SSyncRaftNodeMap* r); - -static void initProgress(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, SyncNodeId id, bool isLearner); - -static void nilAwareDelete(SSyncRaftNodeMap* nodeMap, SyncNodeId id); -static void nilAwareAdd(SSyncRaftNodeMap* nodeMap, SyncNodeId id); - -static void makeVoter(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, SyncNodeId id); -static void makeLearner(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, SyncNodeId id); -static void removeNodeId(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, SyncNodeId id); - -// EnterJoint verifies that the outgoing (=right) majority config of the joint -// config is empty and initializes it with a copy of the incoming (=left) -// majority config. That is, it transitions from -// -// (1 2 3)&&() -// to -// (1 2 3)&&(1 2 3). -// -// The supplied changes are then applied to the incoming majority config, -// resulting in a joint configuration that in terms of the Raft thesis[1] -// (Section 4.3) corresponds to `C_{new,old}`. -// -// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf -int syncRaftChangerEnterJoint(SSyncRaftChanger* changer, bool autoLeave, const SSyncConfChangeSingleArray* css, - SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) { - int ret; - - ret = checkAndCopy(changer, config, progressMap); - if (ret != 0) { - return ret; - } - - if (hasJointConfig(config)) { - syncError("config is already joint"); - return -1; - } - - if(syncRaftJointConfigIsIncomingEmpty(&config->voters) == 0) { - // We allow adding nodes to an empty config for convenience (testing and - // bootstrap), but you can't enter a joint state. - syncError("can't make a zero-voter config joint"); - return -1; - } - - // Clear the outgoing config. - syncRaftJointConfigClearOutgoing(&config->voters); - - // Copy incoming to outgoing. - syncRaftCopyNodeMap(&config->voters.incoming, &config->voters.outgoing); - - ret = applyConfig(changer, config, progressMap, css); - if (ret != 0) { - return ret; - } - - config->autoLeave = autoLeave; - return checkAndReturn(config, progressMap); -} - -// Simple carries out a series of configuration changes that (in aggregate) -// mutates the incoming majority config Voters[0] by at most one. This method -// will return an error if that is not the case, if the resulting quorum is -// zero, or if the configuration is in a joint state (i.e. if there is an -// outgoing configuration). -int syncRaftChangerSimpleConfig(SSyncRaftChanger* changer, const SSyncConfChangeSingleArray* css, - SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) { - int ret; - - ret = checkAndCopy(changer, config, progressMap); - if (ret != 0) { - return ret; - } - - if (hasJointConfig(config)) { - syncError("can't apply simple config change in joint config"); - return -1; - } - - ret = applyConfig(changer, config, progressMap, css); - if (ret != 0) { - return ret; - } - - int n = symDiff(syncRaftJointConfigIncoming(&changer->tracker->config.voters), - syncRaftJointConfigIncoming(&config->voters)); - if (n > 1) { - syncError("more than one voter changed without entering joint config"); - return -1; - } - - return checkAndReturn(config, progressMap); -} - -// apply a change to the configuration. By convention, changes to voters are -// always made to the incoming majority config Voters[0]. Voters[1] is either -// empty or preserves the outgoing majority configuration while in a joint state. -static int applyConfig(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, const SSyncConfChangeSingleArray* css) { - int i; - - for (i = 0; i < css->n; ++i) { - const SSyncConfChangeSingle* cs = &(css->changes[i]); - if (cs->nodeId == SYNC_NON_NODE_ID) { - continue; - } - - ESyncRaftConfChangeType type = cs->type; - switch (type) { - case SYNC_RAFT_Conf_AddNode: - makeVoter(changer, config, progressMap, cs->nodeId); - break; - case SYNC_RAFT_Conf_AddLearnerNode: - makeLearner(changer, config, progressMap, cs->nodeId); - break; - case SYNC_RAFT_Conf_RemoveNode: - removeNodeId(changer, config, progressMap, cs->nodeId); - break; - case SYNC_RAFT_Conf_UpdateNode: - break; - } - } - - if (syncRaftJointConfigIsIncomingEmpty(&config->voters)) { - syncError("removed all voters"); - return -1; - } - - return 0; -} - - -// makeVoter adds or promotes the given ID to be a voter in the incoming -// majority config. -static void makeVoter(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, SyncNodeId id) { - SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, id); - if (progress == NULL) { - initProgress(changer, config, progressMap, id, false); - return; - } - - progress->isLearner = false; - nilAwareDelete(&config->learners, id); - nilAwareDelete(&config->learnersNext, id); - syncRaftJointConfigAddToIncoming(&config->voters, id); -} - -// makeLearner makes the given ID a learner or stages it to be a learner once -// an active joint configuration is exited. -// -// The former happens when the peer is not a part of the outgoing config, in -// which case we either add a new learner or demote a voter in the incoming -// config. -// -// The latter case occurs when the configuration is joint and the peer is a -// voter in the outgoing config. In that case, we do not want to add the peer -// as a learner because then we'd have to track a peer as a voter and learner -// simultaneously. Instead, we add the learner to LearnersNext, so that it will -// be added to Learners the moment the outgoing config is removed by -// LeaveJoint(). -static void makeLearner(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, SyncNodeId id) { - SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, id); - if (progress == NULL) { - initProgress(changer, config, progressMap, id, true); - return; - } - - if (progress->isLearner) { - return; - } - // Remove any existing voter in the incoming config... - removeNodeId(changer, config, progressMap, id); - - // ... but save the Progress. - syncRaftAddToProgressMap(progressMap, progress); - - // Use LearnersNext if we can't add the learner to Learners directly, i.e. - // if the peer is still tracked as a voter in the outgoing config. It will - // be turned into a learner in LeaveJoint(). - // - // Otherwise, add a regular learner right away. - bool inInOutgoing = syncRaftJointConfigIsInOutgoing(&config->voters, id); - if (inInOutgoing) { - nilAwareAdd(&config->learnersNext, id); - } else { - nilAwareAdd(&config->learners, id); - progress->isLearner = true; - } -} - -// removeNodeId this peer as a voter or learner from the incoming config. -static void removeNodeId(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, SyncNodeId id) { - SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, id); - if (progress == NULL) { - return; - } - - syncRaftJointConfigRemoveFromIncoming(&config->voters, id); - nilAwareDelete(&config->learners, id); - nilAwareDelete(&config->learnersNext, id); - - // If the peer is still a voter in the outgoing config, keep the Progress. - bool inInOutgoing = syncRaftJointConfigIsInOutgoing(&config->voters, id); - if (!inInOutgoing) { - syncRaftRemoveFromProgressMap(progressMap, id); - } -} - -// initProgress initializes a new progress for the given node or learner. -static void initProgress(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, - SSyncRaftProgressMap* progressMap, SyncNodeId id, bool isLearner) { - if (!isLearner) { - syncRaftJointConfigAddToIncoming(&config->voters, id); - } else { - nilAwareAdd(&config->learners, id); - } - - SSyncRaftProgress* pProgress = (SSyncRaftProgress*)malloc(sizeof(SSyncRaftProgress)); - assert (pProgress != NULL); - *pProgress = (SSyncRaftProgress) { - // Initializing the Progress with the last index means that the follower - // can be probed (with the last index). - // - // TODO(tbg): seems awfully optimistic. Using the first index would be - // better. The general expectation here is that the follower has no log - // at all (and will thus likely need a snapshot), though the app may - // have applied a snapshot out of band before adding the replica (thus - // making the first index the better choice). - .id = id, - .groupId = changer->tracker->pRaft->selfGroupId, - .nextIndex = changer->lastIndex, - .matchIndex = 0, - .state = PROGRESS_STATE_PROBE, - .pendingSnapshotIndex = 0, - .probeSent = false, - .inflights = syncRaftOpenInflights(changer->tracker->maxInflightMsgs), - .isLearner = isLearner, - // When a node is first added, we should mark it as recently active. - // Otherwise, CheckQuorum may cause us to step down if it is invoked - // before the added node has had a chance to communicate with us. - .recentActive = true, - .refCount = 0, - }; - - syncRaftAddToProgressMap(progressMap, pProgress); -} - -// checkInvariants makes sure that the config and progress are compatible with -// each other. This is used to check both what the Changer is initialized with, -// as well as what it returns. -static int checkInvariants(SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) { - int ret = syncRaftCheckTrackerConfigInProgress(config, progressMap); - if (ret != 0) { - return ret; - } - - // Any staged learner was staged because it could not be directly added due - // to a conflicting voter in the outgoing config. - SyncNodeId* pNodeId = NULL; - while (!syncRaftIterateNodeMap(&config->learnersNext, pNodeId)) { - SyncNodeId nodeId = *pNodeId; - if (!syncRaftJointConfigInOutgoing(&config->voters, nodeId)) { - syncError("[%d] is in LearnersNext, but not outgoing", nodeId); - return -1; - } - SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, nodeId); - assert(progress); - assert(progress->id == nodeId); - if (progress->isLearner) { - syncError("[%d:%d] is in LearnersNext, but is already marked as learner", progress->groupId, nodeId); - return -1; - } - } - - // Conversely Learners and Voters doesn't intersect at all. - pNodeId = NULL; - while (!syncRaftIterateNodeMap(&config->learners, pNodeId)) { - SyncNodeId nodeId = *pNodeId; - if (syncRaftJointConfigInOutgoing(&config->voters, nodeId)) { - syncError("%d is in Learners and outgoing", nodeId); - return -1; - } - SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, nodeId); - assert(progress); - assert(progress->id == nodeId); - - if (!progress->isLearner) { - syncError("[%d:%d] is in Learners, but is not marked as learner", progress->groupId, nodeId); - return -1; - } - } - - if (!hasJointConfig(config)) { - // We enforce that empty maps are nil instead of zero. - if (syncRaftNodeMapSize(&config->learnersNext) > 0) { - syncError("cfg.LearnersNext must be nil when not joint"); - return -1; - } - if (config->autoLeave) { - syncError("AutoLeave must be false when not joint"); - return -1; - } - } - - return 0; -} - -// checkAndCopy copies the tracker's config and progress map (deeply enough for -// the purposes of the Changer) and returns those copies. It returns an error -// if checkInvariants does. -static int checkAndCopy(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) { - syncRaftCopyTrackerConfig(&changer->tracker->config, config); - syncRaftClearProgressMap(progressMap); - - SSyncRaftProgress* pProgress = NULL; - while (!syncRaftIterateProgressMap(&changer->tracker->progressMap, pProgress)) { - syncRaftAddToProgressMap(progressMap, pProgress); - } - - return checkAndReturn(config, progressMap); -} - -// checkAndReturn calls checkInvariants on the input and returns either the -// resulting error or the input. -static int checkAndReturn(SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) { - if (checkInvariants(config, progressMap) != 0) { - return -1; - } - - return 0; -} - -static bool hasJointConfig(const SSyncRaftProgressTrackerConfig* config) { - return !syncRaftJointConfigIsOutgoingEmpty(&config->voters); -} - -// symdiff returns the count of the symmetric difference between the sets of -// uint64s, i.e. len( (l - r) \union (r - l)). -static int symDiff(const SSyncRaftNodeMap* l, const SSyncRaftNodeMap* r) { - int n; - int i; - int j0, j1; - const SSyncRaftNodeMap* pairs[2][2] = { - {l, r}, // count elems in l but not in r - {r, l}, // count elems in r but not in l - }; - - for (n = 0, i = 0; i < 2; ++i) { - const SSyncRaftNodeMap** pp = pairs[i]; - - const SSyncRaftNodeMap* p0 = pp[0]; - const SSyncRaftNodeMap* p1 = pp[1]; - SyncNodeId* pNodeId; - while (!syncRaftIterateNodeMap(p0, pNodeId)) { - if (!syncRaftIsInNodeMap(p1, *pNodeId)) { - n+=1; - } - } - } - - return n; -} - -// nilAwareDelete deletes from a map, nil'ing the map itself if it is empty after. -static void nilAwareDelete(SSyncRaftNodeMap* nodeMap, SyncNodeId id) { - syncRaftRemoveFromNodeMap(nodeMap, id); -} - -// nilAwareAdd populates a map entry, creating the map if necessary. -static void nilAwareAdd(SSyncRaftNodeMap* nodeMap, SyncNodeId id) { - syncRaftAddToNodeMap(nodeMap, id); -} \ No newline at end of file diff --git a/source/libs/sync/src/sync_raft_election.c b/source/libs/sync/src/sync_raft_election.c deleted file mode 100644 index fe2e0fd9d3..0000000000 --- a/source/libs/sync/src/sync_raft_election.c +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncInt.h" -#include "raft.h" -#include "raft_log.h" -#include "raft_message.h" -#include "sync_raft_progress_tracker.h" - -void syncRaftStartElection(SSyncRaft* pRaft, ESyncRaftElectionType cType) { - if (pRaft->state == TAOS_SYNC_STATE_LEADER) { - syncDebug("[%d:%d] ignoring RAFT_MSG_INTERNAL_ELECTION because already leader", pRaft->selfGroupId, pRaft->selfId); - return; - } - - if (!syncRaftIsPromotable(pRaft)) { - syncWarn("[%d:%d] is unpromotable and can not syncRaftCampaign", pRaft->selfGroupId, pRaft->selfId); - return; - } - - // if there is pending uncommitted config,cannot start election - if (syncRaftLogNumOfPendingConf(pRaft->log) > 0 && syncRaftHasUnappliedLog(pRaft->log)) { - syncWarn("[%d:%d] cannot syncRaftStartElection at term %" PRId64 " since there are still pending configuration changes to apply", - pRaft->selfGroupId, pRaft->selfId, pRaft->term); - return; - } - - syncInfo("[%d:%d] is starting a new election at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term); - - syncRaftCampaign(pRaft, cType); -} - -// syncRaftCampaign transitions the raft instance to candidate state. This must only be -// called after verifying that this is a legitimate transition. -void syncRaftCampaign(SSyncRaft* pRaft, ESyncRaftElectionType cType) { - bool preVote; - SyncTerm term; - - if (syncRaftIsPromotable(pRaft)) { - syncDebug("[%d:%d] is unpromotable; syncRaftCampaign() should have been called", pRaft->selfGroupId, pRaft->selfId); - return; - } - - if (cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION) { - syncRaftBecomePreCandidate(pRaft); - preVote = true; - // PreVote RPCs are sent for the next term before we've incremented r.Term. - term = pRaft->term + 1; - } else { - syncRaftBecomeCandidate(pRaft); - term = pRaft->term; - preVote = false; - } - - int quorum = syncRaftQuorum(pRaft); - ESyncRaftVoteResult result = syncRaftPollVote(pRaft, pRaft->selfId, preVote, true, NULL, NULL); - if (result == SYNC_RAFT_VOTE_WON) { - // We won the election after voting for ourselves (which must mean that - // this is a single-node cluster). Advance to the next state. - if (cType == SYNC_RAFT_CAMPAIGN_PRE_ELECTION) { - syncRaftStartElection(pRaft, SYNC_RAFT_CAMPAIGN_ELECTION); - } else { - syncRaftBecomeLeader(pRaft); - } - return; - } - - // broadcast vote message to other peers - int i; - SyncIndex lastIndex = syncRaftLogLastIndex(pRaft->log); - SyncTerm lastTerm = syncRaftLogLastTerm(pRaft->log); - SSyncRaftNodeMap nodeMap; - syncRaftJointConfigIDs(&pRaft->tracker->config.voters, &nodeMap); - SyncNodeId *pNodeId = NULL; - while (!syncRaftIterateNodeMap(&nodeMap, pNodeId)) { - SyncNodeId nodeId = *pNodeId; - if (nodeId == SYNC_NON_NODE_ID) { - continue; - } - - if (nodeId == pRaft->selfId) { - continue; - } - - SNodeInfo* pNode = syncRaftGetNodeById(pRaft, nodeId); - if (pNode == NULL) { - continue; - } - - SSyncMessage* pMsg = syncNewVoteMsg(pRaft->selfGroupId, pRaft->selfId, - term, cType, lastIndex, lastTerm); - if (pMsg == NULL) { - continue; - } - - syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 "] sent vote request to %d at term %" PRId64 "", - pRaft->selfGroupId, pRaft->selfId, lastTerm, - lastIndex, nodeId, pRaft->term); - - pRaft->io.send(pMsg, pNode); - } -} \ No newline at end of file diff --git a/source/libs/sync/src/sync_raft_impl.c b/source/libs/sync/src/sync_raft_impl.c deleted file mode 100644 index 3050bb2c8a..0000000000 --- a/source/libs/sync/src/sync_raft_impl.c +++ /dev/null @@ -1,369 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "raft.h" -#include "sync_raft_impl.h" -#include "raft_log.h" -#include "raft_replication.h" -#include "sync_raft_progress_tracker.h" -#include "syncInt.h" - -static int convertClear(SSyncRaft* pRaft); -static int stepFollower(SSyncRaft* pRaft, const SSyncMessage* pMsg); -static int stepCandidate(SSyncRaft* pRaft, const SSyncMessage* pMsg); -static int stepLeader(SSyncRaft* pRaft, const SSyncMessage* pMsg); - -static bool increaseUncommittedSize(SSyncRaft* pRaft, SSyncRaftEntry* entries, int n); - -static int triggerAll(SSyncRaft* pRaft); - -static void tickElection(SSyncRaft* pRaft); -static void tickHeartbeat(SSyncRaft* pRaft); - -static void appendEntries(SSyncRaft* pRaft, SSyncRaftEntry* entries, int n); - -static void abortLeaderTransfer(SSyncRaft* pRaft); - -static void resetRaft(SSyncRaft* pRaft, SyncTerm term); - -void syncRaftBecomeFollower(SSyncRaft* pRaft, SyncTerm term, SyncNodeId leaderId) { - convertClear(pRaft); - - pRaft->stepFp = stepFollower; - resetRaft(pRaft, term); - pRaft->tickFp = tickElection; - pRaft->leaderId = leaderId; - pRaft->state = TAOS_SYNC_STATE_FOLLOWER; - syncInfo("[%d:%d] became followe at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term); -} - -void syncRaftBecomePreCandidate(SSyncRaft* pRaft) { - convertClear(pRaft); - - /** - * Becoming a pre-candidate changes our step functions and state, - * but doesn't change anything else. In particular it does not increase - * r.Term or change r.Vote. - **/ - pRaft->stepFp = stepCandidate; - pRaft->tickFp = tickElection; - pRaft->state = TAOS_SYNC_STATE_CANDIDATE; - pRaft->candidateState.inPreVote = true; - syncInfo("[%d:%d] became pre-candidate at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term); -} - -void syncRaftBecomeCandidate(SSyncRaft* pRaft) { - convertClear(pRaft); - - pRaft->candidateState.inPreVote = false; - pRaft->stepFp = stepCandidate; - // become candidate make term+1 - resetRaft(pRaft, pRaft->term + 1); - pRaft->tickFp = tickElection; - pRaft->voteFor = pRaft->selfId; - pRaft->state = TAOS_SYNC_STATE_CANDIDATE; - syncInfo("[%d:%d] became candidate at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term); -} - -void syncRaftBecomeLeader(SSyncRaft* pRaft) { - assert(pRaft->state != TAOS_SYNC_STATE_FOLLOWER); - - pRaft->stepFp = stepLeader; - resetRaft(pRaft, pRaft->term); - pRaft->leaderId = pRaft->leaderId; - pRaft->state = TAOS_SYNC_STATE_LEADER; - - SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(&pRaft->tracker->progressMap, pRaft->selfId); - assert(progress != NULL); - // Followers enter replicate mode when they've been successfully probed - // (perhaps after having received a snapshot as a result). The leader is - // trivially in this state. Note that r.reset() has initialized this - // progress with the last index already. - syncRaftProgressBecomeReplicate(progress); - - // Conservatively set the pendingConfIndex to the last index in the - // log. There may or may not be a pending config change, but it's - // safe to delay any future proposals until we commit all our - // pending log entries, and scanning the entire tail of the log - // could be expensive. - SyncIndex lastIndex = syncRaftLogLastIndex(pRaft->log); - pRaft->pendingConfigIndex = lastIndex; - - // after become leader, send a no-op log - SSyncRaftEntry* entry = (SSyncRaftEntry*)malloc(sizeof(SSyncRaftEntry)); - if (entry == NULL) { - return; - } - *entry = (SSyncRaftEntry) { - .buffer = (SSyncBuffer) { - .data = NULL, - .len = 0, - } - }; - appendEntries(pRaft, entry, 1); - //syncRaftTriggerHeartbeat(pRaft); - syncInfo("[%d:%d] became leader at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, pRaft->term); -} - -void syncRaftTriggerHeartbeat(SSyncRaft* pRaft) { - triggerAll(pRaft); -} - -void syncRaftRandomizedElectionTimeout(SSyncRaft* pRaft) { - // electionTimeoutTick in [3,6] tick - pRaft->randomizedElectionTimeout = taosRand() % 4 + 3; -} - -bool syncRaftIsPromotable(SSyncRaft* pRaft) { - return pRaft->selfId != SYNC_NON_NODE_ID; -} - -bool syncRaftIsPastElectionTimeout(SSyncRaft* pRaft) { - return pRaft->electionElapsed >= pRaft->randomizedElectionTimeout; -} - -int syncRaftQuorum(SSyncRaft* pRaft) { - return 0; - //return pRaft->cluster.replica / 2 + 1; -} - -ESyncRaftVoteResult syncRaftPollVote(SSyncRaft* pRaft, SyncNodeId id, - bool preVote, bool grant, - int* rejected, int *granted) { - SNodeInfo* pNode = syncRaftGetNodeById(pRaft, id); - if (pNode == NULL) { - return true; - } - - if (grant) { - syncInfo("[%d:%d] received grant (pre-vote %d) from %d at term %" PRId64 "", - pRaft->selfGroupId, pRaft->selfId, preVote, id, pRaft->term); - } else { - syncInfo("[%d:%d] received rejection (pre-vote %d) from %d at term %" PRId64 "", - pRaft->selfGroupId, pRaft->selfId, preVote, id, pRaft->term); - } - - syncRaftRecordVote(pRaft->tracker, pNode->nodeId, grant); - return syncRaftTallyVotes(pRaft->tracker, rejected, granted); -} -/* - if (accept) { - syncInfo("[%d:%d] received (pre-vote %d) from %d at term %" PRId64 "", - pRaft->selfGroupId, pRaft->selfId, preVote, id, pRaft->term); - } else { - syncInfo("[%d:%d] received rejection from %d at term %" PRId64 "", - pRaft->selfGroupId, pRaft->selfId, id, pRaft->term); - } - - int voteIndex = syncRaftGetNodeById(pRaft, id); - assert(voteIndex < pRaft->cluster.replica && voteIndex >= 0); - assert(pRaft->candidateState.votes[voteIndex] == SYNC_RAFT_VOTE_RESP_UNKNOWN); - - pRaft->candidateState.votes[voteIndex] = accept ? SYNC_RAFT_VOTE_RESP_GRANT : SYNC_RAFT_VOTE_RESP_REJECT; - int granted = 0, rejected = 0; - int i; - for (i = 0; i < pRaft->cluster.replica; ++i) { - if (pRaft->candidateState.votes[i] == SYNC_RAFT_VOTE_RESP_GRANT) granted++; - else if (pRaft->candidateState.votes[i] == SYNC_RAFT_VOTE_RESP_REJECT) rejected++; - } - - if (rejectNum) *rejectNum = rejected; - return granted; -*/ - -void syncRaftLoadState(SSyncRaft* pRaft, const SSyncServerState* serverState) { - SyncIndex commitIndex = serverState->commitIndex; - SyncIndex lastIndex = syncRaftLogLastIndex(pRaft->log); - - if (commitIndex < pRaft->log->commitIndex || commitIndex > lastIndex) { - syncFatal("[%d:%d] state.commit %"PRId64" is out of range [%" PRId64 ",%" PRId64 "", - pRaft->selfGroupId, pRaft->selfId, commitIndex, pRaft->log->commitIndex, lastIndex); - return; - } - - pRaft->log->commitIndex = commitIndex; - pRaft->term = serverState->term; - pRaft->voteFor = serverState->voteFor; -} - -static void visitProgressSendAppend(SSyncRaftProgress* progress, void* arg) { - SSyncRaft* pRaft = (SSyncRaft*)arg; - if (pRaft->selfId == progress->id) { - return; - } - - syncRaftMaybeSendAppend(arg, progress, true); -} - -// bcastAppend sends RPC, with entries to all peers that are not up-to-date -// according to the progress recorded in r.prs. -void syncRaftBroadcastAppend(SSyncRaft* pRaft) { - syncRaftProgressVisit(pRaft->tracker, visitProgressSendAppend, pRaft); -} - -SNodeInfo* syncRaftGetNodeById(SSyncRaft *pRaft, SyncNodeId id) { - SNodeInfo **ppNode = taosHashGet(pRaft->nodeInfoMap, &id, sizeof(SyncNodeId*)); - if (ppNode != NULL) { - return *ppNode; - } - - return NULL; -} - -static int convertClear(SSyncRaft* pRaft) { - -} - -static int stepFollower(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - - return 0; -} - -static int stepCandidate(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - /** - * Only handle vote responses corresponding to our candidacy (while in - * StateCandidate, we may get stale MsgPreVoteResp messages in this term from - * our pre-candidate state). - **/ - ESyncRaftMessageType msgType = pMsg->msgType; - - if (msgType == RAFT_MSG_INTERNAL_PROP) { - return 0; - } - - if (msgType == RAFT_MSG_VOTE_RESP) { - syncRaftHandleVoteRespMessage(pRaft, pMsg); - return 0; - } else if (msgType == RAFT_MSG_APPEND) { - syncRaftBecomeFollower(pRaft, pMsg->term, pMsg->from); - syncRaftHandleAppendEntriesMessage(pRaft, pMsg); - } - return 0; -} - -static int stepLeader(SSyncRaft* pRaft, const SSyncMessage* pMsg) { - convertClear(pRaft); - return 0; -} - -// tickElection is run by followers and candidates after r.electionTimeout. -static void tickElection(SSyncRaft* pRaft) { - pRaft->electionElapsed += 1; - - if (!syncRaftIsPromotable(pRaft)) { - return; - } - - if (!syncRaftIsPastElectionTimeout(pRaft)) { - return; - } - - // election timeout - pRaft->electionElapsed = 0; - SSyncMessage msg; - syncRaftStep(pRaft, syncInitElectionMsg(&msg, pRaft->selfId)); -} - -// tickHeartbeat is run by leaders to send a MsgBeat after r.heartbeatTimeout. -static void tickHeartbeat(SSyncRaft* pRaft) { - -} - -// TODO -static bool increaseUncommittedSize(SSyncRaft* pRaft, SSyncRaftEntry* entries, int n) { - return false; -} - -static void appendEntries(SSyncRaft* pRaft, SSyncRaftEntry* entries, int n) { - SyncIndex lastIndex = syncRaftLogLastIndex(pRaft->log); - SyncTerm term = pRaft->term; - int i; - - for (i = 0; i < n; ++i) { - entries[i].term = term; - entries[i].index = lastIndex + 1 + i; - } - - // Track the size of this uncommitted proposal. - if (!increaseUncommittedSize(pRaft, entries, n)) { - // Drop the proposal. - return; - } - - syncRaftLogAppend(pRaft->log, entries, n); - - SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(&pRaft->tracker->progressMap, pRaft->selfId); - assert(progress != NULL); - syncRaftProgressMaybeUpdate(progress, lastIndex); - // Regardless of syncRaftMaybeCommit's return, our caller will call bcastAppend. - syncRaftMaybeCommit(pRaft); -} - -// syncRaftMaybeCommit attempts to advance the commit index. Returns true if -// the commit index changed (in which case the caller should call -// r.bcastAppend). -bool syncRaftMaybeCommit(SSyncRaft* pRaft) { - - return true; -} - -/** - * trigger I/O requests for newly appended log entries or heartbeats. - **/ -static int triggerAll(SSyncRaft* pRaft) { - #if 0 - assert(pRaft->state == TAOS_SYNC_STATE_LEADER); - int i; - - for (i = 0; i < pRaft->cluster.replica; ++i) { - if (i == pRaft->cluster.selfIndex) { - continue; - } - - syncRaftMaybeSendAppend(pRaft, pRaft->tracker->progressMap.progress[i], true); - } - #endif - return 0; -} - -static void abortLeaderTransfer(SSyncRaft* pRaft) { - pRaft->leadTransferee = SYNC_NON_NODE_ID; -} - -static void resetProgress(SSyncRaftProgress* progress, void* arg) { - syncRaftResetProgress((SSyncRaft*)arg, progress); -} - -static void resetRaft(SSyncRaft* pRaft, SyncTerm term) { - if (pRaft->term != term) { - pRaft->term = term; - pRaft->voteFor = SYNC_NON_NODE_ID; - } - - pRaft->leaderId = SYNC_NON_NODE_ID; - - pRaft->electionElapsed = 0; - pRaft->heartbeatElapsed = 0; - - syncRaftRandomizedElectionTimeout(pRaft); - - abortLeaderTransfer(pRaft); - - syncRaftResetVotes(pRaft->tracker); - syncRaftProgressVisit(pRaft->tracker, resetProgress, pRaft); - - pRaft->pendingConfigIndex = 0; - pRaft->uncommittedSize = 0; -} \ No newline at end of file diff --git a/source/libs/sync/src/sync_raft_inflights.c b/source/libs/sync/src/sync_raft_inflights.c deleted file mode 100644 index 7b97aca014..0000000000 --- a/source/libs/sync/src/sync_raft_inflights.c +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sync_raft_inflights.h" - -SSyncRaftInflights* syncRaftOpenInflights(int size) { - SSyncRaftInflights* inflights = (SSyncRaftInflights*)malloc(sizeof(SSyncRaftInflights)); - if (inflights == NULL) { - return NULL; - } - SyncIndex* buffer = (SyncIndex*)malloc(sizeof(SyncIndex) * size); - if (buffer == NULL) { - free(inflights); - return NULL; - } - *inflights = (SSyncRaftInflights) { - .buffer = buffer, - .count = 0, - .size = 0, - .start = 0, - }; - - return inflights; -} - -void syncRaftCloseInflights(SSyncRaftInflights* inflights) { - free(inflights->buffer); - free(inflights); -} - -// Add notifies the Inflights that a new message with the given index is being -// dispatched. Full() must be called prior to Add() to verify that there is room -// for one more message, and consecutive calls to add Add() must provide a -// monotonic sequence of indexes. -void syncRaftInflightAdd(SSyncRaftInflights* inflights, SyncIndex inflightIndex) { - assert(!syncRaftInflightFull(inflights)); - - int next = inflights->start + inflights->count; - int size = inflights->size; - - if (next >= size) { - next -= size; - } - - inflights->buffer[next] = inflightIndex; - inflights->count++; -} - -// FreeLE frees the inflights smaller or equal to the given `to` flight. -void syncRaftInflightFreeLE(SSyncRaftInflights* inflights, SyncIndex toIndex) { - if (inflights->count == 0 || toIndex < inflights->buffer[inflights->start]) { - // out of the left side of the window - return; - } - - int i, idx; - for (i = 0, idx = inflights->start; i < inflights->count; i++) { - if (toIndex < inflights->buffer[idx]) { // found the first large inflight - break; - } - - // increase index and maybe rotate - int size = inflights->size; - idx++; - if (idx >= size) { - idx -= size; - } - } - - // free i inflights and set new start index - inflights->count -= i; - inflights->start = idx; - assert(inflights->count >= 0); - if (inflights->count == 0) { - // inflights is empty, reset the start index so that we don't grow the - // buffer unnecessarily. - inflights->start = 0; - } -} - -// FreeFirstOne releases the first inflight. This is a no-op if nothing is -// inflight. -void syncRaftInflightFreeFirstOne(SSyncRaftInflights* inflights) { - syncRaftInflightFreeLE(inflights, inflights->buffer[inflights->start]); -} diff --git a/source/libs/sync/src/sync_raft_node_map.c b/source/libs/sync/src/sync_raft_node_map.c deleted file mode 100644 index 642eebe65b..0000000000 --- a/source/libs/sync/src/sync_raft_node_map.c +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sync_raft_node_map.h" -#include "sync_type.h" -#include "sync_raft_progress.h" - -void syncRaftInitNodeMap(SSyncRaftNodeMap* nodeMap) { - nodeMap->nodeIdMap = taosHashInit(TSDB_MAX_REPLICA, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); -} - -void syncRaftFreeNodeMap(SSyncRaftNodeMap* nodeMap) { - taosHashCleanup(nodeMap->nodeIdMap); -} - -void syncRaftClearNodeMap(SSyncRaftNodeMap* nodeMap) { - taosHashClear(nodeMap->nodeIdMap); -} - -bool syncRaftIsInNodeMap(const SSyncRaftNodeMap* nodeMap, SyncNodeId nodeId) { - SyncNodeId** ppId = (SyncNodeId**)taosHashGet(nodeMap->nodeIdMap, &nodeId, sizeof(SyncNodeId*)); - if (ppId == NULL) { - return false; - } - return true; -} - -void syncRaftCopyNodeMap(SSyncRaftNodeMap* from, SSyncRaftNodeMap* to) { - SyncNodeId *pId = NULL; - while (!syncRaftIterateNodeMap(from, pId)) { - taosHashPut(to->nodeIdMap, &pId, sizeof(SyncNodeId*), &pId, sizeof(SyncNodeId*)); - } -} - -bool syncRaftIterateNodeMap(const SSyncRaftNodeMap* nodeMap, SyncNodeId *pId) { - SyncNodeId **ppId = taosHashIterate(nodeMap->nodeIdMap, pId); - if (ppId == NULL) { - return true; - } - - *pId = *(*ppId); - return false; -} - -bool syncRaftIsAllNodeInProgressMap(SSyncRaftNodeMap* nodeMap, SSyncRaftProgressMap* progressMap) { - SyncNodeId *pId = NULL; - while (!syncRaftIterateNodeMap(nodeMap, pId)) { - if (!syncRaftIsInProgressMap(progressMap, *pId)) { - return false; - } - } - - return true; -} - -void syncRaftUnionNodeMap(SSyncRaftNodeMap* nodeMap, SSyncRaftNodeMap* to) { - syncRaftCopyNodeMap(nodeMap, to); -} - -void syncRaftAddToNodeMap(SSyncRaftNodeMap* nodeMap, SyncNodeId nodeId) { - taosHashPut(nodeMap->nodeIdMap, &nodeId, sizeof(SyncNodeId*), &nodeId, sizeof(SyncNodeId*)); -} - -void syncRaftRemoveFromNodeMap(SSyncRaftNodeMap* nodeMap, SyncNodeId nodeId) { - taosHashRemove(nodeMap->nodeIdMap, &nodeId, sizeof(SyncNodeId*)); -} - -int32_t syncRaftNodeMapSize(const SSyncRaftNodeMap* nodeMap) { - return taosHashGetSize(nodeMap->nodeIdMap); -} \ No newline at end of file diff --git a/source/libs/sync/src/sync_raft_progress.c b/source/libs/sync/src/sync_raft_progress.c deleted file mode 100644 index 6577972b29..0000000000 --- a/source/libs/sync/src/sync_raft_progress.c +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "raft.h" -#include "raft_log.h" -#include "sync_raft_progress.h" -#include "sync_raft_progress_tracker.h" -#include "sync.h" -#include "syncInt.h" - -static void copyProgress(SSyncRaftProgress* progress, void* arg); - -static void refProgress(SSyncRaftProgress* progress); -static void unrefProgress(SSyncRaftProgress* progress, void*); - -static void resetProgressState(SSyncRaftProgress* progress, ESyncRaftProgressState state); -static void probeAcked(SSyncRaftProgress* progress); - -static void resumeProgress(SSyncRaftProgress* progress); - -void syncRaftResetProgress(SSyncRaft* pRaft, SSyncRaftProgress* progress) { - if (progress->inflights) { - syncRaftCloseInflights(progress->inflights); - } - SSyncRaftInflights* inflights = syncRaftOpenInflights(pRaft->tracker->maxInflightMsgs); - if (inflights == NULL) { - return; - } - *progress = (SSyncRaftProgress) { - .matchIndex = progress->id == pRaft->selfId ? syncRaftLogLastIndex(pRaft->log) : 0, - .nextIndex = syncRaftLogLastIndex(pRaft->log) + 1, - .inflights = inflights, - .isLearner = false, - .state = PROGRESS_STATE_PROBE, - }; -} - -// MaybeUpdate is called when an MsgAppResp arrives from the follower, with the -// index acked by it. The method returns false if the given n index comes from -// an outdated message. Otherwise it updates the progress and returns true. -bool syncRaftProgressMaybeUpdate(SSyncRaftProgress* progress, SyncIndex lastIndex) { - bool updated = false; - - if (progress->matchIndex < lastIndex) { - progress->matchIndex = lastIndex; - updated = true; - probeAcked(progress); - } - - progress->nextIndex = TMAX(progress->nextIndex, lastIndex + 1); - - return updated; -} - -// MaybeDecrTo adjusts the Progress to the receipt of a MsgApp rejection. The -// arguments are the index of the append message rejected by the follower, and -// the hint that we want to decrease to. -// -// Rejections can happen spuriously as messages are sent out of order or -// duplicated. In such cases, the rejection pertains to an index that the -// Progress already knows were previously acknowledged, and false is returned -// without changing the Progress. -// -// If the rejection is genuine, Next is lowered sensibly, and the Progress is -// cleared for sending log entries. -bool syncRaftProgressMaybeDecrTo(SSyncRaftProgress* progress, - SyncIndex rejected, SyncIndex matchHint) { - if (progress->state == PROGRESS_STATE_REPLICATE) { - // The rejection must be stale if the progress has matched and "rejected" - // is smaller than "match". - if (rejected <= progress->matchIndex) { - syncDebug("match index is up to date,ignore"); - return false; - } - - // Directly decrease next to match + 1. - // - // TODO(tbg): why not use matchHint if it's larger? - progress->nextIndex = progress->matchIndex + 1; - return true; - } - - // The rejection must be stale if "rejected" does not match next - 1. This - // is because non-replicating followers are probed one entry at a time. - if (rejected != progress->nextIndex - 1) { - syncDebug("rejected index %" PRId64 " different from next index %" PRId64 " -> ignore" - , rejected, progress->nextIndex); - return false; - } - - progress->nextIndex = TMAX(TMIN(rejected, matchHint + 1), 1); - - progress->probeSent = false; - return true; -} - -// IsPaused returns whether sending log entries to this node has been throttled. -// This is done when a node has rejected recent MsgApps, is currently waiting -// for a snapshot, or has reached the MaxInflightMsgs limit. In normal -// operation, this is false. A throttled node will be contacted less frequently -// until it has reached a state in which it's able to accept a steady stream of -// log entries again. -bool syncRaftProgressIsPaused(SSyncRaftProgress* progress) { - switch (progress->state) { - case PROGRESS_STATE_PROBE: - return progress->probeSent; - case PROGRESS_STATE_REPLICATE: - return syncRaftInflightFull(progress->inflights); - case PROGRESS_STATE_SNAPSHOT: - return true; - default: - syncFatal("error sync state:%d", progress->state); - } -} - -SSyncRaftProgress* syncRaftFindProgressByNodeId(const SSyncRaftProgressMap* progressMap, SyncNodeId id) { - SSyncRaftProgress** ppProgress = (SSyncRaftProgress**)taosHashGet(progressMap->progressMap, &id, sizeof(SyncNodeId*)); - if (ppProgress == NULL) { - return NULL; - } - - return *ppProgress; -} - -int syncRaftAddToProgressMap(SSyncRaftProgressMap* progressMap, SSyncRaftProgress* progress) { - refProgress(progress); - taosHashPut(progressMap->progressMap, &progress->id, sizeof(SyncNodeId*), &progress, sizeof(SSyncRaftProgress*)); -} - -void syncRaftRemoveFromProgressMap(SSyncRaftProgressMap* progressMap, SyncNodeId id) { - SSyncRaftProgress** ppProgress = (SSyncRaftProgress**)taosHashGet(progressMap->progressMap, &id, sizeof(SyncNodeId*)); - if (ppProgress == NULL) { - return; - } - unrefProgress(*ppProgress, NULL); - - taosHashRemove(progressMap->progressMap, &id, sizeof(SyncNodeId*)); -} - -bool syncRaftIsInProgressMap(SSyncRaftProgressMap* progressMap, SyncNodeId id) { - return taosHashGet(progressMap->progressMap, &id, sizeof(SyncNodeId*)) != NULL; -} - -bool syncRaftProgressIsUptodate(SSyncRaft* pRaft, SSyncRaftProgress* progress) { - return syncRaftLogLastIndex(pRaft->log) + 1 == progress->nextIndex; -} - -// BecomeProbe transitions into StateProbe. Next is reset to Match+1 or, -// optionally and if larger, the index of the pending snapshot. -void syncRaftProgressBecomeProbe(SSyncRaftProgress* progress) { - // If the original state is StateSnapshot, progress knows that - // the pending snapshot has been sent to this peer successfully, then - // probes from pendingSnapshot + 1. - if (progress->state == PROGRESS_STATE_SNAPSHOT) { - SyncIndex pendingSnapshotIndex = progress->pendingSnapshotIndex; - resetProgressState(progress, PROGRESS_STATE_PROBE); - progress->nextIndex = TMAX(progress->matchIndex + 1, pendingSnapshotIndex + 1); - } else { - resetProgressState(progress, PROGRESS_STATE_PROBE); - progress->nextIndex = progress->matchIndex + 1; - } -} - -// BecomeReplicate transitions into StateReplicate, resetting Next to Match+1. -void syncRaftProgressBecomeReplicate(SSyncRaftProgress* progress) { - resetProgressState(progress, PROGRESS_STATE_REPLICATE); - progress->nextIndex = progress->matchIndex + 1; -} - -// BecomeSnapshot moves the Progress to StateSnapshot with the specified pending -// snapshot index. -void syncRaftProgressBecomeSnapshot(SSyncRaftProgress* progress, SyncIndex snapshotIndex) { - resetProgressState(progress, PROGRESS_STATE_SNAPSHOT); - progress->pendingSnapshotIndex = snapshotIndex; -} - -void syncRaftCopyProgress(const SSyncRaftProgress* progress, SSyncRaftProgress* out) { - memcpy(out, progress, sizeof(SSyncRaftProgress)); -} - -void syncRaftInitProgressMap(SSyncRaftProgressMap* progressMap) { - progressMap->progressMap = taosHashInit(TSDB_MAX_REPLICA, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); -} - -void syncRaftFreeProgressMap(SSyncRaftProgressMap* progressMap) { - syncRaftVisitProgressMap(progressMap, unrefProgress, NULL); - taosHashCleanup(progressMap->progressMap); -} - -void syncRaftClearProgressMap(SSyncRaftProgressMap* progressMap) { - taosHashClear(progressMap->progressMap); -} - -void syncRaftCopyProgressMap(SSyncRaftProgressMap* from, SSyncRaftProgressMap* to) { - syncRaftVisitProgressMap(from, copyProgress, to); -} - -bool syncRaftIterateProgressMap(const SSyncRaftProgressMap* progressMap, SSyncRaftProgress *pProgress) { - SSyncRaftProgress **ppProgress = taosHashIterate(progressMap->progressMap, pProgress); - if (ppProgress == NULL) { - return true; - } - - *pProgress = *(*ppProgress); - return false; -} - -bool syncRaftVisitProgressMap(SSyncRaftProgressMap* progressMap, visitProgressFp fp, void* arg) { - SSyncRaftProgress *pProgress; - while (!syncRaftIterateProgressMap(progressMap, pProgress)) { - fp(pProgress, arg); - } -} - -static void copyProgress(SSyncRaftProgress* progress, void* arg) { - assert(progress->refCount > 0); - SSyncRaftProgressMap* to = (SSyncRaftProgressMap*)arg; - syncRaftAddToProgressMap(to, progress); -} - -static void refProgress(SSyncRaftProgress* progress) { - progress->refCount += 1; -} - -static void unrefProgress(SSyncRaftProgress* progress, void* arg) { - (void)arg; - progress->refCount -= 1; - assert(progress->refCount >= 0); - if (progress->refCount == 0) { - free(progress); - } -} - -// ResetState moves the Progress into the specified State, resetting ProbeSent, -// PendingSnapshot, and Inflights. -static void resetProgressState(SSyncRaftProgress* progress, ESyncRaftProgressState state) { - progress->probeSent = false; - progress->pendingSnapshotIndex = 0; - progress->state = state; - syncRaftInflightReset(progress->inflights); -} - -// ProbeAcked is called when this peer has accepted an append. It resets -// ProbeSent to signal that additional append messages should be sent without -// further delay. -static void probeAcked(SSyncRaftProgress* progress) { - progress->probeSent = false; -} diff --git a/source/libs/sync/src/sync_raft_progress_tracker.c b/source/libs/sync/src/sync_raft_progress_tracker.c deleted file mode 100644 index e0b4afae21..0000000000 --- a/source/libs/sync/src/sync_raft_progress_tracker.c +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "raft.h" -#include "sync_const.h" -#include "sync_raft_progress_tracker.h" -#include "sync_raft_proto.h" - -SSyncRaftProgressTracker* syncRaftOpenProgressTracker(SSyncRaft* pRaft) { - SSyncRaftProgressTracker* tracker = (SSyncRaftProgressTracker*)malloc(sizeof(SSyncRaftProgressTracker)); - if (tracker == NULL) { - return NULL; - } - - tracker->votesMap = taosHashInit(TSDB_MAX_REPLICA, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - - syncRaftInitTrackConfig(&tracker->config); - tracker->pRaft = pRaft; - tracker->maxInflightMsgs = kSyncRaftMaxInflghtMsgs; - - return tracker; -} - -void syncRaftInitTrackConfig(SSyncRaftProgressTrackerConfig* config) { - syncRaftInitNodeMap(&config->learners); - syncRaftInitNodeMap(&config->learnersNext); - syncRaftInitQuorumJointConfig(&config->voters); - config->autoLeave = false; -} - -void syncRaftFreeTrackConfig(SSyncRaftProgressTrackerConfig* config) { - syncRaftFreeNodeMap(&config->learners); - syncRaftFreeNodeMap(&config->learnersNext); - syncRaftFreeNodeMap(&config->voters.incoming); - syncRaftFreeNodeMap(&config->voters.outgoing); -} - -// ResetVotes prepares for a new round of vote counting via recordVote. -void syncRaftResetVotes(SSyncRaftProgressTracker* tracker) { - taosHashClear(tracker->votesMap); -} - -void syncRaftProgressVisit(SSyncRaftProgressTracker* tracker, visitProgressFp visit, void* arg) { - syncRaftVisitProgressMap(&tracker->progressMap, visit, arg); -} - -// RecordVote records that the node with the given id voted for this Raft -// instance if v == true (and declined it otherwise). -void syncRaftRecordVote(SSyncRaftProgressTracker* tracker, SyncNodeId id, bool grant) { - ESyncRaftVoteType* pType = taosHashGet(tracker->votesMap, &id, sizeof(SyncNodeId*)); - if (pType != NULL) { - return; - } - - taosHashPut(tracker->votesMap, &id, sizeof(SyncNodeId), &grant, sizeof(bool*)); -} - -void syncRaftCopyTrackerConfig(const SSyncRaftProgressTrackerConfig* from, SSyncRaftProgressTrackerConfig* to) { - memcpy(to, from, sizeof(SSyncRaftProgressTrackerConfig)); -} - -int syncRaftCheckTrackerConfigInProgress(SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) { - // NB: intentionally allow the empty config. In production we'll never see a - // non-empty config (we prevent it from being created) but we will need to - // be able to *create* an initial config, for example during bootstrap (or - // during tests). Instead of having to hand-code this, we allow - // transitioning from an empty config into any other legal and non-empty - // config. - if (!syncRaftIsAllNodeInProgressMap(&config->voters.incoming, progressMap)) return -1; - if (!syncRaftIsAllNodeInProgressMap(&config->voters.outgoing, progressMap)) return -1; - if (!syncRaftIsAllNodeInProgressMap(&config->learners, progressMap)) return -1; - if (!syncRaftIsAllNodeInProgressMap(&config->learnersNext, progressMap)) return -1; - return 0; -} - -// TallyVotes returns the number of granted and rejected Votes, and whether the -// election outcome is known. -ESyncRaftVoteResult syncRaftTallyVotes(SSyncRaftProgressTracker* tracker, int* rejected, int *granted) { - SSyncRaftProgress* progress = NULL; - int r, g; - - // Make sure to populate granted/rejected correctly even if the Votes slice - // contains members no longer part of the configuration. This doesn't really - // matter in the way the numbers are used (they're informational), but might - // as well get it right. - while (!syncRaftIterateProgressMap(&tracker->progressMap, progress)) { - if (progress->id == SYNC_NON_NODE_ID) { - continue; - } - - bool* v = taosHashGet(tracker->votesMap, &progress->id, sizeof(SyncNodeId*)); - if (v == NULL) { - continue; - } - - if (*v) { - g++; - } else { - r++; - } - } - - if (rejected) *rejected = r; - if (granted) *granted = g; - return syncRaftVoteResult(&(tracker->config.voters), tracker->votesMap); -} - -void syncRaftConfigState(SSyncRaftProgressTracker* tracker, SSyncConfigState* cs) { - syncRaftCopyNodeMap(&tracker->config.voters.incoming, &cs->voters); - syncRaftCopyNodeMap(&tracker->config.voters.outgoing, &cs->votersOutgoing); - syncRaftCopyNodeMap(&tracker->config.learners, &cs->learners); - syncRaftCopyNodeMap(&tracker->config.learnersNext, &cs->learnersNext); - cs->autoLeave = tracker->config.autoLeave; -} - -static void matchAckIndexer(SyncNodeId id, void* arg, SyncIndex* index) { - SSyncRaftProgressTracker* tracker = (SSyncRaftProgressTracker*)arg; - SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(&tracker->progressMap, id); - if (progress == NULL) { - *index = 0; - return; - } - *index = progress->matchIndex; -} - -// Committed returns the largest log index known to be committed based on what -// the voting members of the group have acknowledged. -SyncIndex syncRaftCommittedIndex(SSyncRaftProgressTracker* tracker) { - return syncRaftJointConfigCommittedIndex(&tracker->config.voters, matchAckIndexer, tracker); -} - -static void visitProgressActive(SSyncRaftProgress* progress, void* arg) { - SHashObj* votesMap = (SHashObj*)arg; - taosHashPut(votesMap, &progress->id, sizeof(SyncNodeId), &progress->recentActive, sizeof(bool)); -} - -// QuorumActive returns true if the quorum is active from the view of the local -// raft state machine. Otherwise, it returns false. -bool syncRaftQuorumActive(SSyncRaftProgressTracker* tracker) { - SHashObj* votesMap = taosHashInit(TSDB_MAX_REPLICA, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - syncRaftVisitProgressMap(&tracker->progressMap, visitProgressActive, votesMap); - - return syncRaftVoteResult(&tracker->config.voters, votesMap) == SYNC_RAFT_VOTE_WON; -} \ No newline at end of file diff --git a/source/libs/sync/src/sync_raft_quorum_joint.c b/source/libs/sync/src/sync_raft_quorum_joint.c deleted file mode 100644 index 70c078b6f5..0000000000 --- a/source/libs/sync/src/sync_raft_quorum_joint.c +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sync_raft_node_map.h" -#include "sync_raft_quorum_majority.h" -#include "sync_raft_quorum_joint.h" -#include "sync_raft_quorum.h" - -/** - * syncRaftVoteResult takes a mapping of voters to yes/no (true/false) votes and returns - * a result indicating whether the vote is pending, lost, or won. A joint quorum - * requires both majority quorums to vote in favor. - **/ -ESyncRaftVoteType syncRaftVoteResult(SSyncRaftQuorumJointConfig* config, SHashObj* votesMap) { - ESyncRaftVoteResult r1 = syncRaftMajorityVoteResult(&(config->incoming), votesMap); - ESyncRaftVoteResult r2 = syncRaftMajorityVoteResult(&(config->outgoing), votesMap); - - if (r1 == r2) { - // If they agree, return the agreed state. - return r1; - } - - if (r1 == SYNC_RAFT_VOTE_LOST || r2 == SYNC_RAFT_VOTE_LOST) { - // If either config has lost, loss is the only possible outcome. - return SYNC_RAFT_VOTE_LOST; - } - - // One side won, the other one is pending, so the whole outcome is. - return SYNC_RAFT_VOTE_PENDING; -} - -void syncRaftInitQuorumJointConfig(SSyncRaftQuorumJointConfig* config) { - syncRaftInitNodeMap(&config->incoming); - syncRaftInitNodeMap(&config->outgoing); -} - -void syncRaftFreeQuorumJointConfig(SSyncRaftQuorumJointConfig* config) { - syncRaftFreeNodeMap(&config->incoming); - syncRaftFreeNodeMap(&config->outgoing); -} - -void syncRaftJointConfigAddToIncoming(SSyncRaftQuorumJointConfig* config, SyncNodeId id) { - syncRaftAddToNodeMap(&config->incoming, id); -} - -void syncRaftJointConfigRemoveFromIncoming(SSyncRaftQuorumJointConfig* config, SyncNodeId id) { - syncRaftRemoveFromNodeMap(&config->incoming, id); -} - -void syncRaftJointConfigIDs(SSyncRaftQuorumJointConfig* config, SSyncRaftNodeMap* nodeMap) { - syncRaftCopyNodeMap(&config->incoming, nodeMap); - - syncRaftUnionNodeMap(&config->outgoing, nodeMap); -} - -SyncIndex syncRaftJointConfigCommittedIndex(const SSyncRaftQuorumJointConfig* config, matchAckIndexerFp indexer, void* arg) { - SyncIndex index0, index1; - - index0 = syncRaftMajorityConfigCommittedIndex(&config->incoming, indexer, arg); - index1 = syncRaftMajorityConfigCommittedIndex(&config->outgoing, indexer, arg); - - return index0 < index1 ? index0 : index1; -} \ No newline at end of file diff --git a/source/libs/sync/src/sync_raft_quorum_majority.c b/source/libs/sync/src/sync_raft_quorum_majority.c deleted file mode 100644 index 313f213cda..0000000000 --- a/source/libs/sync/src/sync_raft_quorum_majority.c +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sync_const.h" -#include "sync_raft_quorum.h" -#include "sync_raft_quorum_majority.h" -#include "sync_raft_node_map.h" - -// VoteResult takes a mapping of voters to yes/no (true/false) votes and returns -// a result indicating whether the vote is pending (i.e. neither a quorum of -// yes/no has been reached), won (a quorum of yes has been reached), or lost (a -// quorum of no has been reached). -ESyncRaftVoteResult syncRaftMajorityVoteResult(SSyncRaftNodeMap* config, SHashObj* votesMap) { - int n = syncRaftNodeMapSize(config); - if (n == 0) { - // By convention, the elections on an empty config win. This comes in - // handy with joint quorums because it'll make a half-populated joint - // quorum behave like a majority quorum. - return SYNC_RAFT_VOTE_WON; - } - - int i, g, r, missing; - i = g = r = missing = 0; - SyncNodeId* pId = NULL; - while (!syncRaftIterateNodeMap(config, pId)) { - const bool* v = (const bool*)taosHashGet(votesMap, pId, sizeof(SyncNodeId*)); - if (v == NULL) { - missing += 1; - continue; - } - - if (*v) { - g +=1; - } else { - r += 1; - } - } - - int quorum = n / 2 + 1; - if (g >= quorum) { - return SYNC_RAFT_VOTE_WON; - } - if (g + missing >= quorum) { - return SYNC_RAFT_VOTE_PENDING; - } - - return SYNC_RAFT_VOTE_LOST; -} - -int compSyncIndex(const void * elem1, const void * elem2) { - SyncIndex index1 = *((SyncIndex*)elem1); - SyncIndex index2 = *((SyncIndex*)elem1); - if (index1 > index2) return 1; - if (index1 < index2) return -1; - return 0; -} - -SyncIndex syncRaftMajorityConfigCommittedIndex(const SSyncRaftNodeMap* config, matchAckIndexerFp indexer, void* arg) { - int n = syncRaftNodeMapSize(config); - if (n == 0) { - // This plays well with joint quorums which, when one half is the zero - // MajorityConfig, should behave like the other half. - return kMaxCommitIndex; - } - - // Use an on-stack slice to collect the committed indexes when n <= 7 - // (otherwise we alloc). The alternative is to stash a slice on - // MajorityConfig, but this impairs usability (as is, MajorityConfig is just - // a map, and that's nice). The assumption is that running with a - // replication factor of >7 is rare, and in cases in which it happens - // performance is a lesser concern (additionally the performance - // implications of an allocation here are far from drastic). - SyncIndex* srt = NULL; - SyncIndex srk[TSDB_MAX_REPLICA]; - if (n > TSDB_MAX_REPLICA) { - srt = (SyncIndex*)malloc(sizeof(SyncIndex) * n); - if (srt == NULL) { - return kMaxCommitIndex; - } - } else { - srt = &srk[0]; - } - - // Fill the slice with the indexes observed. Any unused slots will be - // left as zero; these correspond to voters that may report in, but - // haven't yet. We fill from the right (since the zeroes will end up on - // the left after sorting below anyway). - SyncNodeId *pId = NULL; - int i = 0; - SyncIndex index; - while (!syncRaftIterateNodeMap(config, pId)) { - indexer(*pId, arg, &index); - srt[i++] = index; - } - - // Sort by index. Use a bespoke algorithm (copied from the stdlib's sort - // package) to keep srt on the stack. - qsort(srt, n, sizeof(SyncIndex), compSyncIndex); - - // The smallest index into the array for which the value is acked by a - // quorum. In other words, from the end of the slice, move n/2+1 to the - // left (accounting for zero-indexing). - index = srt[n - (n/2 + 1)]; - if (srt != &srk[0]) { - free(srt); - } - - return index; -} \ No newline at end of file diff --git a/source/libs/sync/src/sync_raft_restore.c b/source/libs/sync/src/sync_raft_restore.c deleted file mode 100644 index d1acd3e8e9..0000000000 --- a/source/libs/sync/src/sync_raft_restore.c +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sync_raft_config_change.h" -#include "sync_raft_restore.h" -#include "sync_raft_progress_tracker.h" - -static void addToConfChangeSingleArray(SSyncConfChangeSingleArray* out, int* i, const SSyncRaftNodeMap* nodeMap, ESyncRaftConfChangeType t); -static int toConfChangeSingle(const SSyncConfigState* cs, SSyncConfChangeSingleArray* out, SSyncConfChangeSingleArray* in); - -// syncRaftRestoreConfig takes a Changer (which must represent an empty configuration), and -// runs a sequence of changes enacting the configuration described in the -// ConfState. -// -// TODO(tbg) it's silly that this takes a Changer. Unravel this by making sure -// the Changer only needs a ProgressMap (not a whole Tracker) at which point -// this can just take LastIndex and MaxInflight directly instead and cook up -// the results from that alone. -int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs, - SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) { - SSyncConfChangeSingleArray outgoing; - SSyncConfChangeSingleArray incoming; - SSyncConfChangeSingleArray css; - SSyncRaftProgressTracker* tracker = changer->tracker; - int i, ret; - - syncRaftInitConfArray(&outgoing); - syncRaftInitConfArray(&incoming); - - syncRaftInitTrackConfig(config); - syncRaftInitProgressMap(progressMap); - - ret = toConfChangeSingle(cs, &outgoing, &incoming); - if (ret != 0) { - goto out; - } - - if (syncRaftConfArrayIsEmpty(&outgoing)) { - // No outgoing config, so just apply the incoming changes one by one. - for (i = 0; i < incoming.n; ++i) { - css = (SSyncConfChangeSingleArray) { - .n = 1, - .changes = &incoming.changes[i], - }; - ret = syncRaftChangerSimpleConfig(changer, &css, config, progressMap); - if (ret != 0) { - goto out; - } - - syncRaftCopyTrackerConfig(config, &changer->tracker->config); - syncRaftCopyProgressMap(progressMap, &changer->tracker->progressMap); - } - } else { - // The ConfState describes a joint configuration. - // - // First, apply all of the changes of the outgoing config one by one, so - // that it temporarily becomes the incoming active config. For example, - // if the config is (1 2 3)&(2 3 4), this will establish (2 3 4)&(). - for (i = 0; i < outgoing.n; ++i) { - css = (SSyncConfChangeSingleArray) { - .n = 1, - .changes = &outgoing.changes[i], - }; - ret = syncRaftChangerSimpleConfig(changer, &css, config, progressMap); - if (ret != 0) { - goto out; - } - syncRaftCopyTrackerConfig(config, &changer->tracker->config); - syncRaftCopyProgressMap(progressMap, &changer->tracker->progressMap); - } - - ret = syncRaftChangerEnterJoint(changer, cs->autoLeave, &incoming, config, progressMap); - if (ret != 0) { - goto out; - } - } - -out: - syncRaftFreeConfArray(&incoming); - syncRaftFreeConfArray(&outgoing); - - return ret; -} - -static void addToConfChangeSingleArray(SSyncConfChangeSingleArray* out, int* i, const SSyncRaftNodeMap* nodeMap, ESyncRaftConfChangeType t) { - SyncNodeId* pId = NULL; - - while (!syncRaftIterateNodeMap(nodeMap, pId)) { - out->changes[*i] = (SSyncConfChangeSingle) { - .type = t, - .nodeId = *pId, - }; - *i += 1; - } -} - -// toConfChangeSingle translates a conf state into 1) a slice of operations creating -// first the config that will become the outgoing one, and then the incoming one, and -// b) another slice that, when applied to the config resulted from 1), represents the -// ConfState. -static int toConfChangeSingle(const SSyncConfigState* cs, SSyncConfChangeSingleArray* out, SSyncConfChangeSingleArray* in) { - int i; - - out->n = syncRaftNodeMapSize(&cs->votersOutgoing); - out->changes = (SSyncConfChangeSingle*)malloc(sizeof(SSyncConfChangeSingle) * out->n); - if (out->changes == NULL) { - out->n = 0; - return -1; - } - in->n = syncRaftNodeMapSize(&cs->votersOutgoing) + - syncRaftNodeMapSize(&cs->voters) + - syncRaftNodeMapSize(&cs->learners) + - syncRaftNodeMapSize(&cs->learnersNext); - out->changes = (SSyncConfChangeSingle*)malloc(sizeof(SSyncConfChangeSingle) * in->n); - if (in->changes == NULL) { - in->n = 0; - return -1; - } - - // Example to follow along this code: - // voters=(1 2 3) learners=(5) outgoing=(1 2 4 6) learners_next=(4) - // - // This means that before entering the joint config, the configuration - // had voters (1 2 4 6) and perhaps some learners that are already gone. - // The new set of voters is (1 2 3), i.e. (1 2) were kept around, and (4 6) - // are no longer voters; however 4 is poised to become a learner upon leaving - // the joint state. - // We can't tell whether 5 was a learner before entering the joint config, - // but it doesn't matter (we'll pretend that it wasn't). - // - // The code below will construct - // outgoing = add 1; add 2; add 4; add 6 - // incoming = remove 1; remove 2; remove 4; remove 6 - // add 1; add 2; add 3; - // add-learner 5; - // add-learner 4; - // - // So, when starting with an empty config, after applying 'outgoing' we have - // - // quorum=(1 2 4 6) - // - // From which we enter a joint state via 'incoming' - // - // quorum=(1 2 3)&&(1 2 4 6) learners=(5) learners_next=(4) - // - // as desired. - - // If there are outgoing voters, first add them one by one so that the - // (non-joint) config has them all. - i = 0; - addToConfChangeSingleArray(out, &i, &cs->votersOutgoing, SYNC_RAFT_Conf_AddNode); - assert(i == out->n); - - // We're done constructing the outgoing slice, now on to the incoming one - // (which will apply on top of the config created by the outgoing slice). - i = 0; - - // First, we'll remove all of the outgoing voters. - addToConfChangeSingleArray(in, &i, &cs->votersOutgoing, SYNC_RAFT_Conf_RemoveNode); - - // Then we'll add the incoming voters and learners. - addToConfChangeSingleArray(in, &i, &cs->voters, SYNC_RAFT_Conf_AddNode); - addToConfChangeSingleArray(in, &i, &cs->learners, SYNC_RAFT_Conf_AddLearnerNode); - addToConfChangeSingleArray(in, &i, &cs->learnersNext, SYNC_RAFT_Conf_AddLearnerNode); - assert(i == in->n); - - return 0; -} \ No newline at end of file diff --git a/source/libs/sync/test/raftTests.cpp b/source/libs/sync/test/raftTests.cpp deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 5c214b75a1..a2e82201bf 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -14,17 +14,18 @@ target_link_libraries( PUBLIC common ) if (${BUILD_WITH_UV_TRANS}) +if (${BUILD_WITH_UV}) target_include_directories( transport PUBLIC "${CMAKE_SOURCE_DIR}/contrib/libuv/include" ) - -#LINK_DIRECTORIES("${CMAKE_SOURCE_DIR}/debug/contrib/libuv") + target_link_libraries( transport PUBLIC uv_a ) add_definitions(-DUSE_UV) +endif(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV_TRANS}) if (${BUILD_TEST}) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 3f2aa1170e..846f2d5099 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -134,10 +134,12 @@ typedef struct { // int16_t numOfTry; // number of try for different servers // int8_t oldInUse; // server EP inUse passed by app // int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + + SRpcMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + char* ip; uint32_t port; // SEpSet* pSet; // for synchronous API @@ -211,6 +213,24 @@ typedef struct SConnBuffer { int left; } SConnBuffer; +typedef void (*AsyncCB)(uv_async_t* handle); + +typedef struct { + void* pThrd; + queue qmsg; + pthread_mutex_t mtx; // protect qmsg; +} SAsyncItem; + +typedef struct { + int index; + int nAsync; + uv_async_t* asyncs; +} SAsyncPool; + +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb); +void transDestroyAsyncPool(SAsyncPool* pool); +int transSendAsync(SAsyncPool* pool, queue* mq); + int transInitBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index a286482fc1..d8ef0462fb 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -813,8 +813,8 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { SRpcInfo *pRpc = pContext->pRpc; SEpSet * pEpSet = &pContext->epSet; - pConn = - rpcGetConnFromCache(pRpc->pCache, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, pContext->connType); + pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, + pContext->connType); if (pConn == NULL || pConn->user[0] == 0) { pConn = rpcOpenConn(pRpc, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, pContext->connType); } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 91f9a8ead2..5fc937bccd 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -35,6 +35,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + pRpc->parent = pInit->parent; return pRpc; } @@ -63,17 +64,41 @@ void rpcFreeCont(void* cont) { } free((char*)cont - TRANS_MSG_OVERHEAD); } -void* rpcReallocCont(void* ptr, int contLen) { return NULL; } +void* rpcReallocCont(void* ptr, int contLen) { + if (ptr == NULL) { + return rpcMallocCont(contLen); + } + char* st = (char*)ptr - TRANS_MSG_OVERHEAD; + int sz = contLen + TRANS_MSG_OVERHEAD; + st = realloc(st, sz); + if (st == NULL) { + return NULL; + } + return st + TRANS_MSG_OVERHEAD; +} + +void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { + SRpcMsg rpcMsg; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + + rpcMsg.contLen = sizeof(SEpSet); + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + if (rpcMsg.pCont == NULL) return; + + memcpy(rpcMsg.pCont, pEpSet, sizeof(SEpSet)); + + rpcMsg.code = TSDB_CODE_RPC_REDIRECT; + rpcMsg.handle = thandle; + + rpcSendResponse(&rpcMsg); +} -void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} -int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } -void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } void rpcCancelRequest(int64_t rid) { return; } int32_t rpcInit(void) { // impl later - return -1; + return 0; } void rpcCleanup(void) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 069ebaeb8a..e87a7a2ed4 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -42,9 +42,10 @@ typedef struct SCliMsg { } SCliMsg; typedef struct SCliThrdObj { - pthread_t thread; - uv_loop_t* loop; - uv_async_t* cliAsync; // + pthread_t thread; + uv_loop_t* loop; + // uv_async_t* cliAsync; // + SAsyncPool* asyncPool; uv_timer_t* timer; void* pool; // conn pool queue msg; @@ -123,8 +124,15 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; - tDebug("conn %p handle resp", conn); - (pRpc->cfp)(NULL, &rpcMsg, NULL); + tDebug("client conn %p %s received from %s:%d", conn, TMSG_INFO(pHead->msgType), pMsg->ctx->ip, pMsg->ctx->port); + if (pCtx->pSem == NULL) { + tTrace("client conn(sync) %p handle resp", conn); + (pRpc->cfp)(pRpc->parent, &rpcMsg, NULL); + } else { + tTrace("client conn(sync) %p handle resp", conn); + memcpy((char*)pCtx->pRsp, (char*)&rpcMsg, sizeof(rpcMsg)); + tsem_post(pCtx->pSem); + } conn->notifyCount += 1; // buf's mem alread translated to rpcMsg.pCont @@ -148,7 +156,7 @@ static void clientHandleExcept(SCliConn* pConn) { clientConnDestroy(pConn, true); return; } - tDebug("conn %p start to destroy", pConn); + tDebug("client conn %p start to destroy", pConn); SCliMsg* pMsg = pConn->data; destroyUserdata(&pMsg->msg); @@ -158,21 +166,27 @@ static void clientHandleExcept(SCliConn* pConn) { SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - // SRpcInfo* pRpc = pMsg->ctx->pRpc; - (pCtx->pTransInst->cfp)(NULL, &rpcMsg, NULL); - pConn->notifyCount += 1; + if (pCtx->pSem == NULL) { + // SRpcInfo* pRpc = pMsg->ctx->pRpc; + (pCtx->pTransInst->cfp)(pCtx->pTransInst->parent, &rpcMsg, NULL); + } else { + memcpy((char*)(pCtx->pRsp), (char*)(&rpcMsg), sizeof(rpcMsg)); + // SRpcMsg rpcMsg + tsem_post(pCtx->pSem); + } destroyCmsg(pMsg); pConn->data = NULL; // transDestroyConnCtx(pCtx); clientConnDestroy(pConn, true); + pConn->notifyCount += 1; } static void clientTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; SRpcInfo* pRpc = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; - tDebug("timeout, try to remove expire conn from conn pool"); + tDebug("client conn timeout, try to remove expire conn from conn pool"); SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); while (p != NULL) { @@ -241,7 +255,7 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { tstrncpy(key, ip, strlen(ip)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); - tDebug("conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); + tDebug("client conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; @@ -282,10 +296,10 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf pBuf->len += nread; if (clientReadComplete(pBuf)) { uv_read_stop((uv_stream_t*)conn->stream); - tDebug("conn %p read complete", conn); + tDebug("client conn %p read complete", conn); clientHandleResp(conn); } else { - tDebug("conn %p read partial packet, continue to read", conn); + tDebug("client conn %p read partial packet, continue to read", conn); } return; } @@ -297,7 +311,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf return; } if (nread < 0 || nread == UV_EOF) { - tError("conn %p read error: %s", conn, uv_err_name(nread)); + tError("client conn %p read error: %s", conn, uv_err_name(nread)); clientHandleExcept(conn); } // tDebug("Read error %s\n", uv_err_name(nread)); @@ -308,9 +322,9 @@ static void clientConnDestroy(SCliConn* conn, bool clear) { // conn->ref--; if (conn->ref == 0) { - tDebug("conn %p remove from conn pool", conn); + tDebug("client conn %p remove from conn pool", conn); QUEUE_REMOVE(&conn->conn); - tDebug("conn %p remove from conn pool successfully", conn); + tDebug("client conn %p remove from conn pool successfully", conn); if (clear) { uv_close((uv_handle_t*)conn->stream, clientDestroy); } @@ -322,7 +336,7 @@ static void clientDestroy(uv_handle_t* handle) { free(conn->stream); free(conn->writeReq); - tDebug("conn %p destroy successfully", conn); + tDebug("client conn %p destroy successfully", conn); free(conn); // clientConnDestroy(conn, false); @@ -331,7 +345,7 @@ static void clientDestroy(uv_handle_t* handle) { static void clientWriteCb(uv_write_t* req, int status) { SCliConn* pConn = req->data; if (status == 0) { - tDebug("conn %p data already was written out", pConn); + tDebug("client conn %p data already was written out", pConn); SCliMsg* pMsg = pConn->data; if (pMsg == NULL) { // handle @@ -339,7 +353,7 @@ static void clientWriteCb(uv_write_t* req, int status) { } destroyUserdata(&pMsg->msg); } else { - tError("conn %p failed to write: %s", pConn, uv_err_name(status)); + tError("client conn %p failed to write: %s", pConn, uv_err_name(status)); clientHandleExcept(pConn); return; } @@ -358,7 +372,7 @@ static void clientWrite(SCliConn* pConn) { pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("conn %p data write out, msgType : %d, len: %d", pConn, pHead->msgType, msgLen); + tDebug("conn %p %s is send to %s:%d", pConn, TMSG_INFO(pHead->msgType), pCliMsg->ctx->ip, pCliMsg->ctx->port); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static void clientConnCb(uv_connect_t* req, int status) { @@ -366,11 +380,11 @@ static void clientConnCb(uv_connect_t* req, int status) { SCliConn* pConn = req->data; if (status != 0) { // tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); - tError("conn %p failed to connect server: %s", pConn, uv_strerror(status)); + tError("client conn %p failed to connect server: %s", pConn, uv_strerror(status)); clientHandleExcept(pConn); return; } - tDebug("conn %p create", pConn); + tDebug("client conn %p create", pConn); assert(pConn->stream == req->handle); clientWrite(pConn); @@ -379,7 +393,7 @@ static void clientConnCb(uv_connect_t* req, int status) { static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { tDebug("thread %p start to quit", pThrd); destroyCmsg(pMsg); - uv_close((uv_handle_t*)pThrd->cliAsync, NULL); + // transDestroyAsyncPool(pThr) uv_close((uv_handle_t*)pThrd->cliAsync, NULL); uv_timer_stop(pThrd->timer); pThrd->quit = true; // uv__async_stop(pThrd->cliAsync); @@ -388,14 +402,14 @@ static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { uint64_t et = taosGetTimestampUs(); uint64_t el = et - pMsg->st; - tDebug("msg tran time cost: %" PRIu64 "", el); + tDebug("client msg tran time cost: %" PRIu64 "", el); et = taosGetTimestampUs(); STransConnCtx* pCtx = pMsg->ctx; SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { // impl later - tDebug("conn %p get from conn pool", conn); + tDebug("client get conn %p from pool", conn); conn->data = pMsg; conn->writeReq->data = conn; transDestroyBuffer(&conn->readBuf); @@ -431,14 +445,15 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { } } static void clientAsyncCb(uv_async_t* handle) { - SCliThrdObj* pThrd = handle->data; + SAsyncItem* item = handle->data; + SCliThrdObj* pThrd = item->pThrd; SCliMsg* pMsg = NULL; queue wq; // batch process to avoid to lock/unlock frequently - pthread_mutex_lock(&pThrd->msgMtx); - QUEUE_MOVE(&pThrd->msg, &wq); - pthread_mutex_unlock(&pThrd->msgMtx); + pthread_mutex_lock(&item->mtx); + QUEUE_MOVE(&item->qmsg, &wq); + pthread_mutex_unlock(&item->mtx); int count = 0; while (!QUEUE_IS_EMPTY(&wq)) { @@ -461,6 +476,7 @@ static void clientAsyncCb(uv_async_t* handle) { static void* clientThread(void* arg) { SCliThrdObj* pThrd = (SCliThrdObj*)arg; + setThreadName("trans-client-work"); uv_run(pThrd->loop, UV_RUN_DEFAULT); } @@ -501,6 +517,7 @@ static void destroyCmsg(SCliMsg* pMsg) { destroyUserdata(&pMsg->msg); free(pMsg); } + static SCliThrdObj* createThrdObj() { SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); @@ -509,9 +526,7 @@ static SCliThrdObj* createThrdObj() { pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); - pThrd->cliAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); - pThrd->cliAsync->data = pThrd; + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, clientAsyncCb); pThrd->timer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pThrd->timer); @@ -529,7 +544,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { uv_stop(pThrd->loop); pthread_join(pThrd->thread, NULL); pthread_mutex_destroy(&pThrd->msgMtx); - free(pThrd->cliAsync); + transDestroyAsyncPool(pThrd->asyncPool); + // free(pThrd->cliAsync); free(pThrd->timer); free(pThrd->loop); free(pThrd); @@ -547,11 +563,12 @@ static void clientSendQuit(SCliThrdObj* thrd) { SCliMsg* msg = calloc(1, sizeof(SCliMsg)); msg->ctx = NULL; // - pthread_mutex_lock(&thrd->msgMtx); - QUEUE_PUSH(&thrd->msg, &msg->q); - pthread_mutex_unlock(&thrd->msgMtx); + // pthread_mutex_lock(&thrd->msgMtx); + // QUEUE_PUSH(&thrd->msg, &msg->q); + // pthread_mutex_unlock(&thrd->msgMtx); - uv_async_send(thrd->cliAsync); + transSendAsync(thrd->asyncPool, &msg->q); + // uv_async_send(thrd->cliAsync); } void taosCloseClient(void* arg) { // impl later @@ -565,8 +582,8 @@ void taosCloseClient(void* arg) { } void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { // impl later - char* ip = (char*)(pEpSet->fqdn[pEpSet->inUse]); - uint32_t port = pEpSet->port[pEpSet->inUse]; + char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); + uint32_t port = pEpSet->eps[pEpSet->inUse].port; SRpcInfo* pRpc = (SRpcInfo*)shandle; @@ -596,10 +613,55 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; - pthread_mutex_lock(&thrd->msgMtx); - QUEUE_PUSH(&thrd->msg, &cliMsg->q); - pthread_mutex_unlock(&thrd->msgMtx); + // pthread_mutex_lock(&thrd->msgMtx); + // QUEUE_PUSH(&thrd->msg, &cliMsg->q); + // pthread_mutex_unlock(&thrd->msgMtx); - uv_async_send(thrd->cliAsync); + // int start = taosGetTimestampUs(); + transSendAsync(thrd->asyncPool, &(cliMsg->q)); + // uv_async_send(thrd->cliAsync); + // int end = taosGetTimestampUs() - start; + // tError("client sent to rpc, time cost: %d", (int)end); +} +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { + char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); + uint32_t port = pEpSet->eps[pEpSet->inUse].port; + + SRpcInfo* pRpc = (SRpcInfo*)shandle; + + STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); + pCtx->pTransInst = (SRpcInfo*)shandle; + pCtx->ahandle = pReq->ahandle; + pCtx->msgType = pReq->msgType; + pCtx->ip = strdup(ip); + pCtx->port = port; + pCtx->pSem = calloc(1, sizeof(tsem_t)); + pCtx->pRsp = pRsp; + tsem_init(pCtx->pSem, 0, 0); + + int64_t index = pRpc->index; + if (pRpc->index++ >= pRpc->numOfThreads) { + pRpc->index = 0; + } + SCliMsg* cliMsg = malloc(sizeof(SCliMsg)); + cliMsg->ctx = pCtx; + cliMsg->msg = *pReq; + cliMsg->st = taosGetTimestampUs(); + + SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; + + // pthread_mutex_lock(&thrd->msgMtx); + // QUEUE_PUSH(&thrd->msg, &cliMsg->q); + // pthread_mutex_unlock(&thrd->msgMtx); + + // int start = taosGetTimestampUs(); + transSendAsync(thrd->asyncPool, &(cliMsg->q)); + + tsem_t* pSem = pCtx->pSem; + tsem_wait(pSem); + tsem_destroy(pSem); + free(pSem); + + return; } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index ca39f85eb3..d0e504a0a1 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -245,4 +245,58 @@ int transDestroyBuffer(SConnBuffer* buf) { } transClearBuffer(buf); } + +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb) { + static int sz = 10; + + SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); + pool->index = 0; + pool->nAsync = sz; + pool->asyncs = calloc(1, sizeof(uv_async_t) * pool->nAsync); + + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + uv_async_init(loop, async, cb); + + SAsyncItem* item = calloc(1, sizeof(SAsyncItem)); + item->pThrd = arg; + QUEUE_INIT(&item->qmsg); + pthread_mutex_init(&item->mtx, NULL); + + async->data = item; + } + return pool; +} +void transDestroyAsyncPool(SAsyncPool* pool) { + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + + SAsyncItem* item = async->data; + pthread_mutex_destroy(&item->mtx); + free(item); + } + free(pool->asyncs); + free(pool); +} +int transSendAsync(SAsyncPool* pool, queue* q) { + int idx = pool->index; + idx = idx % pool->nAsync; + // no need mutex here + if (pool->index++ > pool->nAsync) { + pool->index = 0; + } + uv_async_t* async = &(pool->asyncs[idx]); + SAsyncItem* item = async->data; + + int64_t st = taosGetTimestampUs(); + pthread_mutex_lock(&item->mtx); + QUEUE_PUSH(&item->qmsg, q); + pthread_mutex_unlock(&item->mtx); + int64_t el = taosGetTimestampUs() - st; + if (el > 50) { + // tInfo("lock and unlock cost: %d", (int)el); + } + + return uv_async_send(async); +} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 475ef32b46..0389e3584e 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -22,7 +22,7 @@ typedef struct SSrvConn { uv_write_t* pWriter; uv_timer_t* pTimer; - uv_async_t* pWorkerAsync; + // uv_async_t* pWorkerAsync; queue queue; int ref; int persist; // persist connection or not @@ -33,6 +33,8 @@ typedef struct SSrvConn { void* hostThrd; void* pSrvMsg; + struct sockaddr_in addr; + // SRpcMsg sendMsg; // del later char secured; @@ -50,11 +52,12 @@ typedef struct SSrvMsg { } SSrvMsg; typedef struct SWorkThrdObj { - pthread_t thread; - uv_pipe_t* pipe; - int fd; - uv_loop_t* loop; - uv_async_t* workerAsync; // + pthread_t thread; + uv_pipe_t* pipe; + int fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; + // uv_async_t* workerAsync; // queue msg; pthread_mutex_t msgMtx; void* pTransInst; @@ -233,14 +236,6 @@ static void uvHandleReq(SSrvConn* pConn) { assert(transIsReq(pHead->msgType)); SRpcInfo* pRpc = (SRpcInfo*)p->shandle; - // auth here - // auth should not do in rpc thread - - // int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); - // if (code != 0) { - // terrno = code; - // return; - //} pHead->code = htonl(pHead->code); int32_t dlen = 0; @@ -263,6 +258,8 @@ static void uvHandleReq(SSrvConn* pConn) { transClearBuffer(&pConn->readBuf); pConn->ref++; + tDebug("%p %s received from %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), inet_ntoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port)); (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth @@ -275,12 +272,12 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { SConnBuffer* pBuf = &conn->readBuf; if (nread > 0) { pBuf->len += nread; - tDebug("conn %p read summroy, total read: %d, current read: %d", conn, pBuf->len, (int)nread); + tTrace("conn %p read summary, total read: %d, current read: %d", conn, pBuf->len, (int)nread); if (readComplete(pBuf)) { - tDebug("conn %p alread read complete packet", conn); + tTrace("conn %p alread read complete packet", conn); uvHandleReq(conn); } else { - tDebug("conn %p read partial packet, continue to read", conn); + tTrace("conn %p read partial packet, continue to read", conn); } return; } @@ -334,7 +331,8 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { // impl later; tDebug("conn %p prepare to send resp", smsg->pConn); - SRpcMsg* pMsg = &smsg->msg; + SRpcMsg* pMsg = &smsg->msg; + SSrvConn* pConn = smsg->pConn; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; @@ -347,6 +345,9 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { if (transCompressMsg(msg, len, NULL)) { // impl later } + tDebug("%p start to send %s to %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port)); + pHead->msgLen = htonl(len); wb->base = msg; wb->len = len; @@ -375,13 +376,15 @@ static void destroySmsg(SSrvMsg* smsg) { free(smsg); } void uvWorkerAsyncCb(uv_async_t* handle) { - SWorkThrdObj* pThrd = handle->data; + SAsyncItem* item = handle->data; + SWorkThrdObj* pThrd = item->pThrd; SSrvConn* conn = NULL; queue wq; // batch process to avoid to lock/unlock frequently - pthread_mutex_lock(&pThrd->msgMtx); - QUEUE_MOVE(&pThrd->msg, &wq); - pthread_mutex_unlock(&pThrd->msgMtx); + pthread_mutex_lock(&item->mtx); + QUEUE_MOVE(&item->qmsg, &wq); + pthread_mutex_unlock(&item->mtx); + // pthread_mutex_unlock(&mtx); while (!QUEUE_IS_EMPTY(&wq)) { queue* head = QUEUE_HEAD(&wq); @@ -469,7 +472,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer->data = pConn; pConn->hostThrd = pThrd; - pConn->pWorkerAsync = pThrd->workerAsync; // thread safty + // pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); @@ -484,7 +487,13 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); tDebug("conn %p created, fd: %d", pConn, fd); - uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + int addrlen = sizeof(pConn->addr); + if (0 != uv_tcp_getpeername(pConn->pTcp, (struct sockaddr*)&pConn->addr, &addrlen)) { + tError("failed to get peer name"); + destroyConn(pConn, true); + } else { + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + } } else { tDebug("failed to create new connection"); destroyConn(pConn, true); @@ -493,6 +502,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { void* acceptThread(void* arg) { // opt + setThreadName("trans-accept"); SServerObj* srv = (SServerObj*)arg; uv_run(srv->loop, UV_RUN_DEFAULT); } @@ -512,10 +522,7 @@ static bool addHandleToWorkloop(void* arg) { QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); - pThrd->workerAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); - pThrd->workerAsync->data = pThrd; - + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; } @@ -541,13 +548,14 @@ static bool addHandleToAcceptloop(void* arg) { tError("failed to bind: %s", uv_err_name(err)); return false; } - if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { + if ((err = uv_listen((uv_stream_t*)&srv->server, 512, uvOnAcceptCb)) != 0) { tError("failed to listen: %s", uv_err_name(err)); return false; } return true; } void* workerThread(void* arg) { + setThreadName("trans-worker"); SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; uv_run(pThrd->loop, UV_RUN_DEFAULT); } @@ -665,18 +673,21 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { } pthread_join(pThrd->thread, NULL); free(pThrd->loop); - free(pThrd->workerAsync); + transDestroyAsyncPool(pThrd->asyncPool); + + // free(pThrd->workerAsync); free(pThrd); } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); - pthread_mutex_lock(&pThrd->msgMtx); - QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - pthread_mutex_unlock(&pThrd->msgMtx); + // pthread_mutex_lock(&pThrd->msgMtx); + // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); + // pthread_mutex_unlock(&pThrd->msgMtx); tDebug("send quit msg to work thread"); - uv_async_send(pThrd->workerAsync); + transSendAsync(pThrd->asyncPool, &srvMsg->q); + // uv_async_send(pThrd->workerAsync); } void taosCloseServer(void* arg) { @@ -704,6 +715,9 @@ void taosCloseServer(void* arg) { } void rpcSendResponse(const SRpcMsg* pMsg) { + if (pMsg->handle == NULL) { + return; + } SSrvConn* pConn = pMsg->handle; SWorkThrdObj* pThrd = pConn->hostThrd; @@ -711,13 +725,25 @@ void rpcSendResponse(const SRpcMsg* pMsg) { srvMsg->pConn = pConn; srvMsg->msg = *pMsg; - pthread_mutex_lock(&pThrd->msgMtx); - QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - pthread_mutex_unlock(&pThrd->msgMtx); + // pthread_mutex_lock(&pThrd->msgMtx); + // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); + // pthread_mutex_unlock(&pThrd->msgMtx); - tDebug("conn %p start to send resp", pConn); + tTrace("conn %p start to send resp", pConn); + transSendAsync(pThrd->asyncPool, &srvMsg->q); + // uv_async_send(pThrd->workerAsync); +} - uv_async_send(pThrd->workerAsync); +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { + SSrvConn* pConn = thandle; + // struct sockaddr* pPeerName = &pConn->peername; + + struct sockaddr_in addr = pConn->addr; + pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); + pInfo->clientPort = ntohs(addr.sin_port); + + tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); + return 0; } #endif diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index 3d9c396336..3c9c40f46a 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -2,6 +2,7 @@ add_executable(transportTest "") add_executable(client "") add_executable(server "") add_executable(transUT "") +add_executable(syncClient "") target_sources(transUT PRIVATE @@ -20,6 +21,10 @@ target_sources (server PRIVATE "rserver.c" ) +target_sources (syncClient + PRIVATE + "syncClient.c" +) target_include_directories(transportTest PUBLIC @@ -67,7 +72,6 @@ target_include_directories(transUT "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) - target_link_libraries (server os util @@ -75,4 +79,17 @@ target_link_libraries (server gtest_main transport ) +target_include_directories(syncClient + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_link_libraries (syncClient + os + util + common + gtest_main + transport +) + diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 6767e704b5..4e29c02508 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include #include #include "os.h" @@ -32,7 +33,6 @@ typedef struct { pthread_t thread; void * pRpc; } SInfo; - static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, @@ -53,6 +53,12 @@ static void *sendRequest(void *param) { tDebug("thread:%d, start to send request", pInfo->index); + tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); + int u100 = 0; + int u500 = 0; + int u1000 = 0; + int u10000 = 0; + while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { pInfo->num++; rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); @@ -60,15 +66,28 @@ static void *sendRequest(void *param) { rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); + int64_t start = taosGetTimestampUs(); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); + int64_t end = taosGetTimestampUs() - start; + if (end <= 100) { + u100++; + } else if (end > 100 && end <= 500) { + u500++; + } else if (end > 500 && end < 1000) { + u1000++; + } else { + u10000++; + } + tDebug("recv response succefully"); // usleep(100000000); } + tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); tDebug("thread:%d, it is over", pInfo->index); tcount++; @@ -161,8 +180,8 @@ int main(int argc, char *argv[]) { tInfo("client is initialized"); tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); - // gettimeofday(&systemTime, NULL); - // startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + gettimeofday(&systemTime, NULL); + startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); @@ -184,13 +203,12 @@ int main(int argc, char *argv[]) { usleep(1); } while (tcount < appThreads); - // gettimeofday(&systemTime, NULL); - // endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - // float usedTime = (endTime - startTime) / 1000.0f; // mseconds + gettimeofday(&systemTime, NULL); + endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + float usedTime = (endTime - startTime) / 1000.0f; // mseconds - // tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - // tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, - // msgSize); + tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); int ch = getchar(); UNUSED(ch); diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c new file mode 100644 index 0000000000..c5d7f5664a --- /dev/null +++ b/source/libs/transport/test/syncClient.c @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include + +#include +#include "os.h" +#include "rpcLog.h" +#include "taoserror.h" +#include "tglobal.h" +#include "trpc.h" +#include "tutil.h" + +typedef struct { + int index; + SEpSet epSet; + int num; + int numOfReqs; + int msgSize; + tsem_t rspSem; + tsem_t * pOverSem; + pthread_t thread; + void * pRpc; +} SInfo; +static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SInfo *pInfo = (SInfo *)pMsg->ahandle; + tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + pMsg->code); + + if (pEpSet) pInfo->epSet = *pEpSet; + + rpcFreeCont(pMsg->pCont); + // tsem_post(&pInfo->rspSem); + tsem_post(&pInfo->rspSem); +} + +static int tcount = 0; + +static void *sendRequest(void *param) { + SInfo * pInfo = (SInfo *)param; + SRpcMsg rpcMsg = {0}; + + tDebug("thread:%d, start to send request", pInfo->index); + + tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); + int u100 = 0; + int u500 = 0; + int u1000 = 0; + int u10000 = 0; + SRpcMsg respMsg = {0}; + while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { + pInfo->num++; + rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); + rpcMsg.contLen = pInfo->msgSize; + rpcMsg.ahandle = pInfo; + rpcMsg.msgType = 1; + // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); + int64_t start = taosGetTimestampUs(); + rpcSendRecv(pInfo->pRpc, &pInfo->epSet, &rpcMsg, &respMsg); + // rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); + if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + // tsem_wait(&pInfo->rspSem); + // wtsem_wait(&pInfo->rspSem); + int64_t end = taosGetTimestampUs() - start; + if (end <= 100) { + u100++; + } else if (end > 100 && end <= 500) { + u500++; + } else if (end > 500 && end < 1000) { + u1000++; + } else { + u10000++; + } + + tDebug("recv response succefully"); + + // usleep(100000000); + } + + tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); + tDebug("thread:%d, it is over", pInfo->index); + tcount++; + + return NULL; +} + +int main(int argc, char *argv[]) { + SRpcInit rpcInit; + SEpSet epSet = {0}; + int msgSize = 128; + int numOfReqs = 0; + int appThreads = 1; + char serverIp[40] = "127.0.0.1"; + char secret[20] = "mypassword"; + struct timeval systemTime; + int64_t startTime, endTime; + pthread_attr_t thattr; + + // server info + epSet.inUse = 0; + addEpIntoEpSet(&epSet, serverIp, 7000); + addEpIntoEpSet(&epSet, "192.168.0.1", 7000); + + // client info + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "APP"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processResponse; + rpcInit.sessions = 100; + rpcInit.idleTime = 100; + rpcInit.user = "michael"; + rpcInit.secret = secret; + rpcInit.ckey = "key"; + rpcInit.spi = 1; + rpcInit.connType = TAOS_CONN_CLIENT; + + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { + epSet.eps[0].port = atoi(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) { + tstrncpy(epSet.eps[0].fqdn, argv[++i], sizeof(epSet.eps[0].fqdn)); + } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { + rpcInit.numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { + msgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { + rpcInit.sessions = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0 && i < argc - 1) { + numOfReqs = atoi(argv[++i]); + } else if (strcmp(argv[i], "-a") == 0 && i < argc - 1) { + appThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { + tsCompressMsgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-u") == 0 && i < argc - 1) { + rpcInit.user = argv[++i]; + } else if (strcmp(argv[i], "-k") == 0 && i < argc - 1) { + rpcInit.secret = argv[++i]; + } else if (strcmp(argv[i], "-spi") == 0 && i < argc - 1) { + rpcInit.spi = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { + rpcDebugFlag = atoi(argv[++i]); + } else { + printf("\nusage: %s [options] \n", argv[0]); + printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); + printf(" [-p port]: server port number, default is:%d\n", epSet.eps[0].port); + printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); + printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions); + printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); + printf(" [-a threads]: number of app threads, default is:%d\n", appThreads); + printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs); + printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); + printf(" [-u user]: user name for the connection, default is:%s\n", rpcInit.user); + printf(" [-k secret]: password for the connection, default is:%s\n", rpcInit.secret); + printf(" [-spi SPI]: security parameter index, default is:%d\n", rpcInit.spi); + printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); + printf(" [-h help]: print out this help\n\n"); + exit(0); + } + } + + taosInitLog("client.log", 100000, 10); + + void *pRpc = rpcOpen(&rpcInit); + if (pRpc == NULL) { + tError("failed to initialize RPC"); + return -1; + } + + tInfo("client is initialized"); + tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); + + gettimeofday(&systemTime, NULL); + startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + + SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); + + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + for (int i = 0; i < appThreads; ++i) { + pInfo->index = i; + pInfo->epSet = epSet; + pInfo->numOfReqs = numOfReqs; + pInfo->msgSize = msgSize; + tsem_init(&pInfo->rspSem, 0, 0); + pInfo->pRpc = pRpc; + pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + pInfo++; + } + + do { + usleep(1); + } while (tcount < appThreads); + + gettimeofday(&systemTime, NULL); + endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + float usedTime = (endTime - startTime) / 1000.0f; // mseconds + + tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); + + int ch = getchar(); + UNUSED(ch); + + taosCloseLog(); + + return 0; +} diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index 08c683590b..6f80ea42ac 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -15,6 +15,7 @@ #include #include #include +#include "tep.h" #include "trpc.h" using namespace std; @@ -50,6 +51,25 @@ class TransObj { trans = rpcOpen(&rpcInit); return trans != NULL ? true : false; } + + bool sendAndRecv() { + SEpSet epSet = {0}; + epSet.inUse = 0; + addEpIntoEpSet(&epSet, "192.168.1.1", 7000); + addEpIntoEpSet(&epSet, "192.168.0.1", 7000); + + if (trans == NULL) { + return false; + } + SRpcMsg rpcMsg = {0}, reqMsg = {0}; + reqMsg.pCont = rpcMallocCont(10); + reqMsg.contLen = 10; + reqMsg.ahandle = NULL; + rpcSendRecv(trans, &epSet, &reqMsg, &rpcMsg); + int code = rpcMsg.code; + std::cout << tstrerror(code) << std::endl; + return true; + } bool stop() { rpcClose(trans); trans = NULL; @@ -75,6 +95,7 @@ class TransEnv : public ::testing::Test { }; TEST_F(TransEnv, test_start_stop) { assert(tr->startCli()); + assert(tr->sendAndRecv()); assert(tr->stop()); assert(tr->startSrv()); diff --git a/source/nodes/src/nodesEqualFuncs.c b/source/nodes/src/nodesEqualFuncs.c index f2752fe7f9..41d1e5b05d 100644 --- a/source/nodes/src/nodesEqualFuncs.c +++ b/source/nodes/src/nodesEqualFuncs.c @@ -91,7 +91,7 @@ static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicCo static bool isNullConditionNodeEqual(const SIsNullCondNode* a, const SIsNullCondNode* b) { COMPARE_NODE_FIELD(pExpr); - COMPARE_SCALAR_FIELD(isNot); + COMPARE_SCALAR_FIELD(isNull); return true; } diff --git a/source/nodes/src/nodesTraverseFuncs.c b/source/nodes/src/nodesTraverseFuncs.c index ffe40073e6..2280d0b922 100644 --- a/source/nodes/src/nodesTraverseFuncs.c +++ b/source/nodes/src/nodesTraverseFuncs.c @@ -15,68 +15,130 @@ #include "nodes.h" -typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); +typedef enum ETraversalOrder { + TRAVERSAL_PREORDER = 1, + TRAVERSAL_POSTORDER +} ETraversalOrder; -bool nodesWalkNodeList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext) { +static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext); + +static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { + if (NULL == pNode) { + return true; + } + + if (TRAVERSAL_PREORDER == order && !walker(pNode, pContext)) { + return false; + } + + bool res = true; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_LIMIT: + // these node types with no subnodes + break; + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + res = walkNode(pOpNode->pLeft, order, walker, pContext); + if (res) { + res = walkNode(pOpNode->pRight, order, walker, pContext); + } + break; + } + case QUERY_NODE_LOGIC_CONDITION: + res = walkList(((SLogicConditionNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_IS_NULL_CONDITION: + res = walkNode(((SIsNullCondNode*)pNode)->pExpr, order, walker, pContext); + break; + case QUERY_NODE_FUNCTION: + res = walkList(((SFunctionNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + break; // todo + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; + res = walkNode(pJoinTableNode->pLeft, order, walker, pContext); + if (res) { + res = walkNode(pJoinTableNode->pRight, order, walker, pContext); + } + if (res) { + res = walkNode(pJoinTableNode->pOnCond, order, walker, pContext); + } + break; + } + case QUERY_NODE_GROUPING_SET: + res = walkList(((SGroupingSetNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_ORDER_BY_EXPR: + res = walkNode(((SOrderByExprNode*)pNode)->pExpr, order, walker, pContext); + break; + case QUERY_NODE_STATE_WINDOW: + res = walkNode(((SStateWindowNode*)pNode)->pCol, order, walker, pContext); + break; + case QUERY_NODE_SESSION_WINDOW: + res = walkNode(((SSessionWindowNode*)pNode)->pCol, order, walker, pContext); + break; + case QUERY_NODE_INTERVAL_WINDOW: { + SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; + res = walkNode(pInterval->pInterval, order, walker, pContext); + if (res) { + res = walkNode(pInterval->pOffset, order, walker, pContext); + } + if (res) { + res = walkNode(pInterval->pSliding, order, walker, pContext); + } + if (res) { + res = walkNode(pInterval->pFill, order, walker, pContext); + } + break; + } + case QUERY_NODE_NODE_LIST: + res = walkList(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext); + break; + case QUERY_NODE_FILL: + res = walkNode(((SFillNode*)pNode)->pValues, order, walker, pContext); + break; + case QUERY_NODE_RAW_EXPR: + res = walkNode(((SRawExprNode*)pNode)->pNode, order, walker, pContext); + break; + default: + break; + } + + if (res && TRAVERSAL_POSTORDER == order) { + res = walker(pNode, pContext); + } + + return res; +} + +static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { SNode* node; FOREACH(node, pNodeList) { - if (!nodesWalkNode(node, walker, pContext)) { + if (!walkNode(node, order, walker, pContext)) { return false; } } return true; } -bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) { - if (NULL == pNode) { - return true; - } +void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) { + (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); +} - if (!walker(pNode, pContext)) { - return false; - } +void nodesWalkList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext) { + (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); +} - switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: - case QUERY_NODE_VALUE: - case QUERY_NODE_LIMIT: - // these node types with no subnodes - return true; - case QUERY_NODE_OPERATOR: { - SOperatorNode* pOpNode = (SOperatorNode*)pNode; - if (!nodesWalkNode(pOpNode->pLeft, walker, pContext)) { - return false; - } - return nodesWalkNode(pOpNode->pRight, walker, pContext); - } - case QUERY_NODE_LOGIC_CONDITION: - return nodesWalkNodeList(((SLogicConditionNode*)pNode)->pParameterList, walker, pContext); - case QUERY_NODE_IS_NULL_CONDITION: - return nodesWalkNode(((SIsNullCondNode*)pNode)->pExpr, walker, pContext); - case QUERY_NODE_FUNCTION: - return nodesWalkNodeList(((SFunctionNode*)pNode)->pParameterList, walker, pContext); - case QUERY_NODE_REAL_TABLE: - case QUERY_NODE_TEMP_TABLE: - return true; // todo - case QUERY_NODE_JOIN_TABLE: { - SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; - if (!nodesWalkNode(pJoinTableNode->pLeft, walker, pContext)) { - return false; - } - if (!nodesWalkNode(pJoinTableNode->pRight, walker, pContext)) { - return false; - } - return nodesWalkNode(pJoinTableNode->pOnCond, walker, pContext); - } - case QUERY_NODE_GROUPING_SET: - return nodesWalkNodeList(((SGroupingSetNode*)pNode)->pParameterList, walker, pContext); - case QUERY_NODE_ORDER_BY_EXPR: - return nodesWalkNode(((SOrderByExprNode*)pNode)->pExpr, walker, pContext); - default: - break; - } +void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext) { + (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); +} - return false; +void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext) { + (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext); } bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext) { diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/nodes/src/nodesUtilFuncs.c index 535bd8e171..5acb9fdf7c 100644 --- a/source/nodes/src/nodesUtilFuncs.c +++ b/source/nodes/src/nodesUtilFuncs.c @@ -15,17 +15,13 @@ #include "nodes.h" #include "nodesShowStmts.h" - -bool nodesIsTimeorderQuery(const SNode* pQuery) { - -} - -bool nodesIsTimelineQuery(const SNode* pQuery) { - -} +#include "taoserror.h" static SNode* makeNode(ENodeType type, size_t size) { SNode* p = calloc(1, size); + if (NULL == p) { + return NULL; + } setNodeType(p, type); return p; } @@ -62,6 +58,12 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSessionWindowNode)); case QUERY_NODE_INTERVAL_WINDOW: return makeNode(type, sizeof(SIntervalWindowNode)); + case QUERY_NODE_NODE_LIST: + return makeNode(type, sizeof(SNodeListNode)); + case QUERY_NODE_FILL: + return makeNode(type, sizeof(SFillNode)); + case QUERY_NODE_RAW_EXPR: + return makeNode(type, sizeof(SRawExprNode)); case QUERY_NODE_SET_OPERATOR: return makeNode(type, sizeof(SSetOperator)); case QUERY_NODE_SELECT_STMT: @@ -74,10 +76,107 @@ SNode* nodesMakeNode(ENodeType type) { return NULL; } +static bool destroyNode(SNode* pNode, void* pContext) { + switch (nodeType(pNode)) { + case QUERY_NODE_VALUE: + tfree(((SValueNode*)pNode)->literal); + break; + default: + break; + } + tfree(pNode); +} + void nodesDestroyNode(SNode* pNode) { - + nodesWalkNodePostOrder(pNode, destroyNode, NULL); } -void nodesDestroyNodeList(SNodeList* pList) { - +SNodeList* nodesMakeList() { + SNodeList* p = calloc(1, sizeof(SNodeList)); + if (NULL == p) { + return NULL; + } + return p; } + +SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode) { + if (NULL == pList || NULL == pNode) { + return NULL; + } + SListCell* p = calloc(1, sizeof(SListCell)); + if (NULL == p) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pList; + } + p->pNode = pNode; + if (NULL == pList->pHead) { + pList->pHead = p; + } + if (NULL != pList->pTail) { + pList->pTail->pNext = p; + } + pList->pTail = p; + return pList; +} + +void nodesDestroyList(SNodeList* pList) { + SNode* node; + FOREACH(node, pList) { + nodesDestroyNode(node); + } + tfree(pList); +} + +bool nodesIsArithmeticOp(const SOperatorNode* pOp) { + switch (pOp->opType) { + case OP_TYPE_ADD: + case OP_TYPE_SUB: + case OP_TYPE_MULTI: + case OP_TYPE_DIV: + case OP_TYPE_MOD: + return true; + default: + break; + } + return false; +} + +bool nodesIsComparisonOp(const SOperatorNode* pOp) { + switch (pOp->opType) { + case OP_TYPE_GREATER_THAN: + case OP_TYPE_GREATER_EQUAL: + case OP_TYPE_LOWER_THAN: + case OP_TYPE_LOWER_EQUAL: + case OP_TYPE_EQUAL: + case OP_TYPE_NOT_EQUAL: + case OP_TYPE_IN: + case OP_TYPE_NOT_IN: + case OP_TYPE_LIKE: + case OP_TYPE_NOT_LIKE: + case OP_TYPE_MATCH: + case OP_TYPE_NMATCH: + return true; + default: + break; + } + return false; +} + +bool nodesIsJsonOp(const SOperatorNode* pOp) { + switch (pOp->opType) { + case OP_TYPE_JSON_GET_VALUE: + case OP_TYPE_JSON_CONTAINS: + return true; + default: + break; + } + return false; +} + +bool nodesIsTimeorderQuery(const SNode* pQuery) { + return false; +} + +bool nodesIsTimelineQuery(const SNode* pQuery) { + return false; +} \ No newline at end of file diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 09aeff5ff3..3e6f97c0c3 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -204,7 +204,12 @@ void* taosArrayGetLast(const SArray* pArray) { return TARRAY_GET_ELEM(pArray, pArray->size - 1); } -size_t taosArrayGetSize(const SArray* pArray) { return pArray->size; } +size_t taosArrayGetSize(const SArray* pArray) { + if (pArray == NULL) { + return 0; + } + return pArray->size; +} void taosArraySetSize(SArray* pArray, size_t size) { assert(size <= pArray->capacity); @@ -296,7 +301,7 @@ SArray* taosArrayDup(const SArray* pSrc) { } void taosArrayClear(SArray* pArray) { - assert( pArray != NULL ); + if (pArray == NULL) return; pArray->size = 0; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ee5bea0ab7..a451b1e708 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -228,17 +228,19 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist // mnode-stable TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "Stable not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "Stable confilct with topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STBS, "Too many stables") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB, "Invalid stable name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION, "Invalid stable options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "Stable options not changed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_ALTER_OPTION, "Invalid stable alter options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "Stable option unchanged") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ROW_BYTES, "Invalid row bytes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, "Tag already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREADY_EXIST, "Tag already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREAY_EXIST, "Column already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES, "Exceed max row bytes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") // mnode-func TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") @@ -418,6 +420,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INVALID_INPUT, "invalid catalog input TAOS_DEFINE_ERROR(TSDB_CODE_CTG_NOT_READY, "catalog is not ready") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_MEM_ERROR, "catalog memory error") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_SYS_ERROR, "catalog system error") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_DB_DROPPED, "Database is dropped") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_OUT_OF_SERVICE, "catalog is out of service") //scheduler TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status error") diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 383a00232a..ee0ade03b8 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -45,4 +45,10 @@ target_link_libraries(freelistTest os util gtest gtest_main) add_executable(encodeTest "encodeTest.cpp") target_link_libraries(encodeTest os util gtest gtest_main) - +# queueTest +add_executable(queue_test "queueTest.cpp") +target_link_libraries(queue_test os util gtest_main) +add_test( + NAME queue_test + COMMAND queue_test +) \ No newline at end of file diff --git a/source/util/test/queueTest.cpp b/source/util/test/queueTest.cpp new file mode 100644 index 0000000000..310ae4350e --- /dev/null +++ b/source/util/test/queueTest.cpp @@ -0,0 +1,29 @@ +/** + * @file queue.cpp + * @author slguan (slguan@taosdata.com) + * @brief UTIL module queue tests + * @version 1.0 + * @date 2022-01-27 + * + * @copyright Copyright (c) 2022 + * + */ + +#include + +#include "os.h" +#include "tqueue.h" + +class UtilTestQueue : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} +}; + +TEST_F(UtilTestQueue, 01_ReadQitemFromQsetByThread) { + EXPECT_EQ(0, 0); +} \ No newline at end of file diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin deleted file mode 160000 index 4a4d79099b..0000000000 --- a/src/connector/grafanaplugin +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4a4d79099b076b8ff12d5b4fdbcba54049a6866d diff --git a/tests/parallel_test/Jenkinsfile b/tests/parallel_test/Jenkinsfile new file mode 100644 index 0000000000..fc2b3562c1 --- /dev/null +++ b/tests/parallel_test/Jenkinsfile @@ -0,0 +1,200 @@ +import hudson.model.Result +import hudson.model.*; +import jenkins.model.CauseOfInterruption +node { +} + +def skipbuild=0 +def win_stop=0 + +def abortPreviousBuilds() { + def currentJobName = env.JOB_NAME + def currentBuildNumber = env.BUILD_NUMBER.toInteger() + def jobs = Jenkins.instance.getItemByFullName(currentJobName) + def builds = jobs.getBuilds() + + for (build in builds) { + if (!build.isBuilding()) { + continue; + } + + if (currentBuildNumber == build.getNumber().toInteger()) { + continue; + } + + build.doKill() //doTerm(),doKill(),doTerm() + } +} +// abort previous build +abortPreviousBuilds() +def abort_previous(){ + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) +} +def pre_test(){ + sh'hostname' + sh ''' + sudo rmtaos || echo "taosd has not installed" + ''' + sh ''' + killall -9 taosd ||echo "no taosd running" + killall -9 gdb || echo "no gdb running" + killall -9 python3.8 || echo "no python program running" + cd ${WKC} + ''' + script { + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WKC} + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } + else if(env.CHANGE_TARGET == '3.0'){ + sh ''' + cd ${WKC} + git checkout 3.0 + ''' + } + else{ + sh ''' + cd ${WKC} + git checkout develop + ''' + } + } + sh''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + export TZ=Asia/Harbin + date + rm -rf debug + mkdir debug + cd debug + cmake .. > /dev/null + make -j4> /dev/null + + ''' + return 1 +} + +pipeline { + agent none + options { skipDefaultCheckout() } + environment{ + WK = '/var/lib/jenkins/workspace/TDinternal' + WKC= '/var/lib/jenkins/workspace/TDengine' + } + stages { + stage('pre_build'){ + agent{label 'slave3_0'} + options { skipDefaultCheckout() } + when { + changeRequest() + } + steps { + script{ + abort_previous() + abortPreviousBuilds() + } + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh''' + cd ${WKC}/tests + ./test-all.sh b1fq + ''' + sh''' + cd ${WKC}/debug + ctest + ''' + } + } + } + } + post { + success { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", + body: """ + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${env.BRANCH_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${env.CHANGE_AUTHOR}
    • +
  • 提交信息：${env.CHANGE_TITLE}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • + +
    +
+
+ + """, + to: "${env.CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' FAIL", + body: """ + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${env.BRANCH_NAME}
    • +
  • 构建结果： Failure
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${env.CHANGE_AUTHOR}
    • +
  • 提交信息：${env.CHANGE_TITLE}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • + +
    +
+
+ + """, + to: "${env.CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + } +}