diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a23ebdecfd..4e6e47f4f9 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,4 +7,4 @@ FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT} # [Optional] Uncomment this section to install additional packages. # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends -RUN apt-get update && apt-get -y install tree vim tmux +RUN apt-get update && apt-get -y install tree vim tmux python3-pip diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 365da2d187..a6dd51b035 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -177,6 +177,7 @@ typedef struct SField { char name[TSDB_COL_NAME_LEN]; uint8_t type; int32_t bytes; + int8_t flags; } SField; typedef struct SRetention { @@ -302,13 +303,11 @@ typedef struct { int32_t ttl; int32_t numOfColumns; int32_t numOfTags; - int32_t numOfSmas; int32_t commentLen; int32_t ast1Len; int32_t ast2Len; SArray* pColumns; // array of SField SArray* pTags; // array of SField - SArray* pSmas; // array of SField char* comment; char* pAst1; char* pAst2; @@ -2384,9 +2383,9 @@ typedef struct { int32_t epoch; uint64_t reqId; int64_t consumerId; - int64_t blockingTime; + int64_t waitTime; int64_t currentOffset; -} SMqPollReqV2; +} SMqPollReq; typedef struct { int32_t vgId; @@ -2401,53 +2400,6 @@ typedef struct { SSchemaWrapper schema; } SMqSubTopicEp; -typedef struct { - SMqRspHead head; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t dataLen; - SArray* blockPos; // beginning pos for each SRetrieveTableRsp - void* blockData; // serialized batched SRetrieveTableRsp -} SMqPollRspV2; - -static FORCE_INLINE int32_t tEncodeSMqPollRspV2(void** buf, const SMqPollRspV2* pRsp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); - tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); - tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); - tlen += taosEncodeFixedI32(buf, pRsp->dataLen); - if (pRsp->dataLen != 0) { - int32_t sz = taosArrayGetSize(pRsp->blockPos); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - int32_t blockPos = *(int32_t*)taosArrayGet(pRsp->blockPos, i); - tlen += taosEncodeFixedI32(buf, blockPos); - } - tlen += taosEncodeBinary(buf, pRsp->blockData, pRsp->dataLen); - } - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); - buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); - buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); - buf = taosDecodeFixedI32(buf, &pRsp->dataLen); - if (pRsp->dataLen != 0) { - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pRsp->blockPos = taosArrayInit(sz, sizeof(int32_t)); - for (int32_t i = 0; i < sz; i++) { - int32_t blockPos; - buf = taosDecodeFixedI32(buf, &blockPos); - taosArrayPush(pRsp->blockPos, &blockPos); - } - buf = taosDecodeBinary(buf, &pRsp->blockData, pRsp->dataLen); - } - return (void*)buf; -} - typedef struct { SMqRspHead head; int64_t reqOffset; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 56e25d49c1..126a2ccf99 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include "querynodes.h" #include "function.h" +#include "querynodes.h" typedef enum EFunctionType { // aggregate function @@ -123,10 +123,10 @@ struct SCatalog; typedef struct SFmGetFuncInfoParam { struct SCatalog* pCtg; - void *pRpc; - const SEpSet* pMgmtEps; - char* pErrBuf; - int32_t errBufLen; + void* pRpc; + const SEpSet* pMgmtEps; + char* pErrBuf; + int32_t errBufLen; } SFmGetFuncInfoParam; int32_t fmFuncMgtInit(); @@ -143,6 +143,7 @@ bool fmIsDatetimeFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); bool fmIsPseudoColumnFunc(int32_t funcId); +bool fmIsScanPseudoColumnFunc(int32_t funcId); bool fmIsWindowPseudoColumnFunc(int32_t funcId); bool fmIsWindowClauseFunc(int32_t funcId); bool fmIsSpecialDataRequiredFunc(int32_t funcId); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 829770ed1b..d4197cd399 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -20,50 +20,46 @@ extern "C" { #endif -#include "querynodes.h" #include "query.h" +#include "querynodes.h" #include "tname.h" typedef struct SLogicNode { - ENodeType type; - SNodeList* pTargets; // SColumnNode - SNode* pConditions; - SNodeList* pChildren; + ENodeType type; + SNodeList* pTargets; // SColumnNode + SNode* pConditions; + SNodeList* pChildren; struct SLogicNode* pParent; - int32_t optimizedFlag; + int32_t optimizedFlag; } SLogicNode; -typedef enum EScanType { - SCAN_TYPE_TAG, - SCAN_TYPE_TABLE, - SCAN_TYPE_SYSTEM_TABLE, - SCAN_TYPE_STREAM -} EScanType; +typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType; typedef struct SScanLogicNode { - SLogicNode node; - SNodeList* pScanCols; + SLogicNode node; + SNodeList* pScanCols; + SNodeList* pScanPseudoCols; struct STableMeta* pMeta; - SVgroupsInfo* pVgroupList; - EScanType scanType; - uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count - STimeWindow scanRange; - SName tableName; - bool showRewrite; - double ratio; - SNodeList* pDynamicScanFuncs; - int32_t dataRequired; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; + SVgroupsInfo* pVgroupList; + EScanType scanType; + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count + STimeWindow scanRange; + SName tableName; + bool showRewrite; + double ratio; + SNodeList* pDynamicScanFuncs; + int32_t dataRequired; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; } SScanLogicNode; typedef struct SJoinLogicNode { SLogicNode node; - EJoinType joinType; - SNode* pOnConditions; + EJoinType joinType; + SNode* pOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -75,47 +71,43 @@ typedef struct SAggLogicNode { typedef struct SProjectLogicNode { SLogicNode node; SNodeList* pProjections; - char stmtName[TSDB_TABLE_NAME_LEN]; - int64_t limit; - int64_t offset; - int64_t slimit; - int64_t soffset; + char stmtName[TSDB_TABLE_NAME_LEN]; + int64_t limit; + int64_t offset; + int64_t slimit; + int64_t soffset; } SProjectLogicNode; typedef struct SVnodeModifLogicNode { - SLogicNode node; - int32_t msgType; - SArray* pDataBlocks; + SLogicNode node; + int32_t msgType; + SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; } SVnodeModifLogicNode; typedef struct SExchangeLogicNode { SLogicNode node; - int32_t srcGroupId; - uint8_t precision; + int32_t srcGroupId; + uint8_t precision; } SExchangeLogicNode; -typedef enum EWindowType { - WINDOW_TYPE_INTERVAL = 1, - WINDOW_TYPE_SESSION, - WINDOW_TYPE_STATE -} EWindowType; +typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; typedef struct SWindowLogicNode { - SLogicNode node; + SLogicNode node; EWindowType winType; - SNodeList* pFuncs; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; - SFillNode* pFill; - int64_t sessionGap; - SNode* pTspk; - SNode* pStateExpr; - int8_t triggerType; - int64_t watermark; + SNodeList* pFuncs; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; + SFillNode* pFill; + int64_t sessionGap; + SNode* pTspk; + SNode* pStateExpr; + int8_t triggerType; + int64_t watermark; } SWindowLogicNode; typedef struct SSortLogicNode { @@ -137,59 +129,60 @@ typedef enum ESubplanType { typedef struct SSubplanId { uint64_t queryId; - int32_t groupId; - int32_t subplanId; + int32_t groupId; + int32_t subplanId; } SSubplanId; typedef struct SLogicSubplan { - ENodeType type; - SSubplanId id; - SNodeList* pChildren; - SNodeList* pParents; - SLogicNode* pNode; - ESubplanType subplanType; + ENodeType type; + SSubplanId id; + SNodeList* pChildren; + SNodeList* pParents; + SLogicNode* pNode; + ESubplanType subplanType; SVgroupsInfo* pVgroupList; - int32_t level; - int32_t splitFlag; + int32_t level; + int32_t splitFlag; } SLogicSubplan; typedef struct SQueryLogicPlan { - ENodeType type; + ENodeType type; SNodeList* pTopSubplans; } SQueryLogicPlan; typedef struct SSlotDescNode { ENodeType type; - int16_t slotId; + int16_t slotId; SDataType dataType; - bool reserve; - bool output; - bool tag; + bool reserve; + bool output; + bool tag; } SSlotDescNode; typedef struct SDataBlockDescNode { - ENodeType type; - int16_t dataBlockId; + ENodeType type; + int16_t dataBlockId; SNodeList* pSlots; - int32_t totalRowSize; - int32_t outputRowSize; - uint8_t precision; + int32_t totalRowSize; + int32_t outputRowSize; + uint8_t precision; } SDataBlockDescNode; typedef struct SPhysiNode { - ENodeType type; + ENodeType type; SDataBlockDescNode* pOutputDataBlockDesc; - SNode* pConditions; - SNodeList* pChildren; - struct SPhysiNode* pParent; + SNode* pConditions; + SNodeList* pChildren; + struct SPhysiNode* pParent; } SPhysiNode; typedef struct SScanPhysiNode { SPhysiNode node; SNodeList* pScanCols; - uint64_t uid; // unique id of the table - int8_t tableType; - SName tableName; + SNodeList* pScanPseudoCols; + uint64_t uid; // unique id of the table + int8_t tableType; + SName tableName; } SScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode; @@ -197,23 +190,23 @@ typedef SScanPhysiNode SStreamScanPhysiNode; typedef struct SSystemTableScanPhysiNode { SScanPhysiNode scan; - SEpSet mgmtEpSet; - bool showRewrite; - int32_t accountId; + SEpSet mgmtEpSet; + bool showRewrite; + int32_t accountId; } SSystemTableScanPhysiNode; typedef struct STableScanPhysiNode { SScanPhysiNode scan; - uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count - STimeWindow scanRange; - double ratio; - int32_t dataRequired; - SNodeList* pDynamicScanFuncs; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count + STimeWindow scanRange; + double ratio; + int32_t dataRequired; + SNodeList* pDynamicScanFuncs; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; @@ -221,89 +214,89 @@ typedef STableScanPhysiNode STableSeqScanPhysiNode; typedef struct SProjectPhysiNode { SPhysiNode node; SNodeList* pProjections; - int64_t limit; - int64_t offset; - int64_t slimit; - int64_t soffset; + int64_t limit; + int64_t offset; + int64_t slimit; + int64_t soffset; } SProjectPhysiNode; typedef struct SJoinPhysiNode { SPhysiNode node; - EJoinType joinType; - SNode* pOnConditions; // in or out tuple ? + EJoinType joinType; + SNode* pOnConditions; // in or out tuple ? SNodeList* pTargets; } SJoinPhysiNode; typedef struct SAggPhysiNode { SPhysiNode node; - SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function + SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function SNodeList* pGroupKeys; SNodeList* pAggFuncs; } SAggPhysiNode; typedef struct SDownstreamSourceNode { - ENodeType type; + ENodeType type; SQueryNodeAddr addr; - uint64_t taskId; - uint64_t schedId; + uint64_t taskId; + uint64_t schedId; } SDownstreamSourceNode; typedef struct SExchangePhysiNode { SPhysiNode node; - int32_t srcGroupId; // group id of datasource suplans + int32_t srcGroupId; // group id of datasource suplans SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; typedef struct SWinodwPhysiNode { SPhysiNode node; - SNodeList* pExprs; // these are expression list of parameter expression of function + SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; - SNode* pTspk; // timestamp primary key - int8_t triggerType; - int64_t watermark; + SNode* pTspk; // timestamp primary key + int8_t triggerType; + int64_t watermark; } SWinodwPhysiNode; typedef struct SIntervalPhysiNode { SWinodwPhysiNode window; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; - SFillNode* pFill; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; + SFillNode* pFill; } SIntervalPhysiNode; typedef struct SMultiTableIntervalPhysiNode { SIntervalPhysiNode interval; - SNodeList* pPartitionKeys; + SNodeList* pPartitionKeys; } SMultiTableIntervalPhysiNode; typedef struct SSessionWinodwPhysiNode { SWinodwPhysiNode window; - int64_t gap; + int64_t gap; } SSessionWinodwPhysiNode; typedef struct SStateWinodwPhysiNode { SWinodwPhysiNode window; - SNode* pStateKey; + SNode* pStateKey; } SStateWinodwPhysiNode; typedef struct SSortPhysiNode { SPhysiNode node; - SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function - SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode + SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function + SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode SNodeList* pTargets; } SSortPhysiNode; typedef struct SPartitionPhysiNode { SPhysiNode node; - SNodeList* pExprs; // these are expression list of partition_by_clause + SNodeList* pExprs; // these are expression list of partition_by_clause SNodeList* pPartitionKeys; SNodeList* pTargets; } SPartitionPhysiNode; typedef struct SDataSinkNode { - ENodeType type; + ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; } SDataSinkNode; @@ -313,45 +306,41 @@ typedef struct SDataDispatcherNode { typedef struct SDataInserterNode { SDataSinkNode sink; - int32_t numOfTables; - uint32_t size; - char *pData; + int32_t numOfTables; + uint32_t size; + char* pData; } SDataInserterNode; typedef struct SSubplan { - ENodeType type; - SSubplanId id; // unique id of the subplan - ESubplanType subplanType; - int32_t msgType; // message type for subplan, used to denote the send message type to vnode. - int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. - char dbFName[TSDB_DB_FNAME_LEN]; - SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node - SQueryNodeStat execNodeStat; // only for scan subplan - SNodeList* pChildren; // the datasource subplan,from which to fetch the result - SNodeList* pParents; // the data destination subplan, get data from current subplan - SPhysiNode* pNode; // physical plan of current subplan - SDataSinkNode* pDataSink; // data of the subplan flow into the datasink + ENodeType type; + SSubplanId id; // unique id of the subplan + ESubplanType subplanType; + int32_t msgType; // message type for subplan, used to denote the send message type to vnode. + int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. + char dbFName[TSDB_DB_FNAME_LEN]; + SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node + SQueryNodeStat execNodeStat; // only for scan subplan + SNodeList* pChildren; // the datasource subplan,from which to fetch the result + SNodeList* pParents; // the data destination subplan, get data from current subplan + SPhysiNode* pNode; // physical plan of current subplan + SDataSinkNode* pDataSink; // data of the subplan flow into the datasink } SSubplan; -typedef enum EExplainMode { - EXPLAIN_MODE_DISABLE = 1, - EXPLAIN_MODE_STATIC, - EXPLAIN_MODE_ANALYZE -} EExplainMode; +typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode; typedef struct SExplainInfo { EExplainMode mode; - bool verbose; - double ratio; + bool verbose; + double ratio; } SExplainInfo; typedef struct SQueryPlan { - ENodeType type; - uint64_t queryId; - int32_t numOfSubplans; - SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. + ENodeType type; + uint64_t queryId; + int32_t numOfSubplans; + SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SExplainInfo explainInfo; - SNodeList* pPlaceholderValues; + SNodeList* pPlaceholderValues; } SQueryPlan; void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 1605fe8ac8..238c4c8538 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -293,7 +293,10 @@ typedef struct SExplainStmt { void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext); -int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols); +typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType; + +int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, + SNodeList** pCols); typedef bool (*FFuncClassifier)(int32_t funcId); int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 7b9f68103d..0e7d486eab 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -59,9 +59,13 @@ typedef struct { void * pNode; } SNodeMsg; -typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf); typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); -typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *); +/// +// // SRpcMsg code +// REDIERE, +// NOT READY, EpSet +typedef bool (*RpcRfp)(int32_t code); typedef struct SRpcInit { uint16_t localPort; // local port diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 7b0d70b769..93a950466b 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -72,8 +72,6 @@ extern "C" { #define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_MAGIC 0xFAFBFCFDULL -#define WAL_CUR_FAILED 1 - #pragma pack(push, 1) typedef enum { TAOS_WAL_NOLOG = 0, diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a00fd4714e..7c873acadb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -246,6 +246,12 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t pResInfo->numOfCols = numOfCols; // TODO handle memory leak + if (pResInfo->fields != NULL) { + taosMemoryFree(pResInfo->fields); + } + if (pResInfo->userFields != NULL) { + taosMemoryFree(pResInfo->userFields); + } pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index e0fb46e122..d0f6a296d9 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -16,7 +16,6 @@ #include "clientInt.h" #include "clientLog.h" #include "parser.h" -#include "planner.h" #include "tdatablock.h" #include "tdef.h" #include "tglobal.h" @@ -68,7 +67,7 @@ struct tmq_conf_t { char* user; char* pass; char* db; - tmq_commit_cb* commit_cb; + tmq_commit_cb* commitCb; }; struct tmq_t { @@ -115,8 +114,8 @@ enum { enum { TMQ_CONSUMER_STATUS__INIT = 0, - TMQ_CONSUMER_STATUS__SUBSCRIBED, TMQ_CONSUMER_STATUS__READY, + TMQ_CONSUMER_STATUS__NO_TOPIC, }; enum { @@ -175,7 +174,6 @@ typedef struct { int32_t epoch; int32_t vgId; tsem_t rspSem; - int32_t sync; } SMqPollCbParam; typedef struct { @@ -300,6 +298,7 @@ void tmqAssignDelayedHbTask(void* param, void* tmrId) { int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t)); *pTaskType = TMQ_DELAYED_TASK__HB; taosWriteQitem(tmq->delayedTask, pTaskType); + tsem_post(&tmq->rspSem); } void tmqAssignDelayedCommitTask(void* param, void* tmrId) { @@ -307,6 +306,7 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t)); *pTaskType = TMQ_DELAYED_TASK__COMMIT; taosWriteQitem(tmq->delayedTask, pTaskType); + tsem_post(&tmq->rspSem); } void tmqAssignDelayedReportTask(void* param, void* tmrId) { @@ -314,6 +314,7 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) { int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t)); *pTaskType = TMQ_DELAYED_TASK__REPORT; taosWriteQitem(tmq->delayedTask, pTaskType); + tsem_post(&tmq->rspSem); } int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { @@ -364,7 +365,6 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; pParam->rspErr = code; tmq_t* tmq = pParam->tmq; - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__SUBSCRIBED); tsem_post(&pParam->rspSem); return 0; } @@ -385,14 +385,16 @@ tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { } for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* topic = taosArrayGetP(tmq->clientTopics, i); - tmq_list_append(*topics, strdup(topic->topicName)); + tmq_list_append(*topics, topic->topicName); } return TMQ_RESP_ERR__SUCCESS; } tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) { - tmq_list_t* lst = tmq_list_new(); - return tmq_subscribe(tmq, lst); + tmq_list_t* lst = tmq_list_new(); + tmq_resp_err_t rsp = tmq_subscribe(tmq, lst); + tmq_list_destroy(lst); + return rsp; } #if 0 @@ -475,7 +477,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { strcpy(pTmq->groupId, conf->groupId); pTmq->autoCommit = conf->autoCommit; pTmq->autoCommitInterval = conf->autoCommitInterval; - pTmq->commit_cb = conf->commit_cb; + pTmq->commit_cb = conf->commitCb; pTmq->resetOffsetCfg = conf->resetOffset; // assign consumerId @@ -655,6 +657,9 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + // avoid double free if msg is sent + buf = NULL; + tsem_wait(¶m.rspSem); tsem_destroy(¶m.rspSem); @@ -686,7 +691,7 @@ FAIL: void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { // - conf->commit_cb = cb; + conf->commitCb = cb; } TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) { @@ -798,7 +803,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { // do not write into queue since updating epoch reset tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); - /*tsem_post(&tmq->rspSem);*/ + tsem_post(&tmq->rspSem); return 0; } @@ -806,25 +811,6 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); } -#if 0 - if (pParam->sync == 1) { - /**pParam->msg = taosMemoryMalloc(sizeof(tmq_message_t));*/ - *pParam->msg = taosAllocateQitem(sizeof(tmq_message_t)); - if (*pParam->msg) { - memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &((*pParam->msg)->consumeRsp)); - if ((*pParam->msg)->consumeRsp.numOfTopics != 0) { - pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset; - } - taosWriteQitem(tmq->mqueue, *pParam->msg); - tsem_post(&pParam->rspSem); - return 0; - } - tsem_post(&pParam->rspSem); - return -1; - } -#endif - SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper)); if (pRspWrapper == NULL) { tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch); @@ -843,14 +829,14 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset); taosWriteQitem(tmq->mqueue, pRspWrapper); - /*tsem_post(&tmq->rspSem);*/ + tsem_post(&tmq->rspSem); return 0; CREATE_MSG_FAIL: if (pParam->epoch == tmq->epoch) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } - /*tsem_post(&tmq->rspSem);*/ + tsem_post(&tmq->rspSem); return -1; } @@ -927,6 +913,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); taosHashCleanup(pHash); tmq->clientTopics = newTopics; + + if (taosArrayGetSize(tmq->clientTopics) == 0) + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC); + else + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); + atomic_store_32(&tmq->epoch, epoch); return set; } @@ -955,9 +947,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ - if (tmqUpdateEp(tmq, head->epoch, &rsp)) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); - } + tmqUpdateEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper)); @@ -972,7 +962,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg); taosWriteQitem(tmq->mqueue, pWrapper); - /*tsem_post(&tmq->rspSem);*/ + tsem_post(&tmq->rspSem); taosMemoryFree(pParam); } @@ -1076,7 +1066,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { return TMQ_RESP_ERR__FAIL; } -SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { +SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t waitTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { int64_t reqOffset; if (pVg->currentOffset >= 0) { reqOffset = pVg->currentOffset; @@ -1088,7 +1078,7 @@ SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClient reqOffset = tmq->resetOffsetCfg; } - SMqPollReqV2* pReq = taosMemoryMalloc(sizeof(SMqPollReqV2)); + SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq)); if (pReq == NULL) { return NULL; } @@ -1101,14 +1091,14 @@ SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClient pReq->subKey[tlen] = TMQ_SEPARATOR; strcpy(pReq->subKey + tlen + 1, pTopic->topicName); - pReq->blockingTime = blockingTime; + pReq->waitTime = waitTime; pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; pReq->currentOffset = reqOffset; pReq->reqId = generateRequestId(); pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqPollReqV2)); + pReq->head.contLen = htonl(sizeof(SMqPollReq)); return pReq; } @@ -1130,7 +1120,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { return pRspObj; } -int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { +int32_t tmqPollImpl(tmq_t* tmq, int64_t waitTime) { /*printf("call poll\n");*/ for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); @@ -1151,17 +1141,17 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { #endif } atomic_store_32(&pVg->vgSkipCnt, 0); - SMqPollReqV2* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, waitTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - /*tsem_post(&tmq->rspSem);*/ + tsem_post(&tmq->rspSem); return -1; } SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { taosMemoryFree(pReq); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - /*tsem_post(&tmq->rspSem);*/ + tsem_post(&tmq->rspSem); return -1; } pParam->tmq = tmq; @@ -1169,20 +1159,19 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { pParam->pTopic = pTopic; pParam->vgId = pVg->vgId; pParam->epoch = tmq->epoch; - pParam->sync = 0; SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(pReq); taosMemoryFree(pParam); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - /*tsem_post(&tmq->rspSem);*/ + tsem_post(&tmq->rspSem); return -1; } sendInfo->msgInfo = (SDataBuf){ .pData = pReq, - .len = sizeof(SMqPollReqV2), + .len = sizeof(SMqPollReq), .handle = NULL, }; sendInfo->requestId = pReq->reqId; @@ -1222,7 +1211,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) return 0; } -SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { +SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t waitTime, bool pollIfReset) { while (1) { SMqRspWrapper* rspWrapper = NULL; taosGetQitem(tmq->qall, (void**)&rspWrapper); @@ -1261,37 +1250,41 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { taosFreeQitem(rspWrapper); if (pollIfReset && reset) { tscDebug("consumer %ld reset and repoll", tmq->consumerId); - tmqPollImpl(tmq, blockingTime); + tmqPollImpl(tmq, waitTime); } } } } -TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { +TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) { SMqRspObj* rspObj; int64_t startTime = taosGetTimestampMs(); - rspObj = tmqHandleAllRsp(tmq, blocking_time, false); + rspObj = tmqHandleAllRsp(tmq, wait_time, false); if (rspObj) { return (TAOS_RES*)rspObj; } + if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { + return NULL; + } + while (1) { tmqHandleAllDelayedTask(tmq); - tmqPollImpl(tmq, blocking_time); + tmqPollImpl(tmq, wait_time); - /*tsem_wait(&tmq->rspSem);*/ - - rspObj = tmqHandleAllRsp(tmq, blocking_time, false); + rspObj = tmqHandleAllRsp(tmq, wait_time, false); if (rspObj) { return (TAOS_RES*)rspObj; } - if (blocking_time != 0) { + if (wait_time != 0) { int64_t endTime = taosGetTimestampMs(); - if (endTime - startTime > blocking_time) { + int64_t leftTime = endTime - startTime; + if (leftTime > wait_time) { tscDebug("consumer %ld (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch); return NULL; } + tsem_timewait(&tmq->rspSem, leftTime * 1000); } } } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 5feec55799..dc52afb382 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -517,7 +517,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1; - if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1; if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1; if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1; if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1; @@ -527,6 +526,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeI8(&encoder, pField->type) < 0) return -1; if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1; + if (tEncodeI8(&encoder, pField->flags) < 0) return -1; } for (int32_t i = 0; i < pReq->numOfTags; ++i) { @@ -534,13 +534,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeI8(&encoder, pField->type) < 0) return -1; if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1; - } - - for (int32_t i = 0; i < pReq->numOfSmas; ++i) { - SField *pField = taosArrayGet(pReq->pSmas, i); - if (tEncodeI8(&encoder, pField->type) < 0) return -1; - if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; - if (tEncodeCStr(&encoder, pField->name) < 0) return -1; + if (tEncodeI8(&encoder, pField->flags) < 0) return -1; } if (pReq->commentLen > 0) { @@ -571,15 +565,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1; if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1; if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1; if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1; pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField)); pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField)); - pReq->pSmas = taosArrayInit(pReq->numOfSmas, sizeof(SField)); - if (pReq->pColumns == NULL || pReq->pTags == NULL || pReq->pSmas == NULL) { + if (pReq->pColumns == NULL || pReq->pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -589,6 +581,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeI8(&decoder, &field.type) < 0) return -1; if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; + if (tDecodeI8(&decoder, &field.flags) < 0) return -1; if (taosArrayPush(pReq->pColumns, &field) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -600,23 +593,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeI8(&decoder, &field.type) < 0) return -1; if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; + if (tDecodeI8(&decoder, &field.flags) < 0) return -1; if (taosArrayPush(pReq->pTags, &field) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } } - for (int32_t i = 0; i < pReq->numOfSmas; ++i) { - SField field = {0}; - if (tDecodeI8(&decoder, &field.type) < 0) return -1; - if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; - if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; - if (taosArrayPush(pReq->pSmas, &field) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - } - if (pReq->commentLen > 0) { pReq->comment = taosMemoryMalloc(pReq->commentLen); if (pReq->comment == NULL) return -1; @@ -644,13 +627,11 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { taosArrayDestroy(pReq->pColumns); taosArrayDestroy(pReq->pTags); - taosArrayDestroy(pReq->pSmas); taosMemoryFreeClear(pReq->comment); taosMemoryFreeClear(pReq->pAst1); taosMemoryFreeClear(pReq->pAst2); pReq->pColumns = NULL; pReq->pTags = NULL; - pReq->pSmas = NULL; } int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index eb721ba8a8..65cc4353e8 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -358,13 +358,11 @@ typedef struct { int32_t ttl; int32_t numOfColumns; int32_t numOfTags; - int32_t numOfSmas; int32_t commentLen; int32_t ast1Len; int32_t ast2Len; SSchema* pColumns; SSchema* pTags; - SSchema* pSmas; char* comment; char* pAst1; char* pAst2; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index b9ec1905e6..5083104039 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -40,12 +40,12 @@ extern "C" { #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); -typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct SQWorkerMgmt SQHandle; @@ -84,32 +84,32 @@ typedef struct { int64_t timeseriesAllowed; } SGrantInfo; -struct SMnode { - int32_t selfId; - int64_t clusterId; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; - tmr_h timer; - tmr_h transTimer; - tmr_h mqTimer; - tmr_h telemTimer; - char *path; - int64_t checkTime; - SSdb *pSdb; - SMgmtWrapper *pWrapper; - SArray *pSteps; - SQHandle *pQuery; - SShowMgmt showMgmt; - SProfileMgmt profileMgmt; - STelemMgmt telemMgmt; - SSyncMgmt syncMgmt; - SHashObj *infosMeta; - SHashObj *perfsMeta; - SGrantInfo grant; - MndMsgFp msgFp[TDMT_MAX]; - SMsgCb msgCb; -}; +typedef struct SMnode { + int32_t selfId; + int64_t clusterId; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; + tmr_h timer; + tmr_h transTimer; + tmr_h mqTimer; + tmr_h telemTimer; + char *path; + int64_t checkTime; + SSdb *pSdb; + SMgmtWrapper *pWrapper; + SArray *pSteps; + SQHandle *pQuery; + SShowMgmt showMgmt; + SProfileMgmt profileMgmt; + STelemMgmt telemMgmt; + SSyncMgmt syncMgmt; + SHashObj *infosMeta; + SHashObj *perfsMeta; + SGrantInfo grant; + MndMsgFp msgFp[TDMT_MAX]; + SMsgCb msgCb; +} SMnode; void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); int64_t mndGenerateUid(char *name, int32_t len); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8f238f0f99..3b6b264d1b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -72,8 +72,8 @@ void mndCleanupStb(SMnode *pMnode) {} SSdbRaw *mndStbActionEncode(SStbObj *pStb) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) + - +pStb->commentLen + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE; + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen + + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; @@ -91,7 +91,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER) @@ -112,14 +111,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } - for (int32_t i = 0; i < pStb->numOfSmas; ++i) { - SSchema *pSchema = &pStb->pSmas[i]; - SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) - SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) - } - if (pStb->commentLen > 0) { SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER) } @@ -178,15 +169,13 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER) pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema)); - pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema)); - if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) { + if (pStb->pColumns == NULL || pStb->pTags == NULL) { goto _OVER; } @@ -206,14 +195,6 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } - for (int32_t i = 0; i < pStb->numOfSmas; ++i) { - SSchema *pSchema = &pStb->pSmas[i]; - SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER) - SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) - SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) - } - if (pStb->commentLen > 0) { pStb->comment = taosMemoryCalloc(pStb->commentLen, 1); if (pStb->comment == NULL) goto _OVER; @@ -291,18 +272,6 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } } - if (pOld->numOfSmas < pNew->numOfSmas) { - void *pSmas = taosMemoryMalloc(pNew->numOfSmas * sizeof(SSchema)); - if (pSmas != NULL) { - taosMemoryFree(pOld->pSmas); - pOld->pSmas = pSmas; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); - taosWUnLockLatch(&pOld->lock); - } - } - if (pOld->commentLen < pNew->commentLen) { void *comment = taosMemoryMalloc(pNew->commentLen); if (comment != NULL) { @@ -411,11 +380,6 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.schemaTag.nCols = pStb->numOfTags; req.schemaTag.pSchema = pStb->pTags; - // TODO: remove here - for (int iCol = 0; iCol < req.schema.nCols; iCol++) { - req.schema.pSchema[iCol].flags = SCHEMA_SMA_ON; - } - if (req.rollup) { req.pRSmaParam.xFilesFactor = pStb->xFilesFactor; req.pRSmaParam.delay = pStb->delay; @@ -674,7 +638,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.ttl = pCreate->ttl; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; - stbObj.numOfSmas = pCreate->numOfSmas; stbObj.commentLen = pCreate->commentLen; if (stbObj.commentLen > 0) { stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1); @@ -707,8 +670,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema)); stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema)); - stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema)); - if (stbObj.pColumns == NULL || stbObj.pTags == NULL || stbObj.pSmas == NULL) { + if (stbObj.pColumns == NULL || stbObj.pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -718,6 +680,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre SSchema *pSchema = &stbObj.pColumns[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; + pSchema->flags = pField->flags; memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); pSchema->colId = stbObj.nextColId; stbObj.nextColId++; @@ -733,18 +696,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.nextColId++; } - for (int32_t i = 0; i < stbObj.numOfSmas; ++i) { - SField *pField = taosArrayGet(pCreate->pSmas, i); - SSchema *pSchema = &stbObj.pSmas[i]; - SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name); - if (pColSchema == NULL) { - mError("stb:%s, sma:%s not found in columns", stbObj.name, pField->name); - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; - } - memcpy(pSchema, pColSchema, sizeof(SSchema)); - } - int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg); if (pTrans == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index e09e297cf3..3dbe3241a7 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -22,13 +22,15 @@ static int32_t mndInitWal(SMnode *pMnode) { char path[PATH_MAX] = {0}; snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); - SWalCfg cfg = {.vgId = 1, - .fsyncPeriod = 0, - .rollPeriod = -1, - .segSize = -1, - .retentionPeriod = -1, - .retentionSize = -1, - .level = TAOS_WAL_FSYNC}; + SWalCfg cfg = { + .vgId = 1, + .fsyncPeriod = 0, + .rollPeriod = -1, + .segSize = -1, + .retentionPeriod = -1, + .retentionSize = -1, + .level = TAOS_WAL_FSYNC, + }; pMgmt->pWal = walOpen(path, &cfg); if (pMgmt->pWal == NULL) return -1; @@ -54,62 +56,62 @@ static int32_t mndRestoreWal(SMnode *pMnode) { int64_t first = walGetFirstVer(pWal); int64_t last = walGetLastVer(pWal); - mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); + mDebug("start to restore wal, sdbver:%" PRId64 ", first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); first = TMAX(lastSdbVer + 1, first); for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) { if (walReadWithHandle(pHandle, ver) < 0) { - mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver); - goto WAL_RESTORE_OVER; + mError("ver:%" PRId64 ", failed to read from wal since %s", ver, terrstr()); + goto _OVER; } SWalHead *pHead = pHandle->pHead; int64_t sdbVer = sdbUpdateVer(pSdb, 0); if (sdbVer + 1 != ver) { terrno = TSDB_CODE_SDB_INVALID_WAl_VER; - mError("failed to read wal from sdb, sdbVer:%" PRId64 " inconsistent with ver:%" PRId64, sdbVer, ver); - goto WAL_RESTORE_OVER; + mError("ver:%" PRId64 ", failed to write to sdb, since inconsistent with sdbver:%" PRId64, ver, sdbVer); + goto _OVER; } - mTrace("wal:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body); + mTrace("ver:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body); if (sdbWriteWithoutFree(pSdb, (void *)pHead->head.body) < 0) { - mError("failed to read wal from sdb since %s, ver:%" PRId64, terrstr(), ver); - goto WAL_RESTORE_OVER; + mError("ver:%" PRId64 ", failed to write to sdb since %s", ver, terrstr()); + goto _OVER; } sdbUpdateVer(pSdb, 1); - mDebug("wal:%" PRId64 ", is restored", ver); + mDebug("ver:%" PRId64 ", is restored", ver); } int64_t sdbVer = sdbUpdateVer(pSdb, 0); - mDebug("restore sdb wal finished, sdb ver:%" PRId64, sdbVer); + mDebug("restore wal finished, sdbver:%" PRId64, sdbVer); mndTransPullup(pMnode); sdbVer = sdbUpdateVer(pSdb, 0); - mDebug("pullup trans finished, sdb ver:%" PRId64, sdbVer); + mDebug("pullup trans finished, sdbver:%" PRId64, sdbVer); if (sdbVer != lastSdbVer) { mInfo("sdb restored from %" PRId64 " to %" PRId64 ", write file", lastSdbVer, sdbVer); if (sdbWriteFile(pSdb) != 0) { - goto WAL_RESTORE_OVER; + goto _OVER; } if (walCommit(pWal, sdbVer) != 0) { - goto WAL_RESTORE_OVER; + goto _OVER; } if (walBeginSnapshot(pWal, sdbVer) < 0) { - goto WAL_RESTORE_OVER; + goto _OVER; } if (walEndSnapshot(pWal) < 0) { - goto WAL_RESTORE_OVER; + goto _OVER; } } code = 0; -WAL_RESTORE_OVER: +_OVER: walCloseReadHandle(pHandle); return code; } @@ -158,11 +160,11 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) { int64_t ver = sdbUpdateVer(pSdb, 1); if (walWrite(pWal, ver, 1, pRaw, sdbGetRawTotalSize(pRaw)) < 0) { sdbUpdateVer(pSdb, -1); - mError("failed to write raw:%p since %s, ver:%" PRId64, pRaw, terrstr(), ver); + mError("ver:%" PRId64 ", failed to write raw:%p to wal since %s", ver, pRaw, terrstr()); return -1; } - mTrace("raw:%p, write to wal, ver:%" PRId64, pRaw, ver); + mTrace("ver:%" PRId64 ", write to wal, raw:%p", ver, pRaw); walCommit(pWal, ver); walFsync(pWal, true); diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index e445022548..e5067c7027 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -146,7 +146,6 @@ int32_t mndInitTelem(SMnode* pMnode) { taosGetEmail(pMgmt->email, sizeof(pMgmt->email)); mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); - mDebug("mnode telemetry is initialized"); return 0; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 054bff466c..d6876782f6 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -21,8 +21,8 @@ #include "mndTrans.h" #include "tbase64.h" -#define TSDB_USER_VER_NUMBER 1 -#define TSDB_USER_RESERVE_SIZE 64 +#define USER_VER_NUMBER 1 +#define USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); static SSdbRaw *mndUserActionEncode(SUserObj *pUser); @@ -35,7 +35,7 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq); static int32_t mndProcessAlterUserReq(SNodeMsg *pReq); static int32_t mndProcessDropUserReq(SNodeMsg *pReq); static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq); -static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); int32_t mndInitUser(SMnode *pMnode) { @@ -93,9 +93,9 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); - int32_t size = sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN; + int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN; - SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, size); + SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) goto USER_ENCODE_OVER; int32_t dataPos = 0; @@ -120,7 +120,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { db = taosHashIterate(pUser->writeDbs, db); } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, USER_ENCODE_OVER) terrno = 0; @@ -142,7 +142,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto USER_DECODE_OVER; - if (sver != TSDB_USER_VER_NUMBER) { + if (sver != USER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto USER_DECODE_OVER; } @@ -184,7 +184,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN); } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_DECODE_OVER) terrno = 0; @@ -639,7 +639,7 @@ GET_AUTH_OVER: return code; } -static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -652,29 +652,29 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pB if (pShow->pIter == NULL) break; cols = 0; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->bytes[cols]); - colDataAppend(pColInfo, numOfRows, (const char*) name, false); + colDataAppend(pColInfo, numOfRows, (const char *)name, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - const char* src = pUser->superUser? "super":"normal"; - char b[10+VARSTR_HEADER_SIZE] = {0}; + const char *src = pUser->superUser ? "super" : "normal"; + char b[10 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src)); - colDataAppend(pColInfo, numOfRows, (const char*) b, false); + colDataAppend(pColInfo, numOfRows, (const char *)b, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataAppend(pColInfo, numOfRows, (const char*) &pUser->createdTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->acct, pShow->bytes[cols]); - colDataAppend(pColInfo, numOfRows, (const char*) name, false); + colDataAppend(pColInfo, numOfRows, (const char *)name, false); numOfRows++; sdbRelease(pSdb, pUser); diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index 0d41d5de20..54a154e684 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -114,18 +114,15 @@ void* MndTestSma::BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen) SMCreateStbReq createReq = {0}; createReq.numOfColumns = 3; createReq.numOfTags = 1; - createReq.numOfSmas = 1; createReq.igExists = 0; createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); - createReq.pSmas = taosArrayInit(createReq.numOfSmas, sizeof(SField)); strcpy(createReq.name, stbname); PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts"); PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1"); PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2"); PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1"); - PushField(createReq.pSmas, 2, TSDB_DATA_TYPE_TINYINT, "col1"); int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); void* pHead = rpcMallocCont(tlen); @@ -190,7 +187,7 @@ void* MndTestSma::BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int3 } TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) { - #if 0 +#if 0 const char* dbname = "1.d1"; const char* stbname = "1.d1.stb"; const char* smaname = "1.d1.sma"; @@ -244,7 +241,7 @@ TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) { test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); } -#endif +#endif } TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { @@ -264,7 +261,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { pReq = BuildCreateBSmaStbReq(stbname, &contLen); pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -280,7 +277,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { pReq = BuildDropStbReq(stbname, &contLen); pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index c645dae5b5..bc878d7b31 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -162,7 +162,4 @@ static int32_t sdbCreateDir(SSdb *pSdb) { return 0; } -int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) { - pSdb->curVer += val; - return pSdb->curVer; -} \ No newline at end of file +int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) { return atomic_add_fetch_64(&pSdb->curVer, val); } \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index a0885f2916..98ac04684e 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -28,7 +28,7 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) { if (fp == NULL) continue; if ((*fp)(pSdb->pMnode) != 0) { - mError("failed to deploy sdb:%d since %s", i, terrstr()); + mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr()); return -1; } } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a9f3c9f49b..2a784c112e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -19,7 +19,6 @@ target_sources( "src/meta/metaOpen.c" "src/meta/metaIdx.c" "src/meta/metaTable.c" - "src/meta/metaTDBImpl.c" "src/meta/metaQuery.c" "src/meta/metaCommit.c" "src/meta/metaEntry.c" @@ -33,7 +32,6 @@ target_sources( "src/tsdb/tsdbFS.c" "src/tsdb/tsdbMain.c" "src/tsdb/tsdbMemTable.c" - "src/tsdb/tsdbOptions.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbScan.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 8984b3d8ae..3314d2f264 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -74,7 +74,7 @@ typedef struct SMeta SMeta; // todo: remove typedef struct SMetaReader SMetaReader; typedef struct SMetaEntry SMetaEntry; -void metaReaderInit(SMetaReader *pReader, SVnode *pVnode, int32_t flags); +void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderClear(SMetaReader *pReader); int metaReadNext(SMetaReader *pReader); @@ -90,8 +90,8 @@ int metaTbCursorNext(SMTbCursor *pTbCur); #endif // tsdb -typedef struct STsdb STsdb; -typedef void *tsdbReaderT; +typedef struct STsdb STsdb; +typedef void *tsdbReaderT; #define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 #define BLOCK_LOAD_TABLE_SEQ_ORDER 2 @@ -111,7 +111,7 @@ bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis); SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList); -void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond); +void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond); void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 076cb74852..9a0b1f4578 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -16,13 +16,14 @@ #ifndef _TD_VNODE_META_H_ #define _TD_VNODE_META_H_ +#include "vnodeInt.h" + #ifdef __cplusplus extern "C" { #endif typedef struct SMetaIdx SMetaIdx; typedef struct SMetaDB SMetaDB; -typedef struct SMCtbCursor SMCtbCursor; typedef struct SMSmaCursor SMSmaCursor; // metaDebug ================== @@ -36,22 +37,16 @@ typedef struct SMSmaCursor SMSmaCursor; // clang-format on // metaOpen ================== -int metaOpen(SVnode* pVnode, SMeta** ppMeta); -int metaClose(SMeta* pMeta); // metaEntry ================== int metaEncodeEntry(SCoder* pCoder, const SMetaEntry* pME); int metaDecodeEntry(SCoder* pCoder, SMetaEntry* pME); // metaTable ================== -int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); -int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); // metaQuery ================== int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid); -int metaGetTableEntryByUid(SMetaReader* pReader, tb_uid_t uid); -int metaGetTableEntryByName(SMetaReader* pReader, const char* name); // metaIdx ================== int metaOpenIdx(SMeta* pMeta); @@ -60,8 +55,6 @@ int metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions); int metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid); // metaCommit ================== -int metaBegin(SMeta* pMeta); - static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); } struct SMeta { @@ -106,27 +99,12 @@ typedef struct { } STtlIdxKey; #if 1 -#define META_SUPER_TABLE TD_SUPER_TABLE -#define META_CHILD_TABLE TD_CHILD_TABLE -#define META_NORMAL_TABLE TD_NORMAL_TABLE // int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaDropTable(SMeta* pMeta, tb_uid_t uid); -int metaCommit(SMeta* pMeta); -int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); -int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); -SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); -STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); -void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode); -STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid); -SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup); -int metaGetTbNum(SMeta* pMeta); -SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); -void metaCloseSmaCursor(SMSmaCursor* pSmaCur); -int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); -SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); -void metaCloseCtbCurosr(SMCtbCursor* pCtbCur); -tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); +int metaDropTable(SMeta* pMeta, tb_uid_t uid); +SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); +void metaCloseSmaCursor(SMSmaCursor* pSmaCur); +int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); #ifndef META_REFACT // SMetaDB diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 347d28f1ea..5eb89e8bb7 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -16,6 +16,8 @@ #ifndef _TD_VNODE_TQ_H_ #define _TD_VNODE_TQ_H_ +#include "vnodeInt.h" + #include "executor.h" #include "os.h" #include "tcache.h" @@ -31,12 +33,12 @@ extern "C" { // tqDebug =================== // clang-format off -#define tqFatal(...) do { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) -#define tqError(...) do { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) -#define tqWarn(...) do { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) -#define tqInfo(...) do { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -#define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0) -#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqFatal(...) do { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define tqError(...) do { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define tqWarn(...) do { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define tqInfo(...) do { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on #define TQ_BUFFER_SIZE 4 @@ -237,17 +239,7 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); -void tqClose(STQ*); // required by vnode -int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); -int tqCommit(STQ*); - -int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); -int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); -int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**); int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 5972e0f9d2..3349accb88 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -16,6 +16,8 @@ #ifndef _TD_VNODE_TSDB_H_ #define _TD_VNODE_TSDB_H_ +#include "vnodeInt.h" + #ifdef __cplusplus extern "C" { #endif @@ -43,7 +45,6 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKe TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); // tsdbCommit ================ -int tsdbBegin(STsdb *pTsdb); #if 1 @@ -60,16 +61,9 @@ struct STable { #define TABLE_TID(t) (t)->tid #define TABLE_UID(t) (t)->uid -int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb); -int tsdbClose(STsdb *pTsdb); -int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp); int tsdbPrepareCommit(STsdb *pTsdb); -int tsdbCommit(STsdb *pTsdb); int32_t tsdbInitSma(STsdb *pTsdb); -int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg); int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg); -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version); -int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg); int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); void tsdbCleanupReadHandle(tsdbReaderT queryHandle); @@ -237,12 +231,6 @@ static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { return TD_ROW_KEY(row); } -// tsdbOptions -extern const STsdbCfg defautlTsdbOptions; - -int tsdbValidateOptions(const STsdbCfg *); -void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc); - // tsdbReadImpl typedef struct SReadH SReadH; diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index e20d2989b9..8fb18ddfea 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -16,9 +16,7 @@ #ifndef _TD_VNODE_TSDB_SMA_H_ #define _TD_VNODE_TSDB_SMA_H_ -#include "os.h" -#include "thash.h" -#include "tmsg.h" +#include "tsdb.h" #ifdef __cplusplus extern "C" { @@ -32,12 +30,6 @@ struct STbDdlH { __tb_ddl_fn_t fp; }; -typedef struct { - tb_uid_t suid; - SArray *tbUids; - SHashObj *uidHash; -} STbUidStore; - static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) { ASSERT(*pStore == NULL); *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 055120568e..3f5435ee47 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -16,6 +16,10 @@ #ifndef _TD_VND_H_ #define _TD_VND_H_ +#include "sync.h" +#include "syncTools.h" +#include "vnodeInt.h" + #ifdef __cplusplus extern "C" { #endif @@ -58,11 +62,9 @@ struct SVBufPool { SVBufPoolNode node; }; -int vnodeOpenBufPool(SVnode* pVnode, int64_t size); -int vnodeCloseBufPool(SVnode* pVnode); -void vnodeBufPoolReset(SVBufPool* pPool); -void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); -void vnodeBufPoolFree(SVBufPool* pPool, void* p); +int vnodeOpenBufPool(SVnode* pVnode, int64_t size); +int vnodeCloseBufPool(SVnode* pVnode); +void vnodeBufPoolReset(SVBufPool* pPool); // vnodeQuery ==================== int vnodeQueryOpen(SVnode* pVnode); @@ -79,6 +81,20 @@ int vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo); int vnodeSyncCommit(SVnode* pVnode); int vnodeAsyncCommit(SVnode* pVnode); +// vnodeCommit ==================== +int32_t vnodeSyncOpen(SVnode* pVnode, char* path); +int32_t vnodeSyncStart(SVnode* pVnode); +void vnodeSyncClose(SVnode* pVnode); +void vnodeSyncSetQ(SVnode* pVnode, void* qHandle); +void vnodeSyncSetRpc(SVnode* pVnode, void* rpcHandle); +int32_t vnodeSyncEqMsg(void* qHandle, SRpcMsg* pMsg); +int32_t vnodeSendMsg(void* rpcHandle, const SEpSet* pEpSet, SRpcMsg* pMsg); +void vnodeSyncCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); +void vnodeSyncPreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); +void vnodeSyncRollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); +int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot); +SSyncFSM* syncVnodeMakeFsm(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0f81ce8dfc..2d4cee3cad 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -59,6 +59,55 @@ typedef struct SQWorkerMgmt SQHandle; #define VNODE_TQ_DIR "tq" #define VNODE_WAL_DIR "wal" +// vnd.h +void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); +void vnodeBufPoolFree(SVBufPool* pPool, void* p); + +// meta +typedef struct SMCtbCursor SMCtbCursor; +typedef struct STbUidStore STbUidStore; + +int metaOpen(SVnode* pVnode, SMeta** ppMeta); +int metaClose(SMeta* pMeta); +int metaBegin(SMeta* pMeta); +int metaCommit(SMeta* pMeta); +int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); +SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); +STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); +int metaGetTableEntryByUid(SMetaReader* pReader, tb_uid_t uid); +int metaGetTableEntryByName(SMetaReader* pReader, const char* name); +int metaGetTbNum(SMeta* pMeta); +SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); +void metaCloseCtbCurosr(SMCtbCursor* pCtbCur); +tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); +SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup); +void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode); +STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid); +int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); +int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); + +// tsdb +int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb); +int tsdbClose(STsdb* pTsdb); +int tsdbBegin(STsdb* pTsdb); +int tsdbCommit(STsdb* pTsdb); +int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version); +int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); +int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); +int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); + +// tq +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); +void tqClose(STQ*); +int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); +int tqCommit(STQ*); +int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); +int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); + typedef struct { int8_t streamType; // sma or other int8_t dstType; @@ -106,6 +155,12 @@ struct SVnode { SQHandle* pQuery; }; +struct STbUidStore { + tb_uid_t suid; + SArray* tbUids; + SHashObj* uidHash; +}; + #define TD_VID(PVNODE) (PVNODE)->config.vgId typedef struct STbDdlH STbDdlH; @@ -113,18 +168,6 @@ typedef struct STbDdlH STbDdlH; // sma void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); -#include "vnd.h" - -#include "meta.h" - -#include "tsdb.h" - -#include "tq.h" - -#include "vnodeSync.h" - -#include "tsdbSma.h" - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h deleted file mode 100644 index fea94c4607..0000000000 --- a/source/dnode/vnode/src/inc/vnodeSync.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_SYNC_H_ -#define _TD_VNODE_SYNC_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t vnodeSyncOpen(SVnode *pVnode, char *path); -int32_t vnodeSyncStart(SVnode *pVnode); -void vnodeSyncClose(SVnode *pVnode); - -void vnodeSyncSetQ(SVnode *pVnode, void *qHandle); -void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle); - -int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg); -int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg); - -void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta); -void vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta); -void vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta); -int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot); - -SSyncFSM *syncVnodeMakeFsm(); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_SYNC_H_*/ diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index 246294d72f..456c4fd7ee 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "meta.h" static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBufPoolMalloc((SVBufPool *)pPool, size); } static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index a3cb812d93..e71a748f97 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "meta.h" int metaEncodeEntry(SCoder *pCoder, const SMetaEntry *pME) { if (tStartEncode(pCoder) < 0) return -1; diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index dbab5dff09..853b2ecefb 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -16,7 +16,7 @@ #ifdef USE_INVERTED_INDEX #include "index.h" #endif -#include "vnodeInt.h" +#include "meta.h" struct SMetaIdx { #ifdef USE_INVERTED_INDEX diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 940bf3e440..e65f5229af 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "meta.h" static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 964637c976..46a39e72f9 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "meta.h" -void metaReaderInit(SMetaReader *pReader, SVnode *pVnode, int32_t flags) { +void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); pReader->flags = flags; - pReader->pMeta = pVnode->pMeta; + pReader->pMeta = pMeta; } void metaReaderClear(SMetaReader *pReader) { @@ -32,6 +32,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u // query table.db if (tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _err; } @@ -54,6 +55,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { // query uid.idx if (tdbDbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -67,6 +69,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { // query name.idx if (tdbDbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -91,7 +94,7 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { return NULL; } - metaReaderInit(&pTbCur->mr, pMeta->pVnode, 0); + metaReaderInit(&pTbCur->mr, pMeta, 0); tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc); @@ -122,7 +125,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) { } metaGetTableEntryByVersion(&pTbCur->mr, *(int64_t *)pTbCur->pVal, *(tb_uid_t *)pTbCur->pKey); - if (pTbCur->mr.me.type == META_SUPER_TABLE) { + if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) { continue; } @@ -234,7 +237,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { STSchemaBuilder sb = {0}; SSchema *pSchema; - metaReaderInit(&mr, pMeta->pVnode, 0); + metaReaderInit(&mr, pMeta, 0); metaGetTableEntryByUid(&mr, uid); if (mr.me.type == TSDB_CHILD_TABLE) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 014af4b10d..452d52ce5e 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "meta.h" static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); @@ -37,7 +37,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaReader mr = {0}; // validate req - metaReaderInit(&mr, pMeta->pVnode, 0); + metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByName(&mr, pReq->name) == 0) { // TODO: just for pass case #if 0 @@ -91,7 +91,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { pReq->ctime = taosGetTimestampSec(); // validate req - metaReaderInit(&mr, pMeta->pVnode, 0); + metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByName(&mr, pReq->name) == 0) { terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8cabada511..7cee82f660 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tq.h" int32_t tqInit() { // @@ -78,7 +78,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pExec->pushHandle.reqOffset; - rsp.blockData = taosArrayInit(0, sizeof(int32_t)); + rsp.blockData = taosArrayInit(0, sizeof(void*)); rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { @@ -176,9 +176,9 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ atomic_store_ptr(&pExec->pushHandle.handle, NULL); taosWUnLockLatch(&pExec->pushHandle.lock); - vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", - TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum, - rsp.reqOffset, rsp.rspOffset); + tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum, + rsp.reqOffset, rsp.rspOffset); // TODO destroy taosArrayDestroy(rsp.blockData); @@ -210,35 +210,6 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req); -#if 0 - void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL); - while (pIter != NULL) { - STqPusher* pusher = *(STqPusher**)pIter; - if (pusher->type == TQ_PUSHER_TYPE__STREAM) { - STqStreamPusher* streamPusher = (STqStreamPusher*)pusher; - // repack - STqStreamToken* token = taosMemoryMalloc(sizeof(STqStreamToken)); - if (token == NULL) { - taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - token->type = TQ_STREAM_TOKEN__DATA; - token->data = msg; - // set input - // exec - } - // send msg to ep - } - // iterate hash - // process all msg - // if waiting - // memcpy and send msg to fetch thread - // TODO: add reference - // if handle waiting, launch query and response to consumer - // - // if no waiting handle, return -#endif return 0; } @@ -375,11 +346,11 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu } int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { - SMqPollReqV2* pReq = pMsg->pCont; - int64_t consumerId = pReq->consumerId; - int64_t waitTime = pReq->blockingTime; - int32_t reqEpoch = pReq->epoch; - int64_t fetchOffset; + SMqPollReq* pReq = pMsg->pCont; + int64_t consumerId = pReq->consumerId; + int64_t waitTime = pReq->waitTime; + int32_t reqEpoch = pReq->epoch; + int64_t fetchOffset; // get offset to fetch message if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { @@ -390,8 +361,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { fetchOffset = pReq->currentOffset + 1; } - vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); + tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); STqExec* pExec = taosHashGet(pTq->execs, pReq->subKey, strlen(pReq->subKey)); ASSERT(pExec); @@ -418,16 +389,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { while (1) { consumerEpoch = atomic_load_32(&pExec->epoch); if (consumerEpoch > reqEpoch) { - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", - consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", + consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); break; } taosThreadMutexLock(&pExec->pWalReader->mutex); if (walFetchHead(pExec->pWalReader, fetchOffset, pHeadWithCkSum) < 0) { - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), fetchOffset); + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), fetchOffset); taosThreadMutexUnlock(&pExec->pWalReader->mutex); break; } @@ -448,7 +419,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); #if 0 @@ -476,8 +447,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } #endif - vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); + tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; @@ -591,8 +562,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pMsg->code = 0; tmsgSendRsp(pMsg); - vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", - TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); + tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); // TODO destroy taosArrayDestroy(rsp.blockData); @@ -618,7 +589,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { fetchOffset = pReq->currentOffset + 1; } - vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, + tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); SMqPollRspV2 rspV2 = {0}; @@ -660,7 +631,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; } - vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, + tqDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, TD_VID(pTq->pVnode)); rspV2.reqOffset = pReq->currentOffset; @@ -671,7 +642,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO consumerEpoch = atomic_load_32(&pConsumer->epoch); if (consumerEpoch > reqEpoch) { - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); break; } @@ -680,11 +651,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user - vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, + tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); break; } - vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, + tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/ /*pHead = pTopic->pReadhandle->pHead;*/ @@ -709,7 +680,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } if (taosArrayGetSize(pRes) == 0) { - vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId, + tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); fetchOffset++; rspV2.skipLogNum++; @@ -770,7 +741,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pMsg->pCont = buf; pMsg->contLen = msgLen; pMsg->code = 0; - vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset, + tqDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset, pHead->msgType, consumerId, pReq->epoch); tmsgSendRsp(pMsg); taosMemoryFree(pHead); @@ -804,7 +775,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pMsg->contLen = tlen; pMsg->code = 0; tmsgSendRsp(pMsg); - vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId, + tqDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch); /*}*/ diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c index 8e59243a9c..e31566f3fa 100644 --- a/source/dnode/vnode/src/tq/tqCommit.c +++ b/source/dnode/vnode/src/tq/tqCommit.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tq.h" diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 72469b74ac..ca09cc1dc1 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tq.h" // #include // #include // #include @@ -86,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } strcpy(pMeta->dirPath, path); - char *name = taosMemoryMalloc(pathLen + 10) ; + char* name = taosMemoryMalloc(pathLen + 10); strcpy(name, path); if (!taosDirExist(name) && taosMkDir(name) != 0) { diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 3cff87340d..90f512611b 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "vnodeInt.h" +#include "tq.h" enum ETqOffsetPersist { TQ_OFFSET_PERSIST__LAZY = 1, diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index aeb9f27eab..cf1154071d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tq.h" STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle)); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index f5483b1141..29f148b64d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" #define TSDB_MAX_SUBBLOCKS 8 diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index b16f2f0fdb..585ef63531 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" int tsdbBegin(STsdb *pTsdb) { STsdbMemTable *pMem; diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c index e24d5974af..9e0721815a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ #if 0 -#include "tsdbint.h" +#include "tsdb.h" typedef struct { STable * pTable; @@ -33,15 +33,15 @@ typedef struct { SDataCols *pDataCols; } SCompactH; -#define TSDB_COMPACT_WSET(pComph) (&((pComph)->wSet)) -#define TSDB_COMPACT_REPO(pComph) TSDB_READ_REPO(&((pComph)->readh)) +#define TSDB_COMPACT_WSET(pComph) (&((pComph)->wSet)) +#define TSDB_COMPACT_REPO(pComph) TSDB_READ_REPO(&((pComph)->readh)) #define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD) #define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA) #define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST) #define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD) #define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL) -#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh)) -#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh)) +#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh)) +#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh)) static int tsdbAsyncCompact(STsdbRepo *pRepo); static void tsdbStartCompact(STsdbRepo *pRepo); @@ -531,4 +531,3 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { } #endif - diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 0012a50ccf..91a7e1cd54 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 2fbe819476..74e3c66a9d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" static const char *TSDB_FNAME_SUFFIX[] = { "head", // TSDB_FILE_HEAD diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index c1025e13eb..de5ff9ac91 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) { STsdb *pTsdb = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d278a8d98d..323fc7970b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" static STbData *tsdbNewTbData(tb_uid_t uid); static void tsdbFreeTbData(STbData *pTbData); diff --git a/source/dnode/vnode/src/tsdb/tsdbOptions.c b/source/dnode/vnode/src/tsdb/tsdbOptions.c deleted file mode 100644 index 3560c9feaa..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbOptions.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeInt.h" - -const STsdbCfg defautlTsdbOptions = {.precision = 0, - .lruCacheSize = 0, - .days = 10, - .minRows = 100, - .maxRows = 4096, - .keep2 = 3650, - .keep0 = 3650, - .keep1 = 3650, - .update = 0, - .compression = TWO_STAGE_COMP}; - -int tsdbValidateOptions(const STsdbCfg *pTsdbOptions) { - // TODO - return 0; -} - -void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc) { memcpy(pDest, pSrc, sizeof(STsdbCfg)); } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2c8819f60b..302ee89e51 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -3618,7 +3618,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) { SMetaReader mr = {0}; - metaReaderInit(&mr, ((SMeta*)pMeta)->pVnode, 0); + metaReaderInit(&mr, (SMeta*)pMeta, 0); if (metaGetTableEntryByUid(&mr, uid) < 0) { tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); @@ -3628,7 +3628,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); } - if (mr.me.type != META_SUPER_TABLE) { + if (mr.me.type != TSDB_SUPER_TABLE) { tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); terrno = TSDB_CODE_OPS_NOT_SUPPORT; // basically, this error is caused by invalid sql issued by client @@ -3687,10 +3687,9 @@ int32_t tsdbQueryTableList(void* pMeta, SArray* pRes, void* filterInfo) { return TSDB_CODE_SUCCESS; } int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { - SMeta* metaP = (SMeta*)pMeta; SMetaReader mr = {0}; - metaReaderInit(&mr, metaP->pVnode, 0); + metaReaderInit(&mr, (SMeta*)pMeta, 0); if (metaGetTableEntryByUid(&mr, uid) < 0) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index f18f36e07b..c39780372b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" #define TSDB_KEY_COL_OFFSET 0 diff --git a/source/dnode/vnode/src/tsdb/tsdbScan.c b/source/dnode/vnode/src/tsdb/tsdbScan.c index c0e468e640..cc6fb473d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbScan.c +++ b/source/dnode/vnode/src/tsdb/tsdbScan.c @@ -13,9 +13,8 @@ * along with this program. If not, see . */ - #if 0 -#include "tsdbint.h" +#include "tsdb.h" #ifndef _TSDB_PLUGINS int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 930ea58151..c9e33cefc7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -13,7 +13,8 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdbSma.h" +#include "tsdb.h" static const char *TSDB_SMA_DNAME[] = { "", // TSDB_SMA_TYPE_BLOCK @@ -678,7 +679,6 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers return TSDB_CODE_FAILED; } - if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index ebfa1ecaeb..ac3baf8c3a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -15,14 +15,14 @@ #define ALLOW_FORBID_FUNC -#include "vnodeInt.h" +#include "tsdb.h" int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { - int ret = 0; + int ret = 0; if (path == NULL) return -1; - ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param if (ret != 0) { tsdbError("Failed to create tsdb db env, ret = %d", ret); diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 79a56c3c9d..82fa6b9ae5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "tsdb.h" static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 0125cdc82f..9122913cda 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" /* ------------------------ STRUCTURES ------------------------ */ diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 905bf89be4..df8cf8d503 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" const SVnodeCfg vnodeCfgDefault = { .vgId = -1, diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index cee1018b71..1c0e65aa95 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" @@ -175,7 +175,7 @@ int vnodeAsyncCommit(SVnode *pVnode) { vnodeWaitCommit(pVnode); // vnodeBufPoolSwitch(pVnode); - tsdbPrepareCommit(pVnode->pTsdb); + // tsdbPrepareCommit(pVnode->pTsdb); vnodeScheduleTask(vnodeCommitImpl, pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeInt.c b/source/dnode/vnode/src/vnd/vnodeInt.c index 10d8154a15..14dfc75ced 100644 --- a/source/dnode/vnode/src/vnd/vnodeInt.c +++ b/source/dnode/vnode/src/vnd/vnodeInt.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "vnodeInt.h" +#include "vnd.h" // #include "vnodeInt.h" int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 2b5b46a45d..efae74b55a 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" typedef struct SVnodeTask SVnodeTask; struct SVnodeTask { diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 38f39ae1e3..ee64976df9 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" -#include "vnodeSync.h" +#include "vnd.h" int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { SVnodeInfo info = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f5c18aa2bd..ef7b9c97b5 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#include "vnd.h" int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, TD_VID(pVnode), NULL, (void **)&pVnode->pQuery, &pVnode->msgCb); @@ -51,9 +51,10 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { } // query meta - metaReaderInit(&mer1, pVnode, 0); + metaReaderInit(&mer1, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) { + code = terrno; goto _exit; } @@ -67,7 +68,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { schemaTag = mer1.me.stbEntry.schemaTag; metaRsp.suid = mer1.me.uid; } else if (mer1.me.type == TSDB_CHILD_TABLE) { - metaReaderInit(&mer2, pVnode, 0); + metaReaderInit(&mer2, pVnode->pMeta, 0); if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; strcpy(metaRsp.stbName, mer2.me.name); @@ -105,6 +106,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { } tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); +_exit: rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; rpcMsg.refId = pMsg->refId; @@ -114,7 +116,6 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { tmsgSendRsp(&rpcMsg); -_exit: taosMemoryFree(metaRsp.pSchemas); metaReaderClear(&mer2); metaReaderClear(&mer1); @@ -123,8 +124,8 @@ _exit: int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->vgId = TD_VID(pVnode); - //pLoad->syncState = TAOS_SYNC_STATE_LEADER; - pLoad->syncState = syncGetMyRole(pVnode->sync); // sync integration + // pLoad->syncState = TAOS_SYNC_STATE_LEADER; + pLoad->syncState = syncGetMyRole(pVnode->sync); // sync integration pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTimeSeries = 400; pLoad->totalStorage = 300; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 61cedc6b50..efcc82853c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -13,9 +13,7 @@ * along with this program. If not, see . */ -#include "sync.h" -#include "syncTools.h" -#include "vnodeInt.h" +#include "vnd.h" static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -201,9 +199,7 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // sync integration int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - if (syncEnvIsStart()) { - SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); assert(pSyncNode != NULL); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5f8bf3c7b5..26393394e9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -13,10 +13,11 @@ * along with this program. If not, see . */ -#include "sync.h" -#include "syncTools.h" -#include "tmsgcb.h" -#include "vnodeInt.h" +#include "vnd.h" +// #include "sync.h" +// #include "syncTools.h" +// #include "tmsgcb.h" +// #include "vnodeInt.h" // sync integration @@ -113,7 +114,7 @@ void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - SVnode * pVnode = (SVnode *)(pFsm->data); + SVnode *pVnode = (SVnode *)(pFsm->data); SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta); SRpcMsg applyMsg; syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg); diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index af4c8a7bdb..e19a332e66 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -24,7 +24,7 @@ extern "C" { #define FUNCTION_NAME_MAX_LENGTH 32 -#define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n) (1 << n) +#define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n) (1 << n) #define FUNC_MGT_AGG_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(0) #define FUNC_MGT_SCALAR_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(1) @@ -38,6 +38,7 @@ extern "C" { #define FUNC_MGT_SPECIAL_DATA_REQUIRED FUNC_MGT_FUNC_CLASSIFICATION_MASK(9) #define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10) #define FUNC_MGT_MULTI_RES_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(11) +#define FUNC_MGT_SCAN_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(12) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 805df08f27..45cfbae1ad 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -807,7 +807,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = NULL}, {.name = "tbname", .type = FUNCTION_TYPE_TBNAME, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 91ccdfb71e..72c7175c6d 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -15,12 +15,12 @@ #include "functionMgt.h" +#include "builtins.h" +#include "catalog.h" #include "functionMgtInt.h" #include "taos.h" #include "taoserror.h" #include "thash.h" -#include "builtins.h" -#include "catalog.h" typedef struct SFuncMgtService { SHashObj* pFuncNameHashTable; @@ -28,22 +28,24 @@ typedef struct SFuncMgtService { typedef struct SUdfInfo { SDataType outputDt; - int8_t funcType; + int8_t funcType; } SUdfInfo; static SFuncMgtService gFunMgtService; -static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT; -static int32_t initFunctionCode = 0; +static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT; +static int32_t initFunctionCode = 0; static void doInitFunctionTable() { - gFunMgtService.pFuncNameHashTable = taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + gFunMgtService.pFuncNameHashTable = + taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (NULL == gFunMgtService.pFuncNameHashTable) { initFunctionCode = TSDB_CODE_FAILED; return; } for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) { - if (TSDB_CODE_SUCCESS != taosHashPut(gFunMgtService.pFuncNameHashTable, funcMgtBuiltins[i].name, strlen(funcMgtBuiltins[i].name), &i, sizeof(int32_t))) { + if (TSDB_CODE_SUCCESS != taosHashPut(gFunMgtService.pFuncNameHashTable, funcMgtBuiltins[i].name, + strlen(funcMgtBuiltins[i].name), &i, sizeof(int32_t))) { initFunctionCode = TSDB_CODE_FAILED; return; } @@ -52,8 +54,9 @@ static void doInitFunctionTable() { static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { if (fmIsUserDefinedFunc(funcId)) { - return FUNC_MGT_AGG_FUNC == classification ? FUNC_AGGREGATE_UDF_ID == funcId : - (FUNC_MGT_SCALAR_FUNC == classification ? FUNC_SCALAR_UDF_ID == funcId : false); + return FUNC_MGT_AGG_FUNC == classification + ? FUNC_AGGREGATE_UDF_ID == funcId + : (FUNC_MGT_SCALAR_FUNC == classification ? FUNC_SCALAR_UDF_ID == funcId : false); } if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; @@ -63,7 +66,7 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { static int32_t getUdfInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) { SFuncInfo* pInfo = NULL; - int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &pInfo); + int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &pInfo); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -122,33 +125,23 @@ int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) { return TSDB_CODE_FAILED; } pFpSet->process = funcMgtBuiltins[funcId].sprocessFunc; - pFpSet->getEnv = funcMgtBuiltins[funcId].getEnvFunc; + pFpSet->getEnv = funcMgtBuiltins[funcId].getEnvFunc; return TSDB_CODE_SUCCESS; } -bool fmIsAggFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_AGG_FUNC); -} +bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_AGG_FUNC); } -bool fmIsScalarFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); -} +bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } -bool fmIsPseudoColumnFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); -} +bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } -bool fmIsWindowPseudoColumnFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC); -} +bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); } -bool fmIsWindowClauseFunc(int32_t funcId) { - return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId); -} +bool fmIsWindowPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC); } -bool fmIsNonstandardSQLFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_NONSTANDARD_SQL_FUNC); -} +bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId); } + +bool fmIsNonstandardSQLFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_NONSTANDARD_SQL_FUNC); } bool fmIsSpecialDataRequiredFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED); @@ -158,13 +151,9 @@ bool fmIsDynamicScanOptimizedFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED); } -bool fmIsMultiResFunc(int32_t funcId) { - return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_RES_FUNC); -} +bool fmIsMultiResFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_RES_FUNC); } -bool fmIsUserDefinedFunc(int32_t funcId) { - return funcId > FUNC_UDF_ID_START; -} +bool fmIsUserDefinedFunc(int32_t funcId) { return funcId > FUNC_UDF_ID_START; } void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 89cacf9298..f1fe68c55a 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -224,6 +224,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_ALL_SCALAR_FIELDS; COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pScanCols); + CLONE_NODE_LIST_FIELD(pScanPseudoCols); CLONE_OBJECT_FIELD(pMeta, tableMetaClone); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index f8a56109ee..0e6ec4f945 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -467,6 +467,7 @@ static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { } static const char* jkScanLogicPlanScanCols = "ScanCols"; +static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize"; static const char* jkScanLogicPlanTableMeta = "TableMeta"; @@ -477,6 +478,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkScanLogicPlanScanCols, pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkScanLogicPlanScanPseudoCols, pNode->pScanPseudoCols); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); } @@ -495,6 +499,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkScanLogicPlanScanCols, &pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanLogicPlanScanPseudoCols, &pNode->pScanPseudoCols); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize); } @@ -670,6 +677,7 @@ static int32_t jsonToName(const SJson* pJson, void* pObj) { } static const char* jkScanPhysiPlanScanCols = "ScanCols"; +static const char* jkScanPhysiPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanPhysiPlanTableId = "TableId"; static const char* jkScanPhysiPlanTableType = "TableType"; static const char* jkScanPhysiPlanTableName = "TableName"; @@ -681,6 +689,9 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkScanPhysiPlanScanCols, pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkScanPhysiPlanScanPseudoCols, pNode->pScanPseudoCols); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableId, pNode->uid); } @@ -701,6 +712,9 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkScanPhysiPlanScanCols, &pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanPhysiPlanScanPseudoCols, &pNode->pScanPseudoCols); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkScanPhysiPlanTableId, &pNode->uid); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 092d0f435d..351b50133c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1041,10 +1041,11 @@ bool nodesIsTimeorderQuery(const SNode* pQuery) { return false; } bool nodesIsTimelineQuery(const SNode* pQuery) { return false; } typedef struct SCollectColumnsCxt { - int32_t errCode; - const char* pTableAlias; - SNodeList* pCols; - SHashObj* pColHash; + int32_t errCode; + const char* pTableAlias; + ECollectColType collectType; + SNodeList* pCols; + SHashObj* pColHash; } SCollectColumnsCxt; static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pNode) { @@ -1057,25 +1058,33 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN if (NULL == taosHashGet(pCxt->pColHash, name, len)) { pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0); if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pCxt->errCode = nodesListAppend(pCxt->pCols, pNode); + pCxt->errCode = nodesListStrictAppend(pCxt->pCols, nodesCloneNode(pNode)); } return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } return DEAL_RES_CONTINUE; } +static bool isCollectType(ECollectColType collectType, EColumnType colType) { + return COLLECT_COL_TYPE_ALL == collectType + ? true + : (COLLECT_COL_TYPE_TAG == collectType ? COLUMN_TYPE_TAG == colType : COLUMN_TYPE_TAG != colType); +} + static EDealRes collectColumns(SNode* pNode, void* pContext) { SCollectColumnsCxt* pCxt = (SCollectColumnsCxt*)pContext; if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; - if (NULL == pCxt->pTableAlias || 0 == strcmp(pCxt->pTableAlias, pCol->tableAlias)) { + if (isCollectType(pCxt->collectType, pCol->colType) && + (NULL == pCxt->pTableAlias || 0 == strcmp(pCxt->pTableAlias, pCol->tableAlias))) { return doCollect(pCxt, pCol, pNode); } } return DEAL_RES_CONTINUE; } -int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols) { +int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, + SNodeList** pCols) { if (NULL == pSelect || NULL == pCols) { return TSDB_CODE_SUCCESS; } @@ -1083,6 +1092,7 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* SCollectColumnsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pTableAlias = pTableAlias, + .collectType = type, .pCols = nodesMakeList(), .pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)}; if (NULL == cxt.pCols || NULL == cxt.pColHash) { @@ -1092,11 +1102,11 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* nodesWalkSelectStmt(pSelect, clause, collectColumns, &cxt); taosHashCleanup(cxt.pColHash); if (TSDB_CODE_SUCCESS != cxt.errCode) { - nodesClearList(cxt.pCols); + nodesDestroyList(cxt.pCols); return cxt.errCode; } if (0 == LIST_LENGTH(cxt.pCols)) { - nodesClearList(cxt.pCols); + nodesDestroyList(cxt.pCols); cxt.pCols = NULL; } *pCols = cxt.pCols; @@ -1123,10 +1133,12 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod return TSDB_CODE_SUCCESS; } - SCollectFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .classifier = classifier, .pFuncs = nodesMakeList()}; + SCollectFuncsCxt cxt = { + .errCode = TSDB_CODE_SUCCESS, .classifier = classifier, .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs)}; if (NULL == cxt.pFuncs) { return TSDB_CODE_OUT_OF_MEMORY; } + *pFuncs = NULL; nodesWalkSelectStmt(pSelect, SQL_CLAUSE_GROUP_BY, collectFuncs, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pFuncs); @@ -1136,7 +1148,6 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod *pFuncs = cxt.pFuncs; } else { nodesDestroyList(cxt.pFuncs); - *pFuncs = NULL; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 33f5fd19ef..aae932471d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1905,18 +1905,9 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; SField field = {.type = pCol->dataType.type, .bytes = calcTypeBytes(pCol->dataType)}; strcpy(field.name, pCol->colName); - taosArrayPush(*pArray, &field); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) { - *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); - SNode* pNode; - FOREACH(pNode, pList) { - SColumnNode* pCol = (SColumnNode*)pNode; - SField field = {.type = pCol->node.resType.type, .bytes = calcTypeBytes(pCol->node.resType)}; - strcpy(field.name, pCol->colName); + if (pCol->sma) { + field.flags |= SCHEMA_SMA_ON; + } taosArrayPush(*pArray, &field); } return TSDB_CODE_SUCCESS; @@ -2249,13 +2240,6 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm columnDefNodeToField(pStmt->pTags, &pReq->pTags); pReq->numOfColumns = LIST_LENGTH(pStmt->pCols); pReq->numOfTags = LIST_LENGTH(pStmt->pTags); - if (NULL == pStmt->pOptions->pSma) { - columnDefNodeToField(pStmt->pCols, &pReq->pSmas); - pReq->numOfSmas = pReq->numOfColumns; - } else { - columnNodeToField(pStmt->pOptions->pSma, &pReq->pSmas); - pReq->numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma); - } SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index f8ff2848af..450fc307ea 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -129,14 +129,66 @@ static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelec return code; } -static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanCols, STableMeta* pMeta) { +typedef struct SCreateColumnCxt { + int32_t errCode; + SNodeList* pList; +} SCreateColumnCxt; + +static EDealRes doCreateColumn(SNode* pNode, void* pContext) { + SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: { + SNode* pCol = nodesCloneNode(pNode); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_FUNCTION: { + SExprNode* pExpr = (SExprNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } + pCol->node.resType = pExpr->resType; + strcpy(pCol->colName, pExpr->aliasName); + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + default: + break; + } + + return DEAL_RES_CONTINUE; +} + +static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) { + SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; + if (NULL == cxt.pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + nodesWalkExprs(pExprs, doCreateColumn, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(cxt.pList); + return cxt.errCode; + } + if (NULL == *pList) { + *pList = cxt.pList; + } + return cxt.errCode; +} + +static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols, SNodeList* pScanCols, + STableMeta* pMeta) { if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { return SCAN_TYPE_STREAM; } if (NULL == pScanCols) { // select count(*) from t - return SCAN_TYPE_TABLE; + return NULL == pScanPseudoCols ? SCAN_TYPE_TABLE : SCAN_TYPE_TAG; } if (TSDB_SYSTEM_TABLE == pMeta->tableType) { @@ -186,7 +238,6 @@ static int32_t addPrimaryKeyCol(uint64_t tableId, SNodeList** pCols) { if (!found) { if (TSDB_CODE_SUCCESS != nodesListStrictAppend(*pCols, createPrimaryKeyCol(tableId))) { - nodesDestroyList(*pCols); return TSDB_CODE_OUT_OF_MEMORY; } } @@ -214,30 +265,31 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; // set columns to scan - SNodeList* pCols = NULL; - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols); + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL, + &pScan->pScanCols); + if (TSDB_CODE_SUCCESS == code) { - code = addPrimaryKeyCol(pScan->pMeta->uid, &pCols); + code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_TAG, + &pScan->pScanPseudoCols); } if (TSDB_CODE_SUCCESS == code) { - pScan->pScanCols = nodesCloneList(pCols); - if (NULL == pScan) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = nodesCollectFuncs(pSelect, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols); } - pScan->scanType = getScanType(pCxt, pCols, pScan->pMeta); + pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta); + + if (TSDB_CODE_SUCCESS == code) { + code = addPrimaryKeyCol(pScan->pMeta->uid, &pScan->pScanCols); + } // set output if (TSDB_CODE_SUCCESS == code) { - pScan->node.pTargets = nodesCloneList(pCols); - if (NULL == pScan) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = createColumnByRewriteExps(pCxt, pScan->pScanCols, &pScan->node.pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pCxt, pScan->pScanPseudoCols, &pScan->node.pTargets); } - - nodesClearList(pCols); if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pScan; @@ -362,57 +414,6 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) return pCol; } -typedef struct SCreateColumnCxt { - int32_t errCode; - SNodeList* pList; -} SCreateColumnCxt; - -static EDealRes doCreateColumn(SNode* pNode, void* pContext) { - SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; - switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: { - SNode* pCol = nodesCloneNode(pNode); - if (NULL == pCol) { - return DEAL_RES_ERROR; - } - return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); - } - case QUERY_NODE_OPERATOR: - case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_FUNCTION: { - SExprNode* pExpr = (SExprNode*)pNode; - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - return DEAL_RES_ERROR; - } - pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); - return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); - } - default: - break; - } - - return DEAL_RES_CONTINUE; -} - -static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) { - SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; - if (NULL == cxt.pList) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - nodesWalkExprs(pExprs, doCreateColumn, &cxt); - if (TSDB_CODE_SUCCESS != cxt.errCode) { - nodesDestroyList(cxt.pList); - return cxt.errCode; - } - if (NULL == *pList) { - *pList = cxt.pList; - } - return cxt.errCode; -} - static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; @@ -598,14 +599,7 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } - SNodeList* pCols = NULL; - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, &pCols); - if (TSDB_CODE_SUCCESS == code && NULL != pCols) { - pSort->node.pTargets = nodesCloneList(pCols); - if (NULL == pSort->node.pTargets) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); if (TSDB_CODE_SUCCESS == code) { pSort->pSortKeys = nodesCloneList(pSelect->pOrderByList); @@ -695,14 +689,8 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS return TSDB_CODE_OUT_OF_MEMORY; } - SNodeList* pCols = NULL; - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, NULL, &pCols); - if (TSDB_CODE_SUCCESS == code && NULL != pCols) { - pPartition->node.pTargets = nodesCloneList(pCols); - if (NULL == pPartition->node.pTargets) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } + int32_t code = + nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, NULL, COLLECT_COL_TYPE_ALL, &pPartition->node.pTargets); if (TSDB_CODE_SUCCESS == code) { pPartition->pPartitionKeys = nodesCloneList(pSelect->pPartitionByList); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4997cfd644..707e6aac14 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -380,6 +380,10 @@ static int32_t sortScanCols(SNodeList* pScanCols) { } static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) { + if (NULL == pScanCols) { + return TSDB_CODE_SUCCESS; + } + pScanPhysiNode->pScanCols = nodesCloneList(pScanCols); if (NULL == pScanPhysiNode->pScanCols) { return TSDB_CODE_OUT_OF_MEMORY; @@ -394,9 +398,22 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNo // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t code = addDataBlockSlots(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); } + + if (TSDB_CODE_SUCCESS == code && NULL != pScanLogicNode->pScanPseudoCols) { + pScanPhysiNode->pScanPseudoCols = nodesCloneList(pScanLogicNode->pScanPseudoCols); + if (NULL == pScanPhysiNode->pScanPseudoCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pScanPhysiNode->pScanPseudoCols, pScanPhysiNode->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode); } + if (TSDB_CODE_SUCCESS == code) { pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; @@ -1190,6 +1207,11 @@ static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubp ++(pQueryPlan->numOfSubplans); } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pSubplan); + return code; + } + if (TSDB_CODE_SUCCESS == code && NULL != pParent) { code = nodesListMakeAppend(&pParent->pChildren, pSubplan); if (TSDB_CODE_SUCCESS == code) { @@ -1207,10 +1229,6 @@ static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubp } } - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pSubplan); - } - return code; } diff --git a/source/libs/planner/test/planSTableTest.cpp b/source/libs/planner/test/planSTableTest.cpp index 49002fe826..0b60eaea32 100644 --- a/source/libs/planner/test/planSTableTest.cpp +++ b/source/libs/planner/test/planSTableTest.cpp @@ -19,7 +19,7 @@ using namespace std; class PlanSuperTableTest : public PlannerTestBase {}; -TEST_F(PlanSuperTableTest, unionAll) { +TEST_F(PlanSuperTableTest, tbname) { useDb("root", "test"); run("select tbname from st1"); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 5dab6f0a97..aa8a03f3d2 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -103,6 +103,9 @@ typedef void* queue[2]; /* Return the structure holding the given element. */ #define QUEUE_DATA(e, type, field) ((type*)((void*)((char*)(e)-offsetof(type, field)))) +#define TRANS_RETRY_COUNT_LIMIT 10 // retry count limit +#define TRANS_RETRY_INTERVAL 5 // ms retry interval + typedef struct { SRpcInfo* pRpc; // associated SRpcInfo SEpSet epSet; // ip list provided by app @@ -137,14 +140,12 @@ typedef struct { int8_t connType; // connection type cli/srv int64_t rid; // refId returned by taosAddRef + int8_t retryCount; STransCtx appCtx; // STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API - int hThrdIdx; - char* ip; - uint32_t port; - // SEpSet* pSet; // for synchronous API + int hThrdIdx; } STransConnCtx; #pragma pack(push, 1) @@ -215,8 +216,6 @@ void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); bool transCompressMsg(char* msg, int32_t len, int32_t* flen); bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); -void transConnCtxDestroy(STransConnCtx* ctx); - void transFreeMsg(void* msg); // @@ -262,8 +261,8 @@ void transUnrefCliHandle(void* handle); void transReleaseCliHandle(void* handle); void transReleaseSrvHandle(void* handle); -void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* pCtx); -void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransMsg* pRsp); +void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); +void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); void transSendResponse(const STransMsg* msg); void transRegisterMsg(const STransMsg* msg); int transGetConnInfo(void* thandle, STransHandleInfo* pInfo); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index eaca9b0fc7..56f38a7a55 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -62,8 +62,7 @@ typedef struct { char ckey[TSDB_PASSWORD_LEN]; // ciphering key void (*cfp)(void* parent, SRpcMsg*, SEpSet*); - int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); - int (*retry)(void* parent, SRpcMsg*, SEpSet*); + bool (*retry)(int32_t code); int32_t refCount; void* parent; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index fa517d6d61..c0da3f9c1f 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -38,7 +38,6 @@ void* rpcOpen(const SRpcInit* pInit) { // register callback handle pRpc->cfp = pInit->cfp; - pRpc->afp = pInit->afp; pRpc->retry = pInit->rfp; if (pInit->connType == TAOS_CONN_SERVER) { @@ -116,19 +115,13 @@ int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } void rpcCancelRequest(int64_t rid) { return; } void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { - char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); - uint32_t port = pEpSet->eps[pEpSet->inUse].port; - transSendRequest(shandle, ip, port, pMsg, NULL); + transSendRequest(shandle, pEpSet, pMsg, NULL); } void rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { - char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); - uint32_t port = pEpSet->eps[pEpSet->inUse].port; - transSendRequest(shandle, ip, port, pMsg, pCtx); + transSendRequest(shandle, pEpSet, pMsg, pCtx); } void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { - char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); - uint32_t port = pEpSet->eps[pEpSet->inUse].port; - transSendRecv(shandle, ip, port, pMsg, pRsp); + transSendRecv(shandle, pEpSet, pMsg, pRsp); } void rpcSendResponse(const SRpcMsg* pMsg) { transSendResponse(pMsg); } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b43b8a1e0c..ab57f7d017 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1,5 +1,4 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/* * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 @@ -97,7 +96,7 @@ static void cliSendCb(uv_write_t* req, int status); static void cliConnCb(uv_connect_t* req, int status); static void cliAsyncCb(uv_async_t* handle); -static void cliAppCb(SCliConn* pConn, STransMsg* pMsg); +static int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg); static SCliConn* cliCreateConn(SCliThrdObj* thrd); static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/); @@ -227,6 +226,9 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); #define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) +#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) +#define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) + static void* cliWorkThread(void* arg); bool cliMaySendCachedMsg(SCliConn* conn) { @@ -311,14 +313,10 @@ void cliHandleResp(SCliConn* conn) { return; } - if (pCtx == NULL || pCtx->pSem == NULL) { - tTrace("%s cli conn %p handle resp", pTransInst->label, conn); - cliAppCb(conn, &transMsg); - //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); - } else { - tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, conn); - memcpy((char*)pCtx->pRsp, (char*)&transMsg, sizeof(transMsg)); - tsem_post(pCtx->pSem); + int ret = cliAppCb(conn, &transMsg, pMsg); + if (ret != 0) { + tTrace("try to send req to next node"); + return; } destroyCmsg(pMsg); @@ -375,17 +373,15 @@ void cliHandleExcept(SCliConn* pConn) { } if (pCtx == NULL || pCtx->pSem == NULL) { - tTrace("%s cli conn %p handle except", pTransInst->label, pConn); if (transMsg.ahandle == NULL) { once = true; continue; } - cliAppCb(pConn, &transMsg); - //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); - } else { - tTrace("%s cli conn(sync) %p handle except", pTransInst->label, pConn); - memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg)); - tsem_post(pCtx->pSem); + } + int ret = cliAppCb(pConn, &transMsg, pMsg); + if (ret != 0) { + tTrace("try to send req to next node"); + return; } destroyCmsg(pMsg); tTrace("%s cli conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn); @@ -695,7 +691,7 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { } } else { STransConnCtx* pCtx = pMsg->ctx; - conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); + conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); if (conn != NULL) { tTrace("%s cli conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); } else { @@ -719,10 +715,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); - // tTrace("%s cli conn %p queue msg size %d", ((STrans*)pThrd->pTransInst)->label, conn, 2); - // return; - //} - // transDestroyBuffer(&conn->readBuf); cliSend(conn); } else { conn = cliCreateConn(pThrd); @@ -730,8 +722,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { transQueuePush(&conn->cliMsgs, pMsg); conn->hThrdIdx = pCtx->hThrdIdx; - conn->ip = strdup(pMsg->ctx->ip); - conn->port = pMsg->ctx->port; + conn->ip = strdup(EPSET_GET_INUSE_IP(&pCtx->epSet)); + conn->port = EPSET_GET_INUSE_PORT(&pCtx->epSet); int ret = transSetConnOption((uv_tcp_t*)conn->stream); if (ret) { @@ -743,10 +735,14 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { addr.sin_family = AF_INET; addr.sin_addr.s_addr = taosGetIpv4FromFqdn(conn->ip); addr.sin_port = (uint16_t)htons((uint16_t)conn->port); - // uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); - // handle error in callback if fail to connect - tTrace("%s cli conn %p try to connect to %s:%d", pTransInst->label, conn, pMsg->ctx->ip, pMsg->ctx->port); - uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); + tTrace("%s cli conn %p try to connect to %s:%d", pTransInst->label, conn, conn->ip, conn->port); + ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); + if (ret != 0) { + tTrace("%s cli conn %p failed to connect to %s:%d, reason: %s", pTransInst->label, conn, conn->ip, conn->port, + uv_err_name(ret)); + cliHandleExcept(conn); + return; + } } } static void cliAsyncCb(uv_async_t* handle) { @@ -856,12 +852,10 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { } static void transDestroyConnCtx(STransConnCtx* ctx) { - if (ctx != NULL) { - taosMemoryFree(ctx->ip); - } + // taosMemoryFree(ctx); } -// + void cliSendQuit(SCliThrdObj* thrd) { // cli can stop gracefully SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); @@ -881,17 +875,58 @@ int cliRBChoseIdx(STrans* pTransInst) { } return index % pTransInst->numOfThreads; } -void cliAppCb(SCliConn* pConn, STransMsg* transMsg) { +int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SCliThrdObj* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; - if (transMsg->code == TSDB_CODE_RPC_REDIRECT && pTransInst->retry != NULL) { - SMEpSet emsg = {0}; - tDeserializeSMEpSet(transMsg->pCont, transMsg->contLen, &emsg); - pTransInst->retry(pTransInst, transMsg, &(emsg.epSet)); - } else { - pTransInst->cfp(pTransInst->parent, transMsg, NULL); + if (pMsg == NULL || pMsg->ctx == NULL) { + tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); + pTransInst->cfp(pTransInst->parent, pResp, NULL); + return 0; } + + STransConnCtx* pCtx = pMsg->ctx; + SEpSet* pEpSet = &pCtx->epSet; + /* + * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL + */ + tmsg_t msgType = pCtx->msgType; + if ((pTransInst->retry != NULL && (pTransInst->retry(pResp->code))) || + ((pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && msgType == TDMT_MND_CONNECT)) { + pCtx->retryCount += 1; + pMsg->st = taosGetTimestampUs(); + if (msgType == TDMT_MND_CONNECT && pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (pCtx->retryCount < pEpSet->numOfEps) { + pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + cliHandleReq(pMsg, pThrd); + cliDestroy((uv_handle_t*)pConn->stream); + return -1; + } + + } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { + if (pResp->contLen == 0) { + pEpSet->inUse = (pEpSet->inUse++) % pEpSet->numOfEps; + } else { + SMEpSet emsg = {0}; + tDeserializeSMEpSet(pResp->pCont, pResp->contLen, &emsg); + pCtx->epSet = emsg.epSet; + } + cliHandleReq(pMsg, pThrd); + // release pConn + addConnToPool(pThrd, pConn); + return -1; + } + } + + if (pCtx->pSem != NULL) { + tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); + memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + tsem_post(pCtx->pSem); + } else { + tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); + pTransInst->cfp(pTransInst->parent, pResp, pEpSet); + } + return 0; } void transCloseClient(void* arg) { @@ -934,18 +969,17 @@ void transReleaseCliHandle(void* handle) { transSendAsync(thrd->asyncPool, &cmsg->q); } -void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* ctx) { +void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { STrans* pTransInst = (STrans*)shandle; - int index = CONN_HOST_THREAD_INDEX((SCliConn*)pMsg->handle); + int index = CONN_HOST_THREAD_INDEX((SCliConn*)pReq->handle); if (index == -1) { index = cliRBChoseIdx(pTransInst); } STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); - pCtx->ahandle = pMsg->ahandle; - pCtx->msgType = pMsg->msgType; - pCtx->ip = strdup(ip); - pCtx->port = port; + pCtx->epSet = *pEpSet; + pCtx->ahandle = pReq->ahandle; + pCtx->msgType = pReq->msgType; pCtx->hThrdIdx = index; if (ctx != NULL) { @@ -955,17 +989,18 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; - cliMsg->msg = *pMsg; + cliMsg->msg = *pReq; cliMsg->st = taosGetTimestampUs(); cliMsg->type = Normal; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; - tDebug("send request at thread:%d %p, dst: %s:%d, app:%p", index, pMsg, ip, port, pMsg->ahandle); + tDebug("send request at thread:%d %p, dst: %s:%d, app:%p", index, pReq, EPSET_GET_INUSE_IP(&pCtx->epSet), + EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->ahandle); ASSERT(transSendAsync(thrd->asyncPool, &(cliMsg->q)) == 0); } -void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) { +void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { STrans* pTransInst = (STrans*)shandle; int index = CONN_HOST_THREAD_INDEX(pReq->handle); if (index == -1) { @@ -973,10 +1008,9 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq } STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); + pCtx->epSet = *pEpSet; pCtx->ahandle = pReq->ahandle; pCtx->msgType = pReq->msgType; - pCtx->ip = strdup(ip); - pCtx->port = port; pCtx->hThrdIdx = index; pCtx->pSem = taosMemoryCalloc(1, sizeof(tsem_t)); pCtx->pRsp = pRsp; @@ -989,6 +1023,9 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq cliMsg->type = Normal; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; + tDebug("send request at thread:%d %p, dst: %s:%d, app:%p", index, pReq, EPSET_GET_INUSE_IP(&pCtx->epSet), + EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->ahandle); + transSendAsync(thrd->asyncPool, &(cliMsg->q)); tsem_t* pSem = pCtx->pSem; tsem_wait(pSem); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index ef595fb0ec..eb42029090 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -93,11 +93,6 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { return false; } -void transConnCtxDestroy(STransConnCtx* ctx) { - taosMemoryFree(ctx->ip); - taosMemoryFree(ctx); -} - void transFreeMsg(void* msg) { if (msg == NULL) { return; @@ -363,10 +358,4 @@ void transQueueDestroy(STransQueue* queue) { transQueueClear(queue); taosArrayDestroy(queue->q); } -// int32_t transGetExHandle() { -// static -//} -// void transThreadOnce() { -// taosThreadOnce(&transModuleInit, ); -//} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index c941cace3b..c02cb07101 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -802,7 +802,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, taosThreadOnce(&transModuleInit, uvInitExHandleMgt); transSrvInst++; - // uvOpenExHandleMgt(10000); for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); @@ -831,6 +830,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, } else { // TODO: clear all other resource later tError("failed to create worker-thread %d", i); + goto End; } } if (false == addHandleToAcceptloop(srv)) { @@ -840,6 +840,8 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, if (err == 0) { tDebug("success to create accept-thread"); } else { + tError("failed to create accept-thread"); + goto End; // clear all resource later } @@ -1078,6 +1080,7 @@ void transRegisterMsg(const STransMsg* msg) { transSendAsync(pThrd->asyncPool, &srvMsg->q); uvReleaseExHandle(refId); return; + _return1: tTrace("server handle %p failed to send to register brokenlink", exh); rpcFreeCont(msg->pCont); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 5b733daec2..73c37c28f7 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -19,7 +19,7 @@ #ifdef USE_TD_MEMORY -#define TD_MEMORY_SYMBOL ('T'<<24|'A'<<16|'O'<<8|'S') +#define TD_MEMORY_SYMBOL ('T' << 24 | 'A' << 16 | 'O' << 8 | 'S') #define TD_MEMORY_STACK_TRACE_DEPTH 10 @@ -28,7 +28,7 @@ typedef struct TdMemoryInfo *TdMemoryInfoPtr; typedef struct TdMemoryInfo { int32_t symbol; int32_t memorySize; - void *stackTrace[TD_MEMORY_STACK_TRACE_DEPTH]; // gdb: disassemble /m 0xXXX + void *stackTrace[TD_MEMORY_STACK_TRACE_DEPTH]; // gdb: disassemble /m 0xXXX // TdMemoryInfoPtr pNext; // TdMemoryInfoPtr pPrev; } TdMemoryInfo; @@ -36,11 +36,11 @@ typedef struct TdMemoryInfo { // static TdMemoryInfoPtr GlobalMemoryPtr = NULL; #ifdef WINDOWS - #define tstrdup(str) _strdup(str) +#define tstrdup(str) _strdup(str) #else - #define tstrdup(str) strdup(str) +#define tstrdup(str) strdup(str) -#include +#include #define STACKCALL __attribute__((regparm(1), noinline)) void **STACKCALL taosGetEbp(void) { @@ -54,9 +54,9 @@ void **STACKCALL taosGetEbp(void) { int32_t taosBackTrace(void **buffer, int32_t size) { int32_t frame = 0; - void **ebp; - void **ret = NULL; - size_t func_frame_distance = 0; + void **ebp; + void **ret = NULL; + size_t func_frame_distance = 0; if (buffer != NULL && size > 0) { ebp = taosGetEbp(); func_frame_distance = (size_t)*ebp - (size_t)ebp; @@ -89,9 +89,9 @@ void *taosMemoryMalloc(int32_t size) { TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)tmp; pTdMemoryInfo->memorySize = size; pTdMemoryInfo->symbol = TD_MEMORY_SYMBOL; - taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH); + taosBackTrace(pTdMemoryInfo->stackTrace, TD_MEMORY_STACK_TRACE_DEPTH); - return (char*)tmp + sizeof(TdMemoryInfo); + return (char *)tmp + sizeof(TdMemoryInfo); #else return malloc(size); #endif @@ -100,15 +100,15 @@ void *taosMemoryMalloc(int32_t size) { void *taosMemoryCalloc(int32_t num, int32_t size) { #ifdef USE_TD_MEMORY int32_t memorySize = num * size; - char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); + char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); if (tmp == NULL) return NULL; TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)tmp; pTdMemoryInfo->memorySize = memorySize; pTdMemoryInfo->symbol = TD_MEMORY_SYMBOL; - taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH); + taosBackTrace(pTdMemoryInfo->stackTrace, TD_MEMORY_STACK_TRACE_DEPTH); - return (char*)tmp + sizeof(TdMemoryInfo); + return (char *)tmp + sizeof(TdMemoryInfo); #else return calloc(num, size); #endif @@ -117,8 +117,8 @@ void *taosMemoryCalloc(int32_t num, int32_t size) { void *taosMemoryRealloc(void *ptr, int32_t size) { #ifdef USE_TD_MEMORY if (ptr == NULL) return taosMemoryMalloc(size); - - TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); TdMemoryInfo tdMemoryInfo; @@ -126,11 +126,11 @@ void *taosMemoryRealloc(void *ptr, int32_t size) { void *tmp = realloc(pTdMemoryInfo, size + sizeof(TdMemoryInfo)); if (tmp == NULL) return NULL; - + memcpy(tmp, &tdMemoryInfo, sizeof(TdMemoryInfo)); ((TdMemoryInfoPtr)tmp)->memorySize = size; - return (char*)tmp + sizeof(TdMemoryInfo); + return (char *)tmp + sizeof(TdMemoryInfo); #else return realloc(ptr, size); #endif @@ -139,29 +139,26 @@ void *taosMemoryRealloc(void *ptr, int32_t size) { void *taosMemoryStrDup(void *ptr) { #ifdef USE_TD_MEMORY if (ptr == NULL) return NULL; - - TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); void *tmp = tstrdup((const char *)pTdMemoryInfo); if (tmp == NULL) return NULL; - - memcpy(tmp, pTdMemoryInfo, sizeof(TdMemoryInfo)); - taosBackTrace(((TdMemoryInfoPtr)tmp)->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH); - return (char*)tmp + sizeof(TdMemoryInfo); + memcpy(tmp, pTdMemoryInfo, sizeof(TdMemoryInfo)); + taosBackTrace(((TdMemoryInfoPtr)tmp)->stackTrace, TD_MEMORY_STACK_TRACE_DEPTH); + + return (char *)tmp + sizeof(TdMemoryInfo); #else return tstrdup((const char *)ptr); #endif } - void taosMemoryFree(void *ptr) { - if (ptr == NULL) return; - #ifdef USE_TD_MEMORY - TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); - if(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL) { + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); + if (pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL) { pTdMemoryInfo->memorySize = 0; // memset(pTdMemoryInfo, 0, sizeof(TdMemoryInfo)); free(pTdMemoryInfo); @@ -177,7 +174,7 @@ int32_t taosMemorySize(void *ptr) { if (ptr == NULL) return 0; #ifdef USE_TD_MEMORY - TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); return pTdMemoryInfo->memorySize; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 6ec6d15028..72ff6179db 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -13,6 +13,7 @@ ./test.sh -f tsim/db/basic6.sim ./test.sh -f tsim/db/basic7.sim ./test.sh -f tsim/db/error1.sim +./test.sh -f tsim/db/taosdlog.sim # ---- dnode ./test.sh -f tsim/dnode/basic1.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index ec847dedbb..da295f640e 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -128,6 +128,7 @@ echo "debugFlag 0" >> $TAOS_CFG echo "mDebugFlag 143" >> $TAOS_CFG echo "dDebugFlag 143" >> $TAOS_CFG echo "vDebugFlag 143" >> $TAOS_CFG +echo "tqDebugFlag 143" >> $TAOS_CFG echo "tsdbDebugFlag 143" >> $TAOS_CFG echo "cDebugFlag 143" >> $TAOS_CFG echo "jniDebugFlag 143" >> $TAOS_CFG diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 57324367b5..3b61b907de 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -126,6 +126,49 @@ if $data16_db != ns then # precision return -1 endi +sleep 3000 +sql show db.vgroups +if $data[0][4] == LEADER then + if $data[0][6] != FOLLOWER then + return -1 + endi + if $data[0][8] != FOLLOWER then + return -1 + endi +endi +if $data[0][6] == LEADER then + if $data[0][4] != FOLLOWER then + return -1 + endi + if $data[0][8] != FOLLOWER then + return -1 + endi +endi +if $data[0][8] == LEADER then + if $data[0][4] != FOLLOWER then + return -1 + endi + if $data[0][6] != FOLLOWER then + return -1 + endi +endi + +if $data[0][4] != LEADER then + if $data[0][4] != FOLLOWER then + return -1 + endi +endi +if $data[0][6] != LEADER then + if $data[0][6] != FOLLOWER then + return -1 + endi +endi +if $data[0][8] != LEADER then + if $data[0][8] != FOLLOWER then + return -1 + endi +endi + print ============== not support modify options: name, create_time, vgroups, ntables sql_error alter database db name dba sql_error alter database db create_time "2022-03-03 15:08:13.329" diff --git a/tests/script/tsim/db/taosdlog.sim b/tests/script/tsim/db/taosdlog.sim new file mode 100644 index 0000000000..c0a0c2b519 --- /dev/null +++ b/tests/script/tsim/db/taosdlog.sim @@ -0,0 +1,31 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +system rm -rf ../../sim/dnode1/log + +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database +sql create database d1 vgroups 2 +sql show databases +if $rows != 3 then + return -1 +endi + +print =============== restart + +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 2000 +system rm -rf ../../sim/dnode1/log +system sh/exec.sh -n dnode1 -s start +sleep 2000 + +print =============== show databases +sql create database d2 vgroups 6 +sql show databases +if $rows != 4 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py new file mode 100644 index 0000000000..f9f80bb910 --- /dev/null +++ b/tests/system-test/0-others/taosdlog.py @@ -0,0 +1,65 @@ +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + tdSql.query("create user testpy pass 'testpy'") + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + logPath = buildPath + "/../sim/dnode1/log" + tdLog.info("log path: %s" % logPath) + + tdDnodes.stop(1) + time.sleep(2) + tdSql.error("show databases") + os.system("rm -rf %s" % logPath) + if os.path.exists(logPath) == True: + tdLog.exit("log pat still exist!") + + tdDnodes.start(1) + time.sleep(2) + if os.path.exists(logPath) != True: + tdLog.exit("log pat is not generated!") + + tdSql.query("show databases") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index af28373510..d218bbf373 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -85,7 +85,6 @@ typedef struct { TAOS* conn; TdThread pid; tsem_t cancelSem; - int64_t result; } SShellObj; // shellArguments.c diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1f036ab25b..befcfc0731 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -213,13 +213,10 @@ void shellRunSingleCommandImp(char *command) { return; } - int64_t oresult = atomic_load_64(&shell.result); - if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - atomic_store_64(&shell.result, 0); taos_free_result(pSql); return; @@ -230,10 +227,7 @@ void shellRunSingleCommandImp(char *command) { int32_t error_no = 0; int32_t numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); - if (numOfRows < 0) { - atomic_store_64(&shell.result, 0); - return; - } + if (numOfRows < 0) return; et = taosGetTimestampUs(); if (error_no == 0) { @@ -250,8 +244,6 @@ void shellRunSingleCommandImp(char *command) { } printf("\n"); - - atomic_store_64(&shell.result, 0); } char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { @@ -398,7 +390,6 @@ int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres) { row = taos_fetch_row(tres); } while (row != NULL); - atomic_store_64(&shell.result, 0); taosCloseFile(&pFile); return numOfRows; @@ -766,7 +757,6 @@ void shellWriteHistory() { void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); - atomic_store_ptr(&shell.result, 0); fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6); taos_free_result(tres); } @@ -872,7 +862,6 @@ void shellGetGrantInfo() { fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\n", serverVersion, sinfo, expiretime); } - atomic_store_64(&shell.result, 0); taos_free_result(tres); }