diff --git a/Jenkinsfile b/Jenkinsfile index 23dc54d963..fc2b3562c1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -105,15 +105,17 @@ pipeline { abort_previous() abortPreviousBuilds() } - pre_test() - sh''' - cd ${WKC}/tests - ./test-all.sh b1fq - ''' - sh''' - cd ${WKC}/debug - ctest - ''' + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh''' + cd ${WKC}/tests + ./test-all.sh b1fq + ''' + sh''' + cd ${WKC}/debug + ctest + ''' + } } } } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 2519f4247d..632d99878c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -133,47 +133,18 @@ typedef enum _mgmt_table { #define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) -typedef struct SKlv { +typedef struct SKv { int32_t keyLen; int32_t valueLen; void* key; void* value; -} SKlv; +} SKv; -static FORCE_INLINE int taosEncodeSKlv(void** buf, const SKlv* pKlv) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKlv->keyLen); - tlen += taosEncodeFixedI32(buf, pKlv->valueLen); - tlen += taosEncodeBinary(buf, pKlv->key, pKlv->keyLen); - tlen += taosEncodeBinary(buf, pKlv->value, pKlv->valueLen); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSKlv(void* buf, SKlv* pKlv) { - buf = taosDecodeFixedI32(buf, &pKlv->keyLen); - buf = taosDecodeFixedI32(buf, &pKlv->valueLen); - buf = taosDecodeBinary(buf, &pKlv->key, pKlv->keyLen); - buf = taosDecodeBinary(buf, &pKlv->value, pKlv->valueLen); - return buf; -} typedef struct SClientHbKey { int32_t connId; int32_t hbType; } SClientHbKey; -static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKey->connId); - tlen += taosEncodeFixedI32(buf, pKey->hbType); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { - buf = taosDecodeFixedI32(buf, &pKey->connId); - buf = taosDecodeFixedI32(buf, &pKey->hbType); - return buf; -} - typedef struct SClientHbReq { SClientHbKey connKey; SHashObj* info; // hash @@ -184,9 +155,6 @@ typedef struct SClientHbBatchReq { SArray* reqs; // SArray } SClientHbBatchReq; -int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); -void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq); - typedef struct SClientHbRsp { SClientHbKey connKey; int32_t status; @@ -200,6 +168,58 @@ typedef struct SClientHbBatchRsp { SArray* rsps; // SArray } SClientHbBatchRsp; +static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { + return taosIntHash_64(key, keyLen); +} + +int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); +void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq); + +static FORCE_INLINE void tFreeClientHbReq(void *pReq) { + SClientHbReq* req = (SClientHbReq*)pReq; + taosHashCleanup(req->info); + free(pReq); +} + +int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); +void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); + +static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) { + SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; + taosArrayDestroyEx(req->reqs, tFreeClientHbReq); + free(pReq); +} + +static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKv->keyLen); + tlen += taosEncodeFixedI32(buf, pKv->valueLen); + tlen += taosEncodeBinary(buf, pKv->key, pKv->keyLen); + tlen += taosEncodeBinary(buf, pKv->value, pKv->valueLen); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSKv(void* buf, SKv* pKv) { + buf = taosDecodeFixedI32(buf, &pKv->keyLen); + buf = taosDecodeFixedI32(buf, &pKv->valueLen); + buf = taosDecodeBinary(buf, &pKv->key, pKv->keyLen); + buf = taosDecodeBinary(buf, &pKv->value, pKv->valueLen); + return buf; +} + +static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKey->connId); + tlen += taosEncodeFixedI32(buf, pKey->hbType); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { + buf = taosDecodeFixedI32(buf, &pKey->connId); + buf = taosDecodeFixedI32(buf, &pKey->hbType); + return buf; +} + typedef struct SBuildTableMetaInput { int32_t vgId; char* dbName; @@ -805,19 +825,19 @@ typedef struct { int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; -} SCreateVnodeMsg, SAlterVnodeMsg; +} SCreateVnodeReq, SAlterVnodeReq; typedef struct { int32_t vgId; int32_t dnodeId; - char db[TSDB_DB_FNAME_LEN]; uint64_t dbUid; -} SDropVnodeMsg, SSyncVnodeMsg, SCompactVnodeMsg; + char db[TSDB_DB_FNAME_LEN]; +} SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq; typedef struct { int32_t vgId; int8_t accessState; -} SAuthVnodeMsg; +} SAuthVnodeReq; typedef struct { SMsgHead header; diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index af56d69b11..f3ad0b0176 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -32,6 +32,8 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SVnode SVnode; typedef struct SVnodeCfg { + int32_t vgId; + /** vnode buffer pool options */ struct { /** write buffer size */ diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index c3c7d740f7..17c11b5d09 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -20,16 +20,16 @@ extern "C" { #endif -typedef void* qinfo_t; +typedef void* qTaskInfo_t; /** * create the qinfo object according to QueryTableMsg * @param tsdb * @param pQueryTableMsg - * @param qinfo + * @param pTaskInfo * @return */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableInfo* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId); +int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryTableMsg, qTaskInfo_t* pTaskInfo, uint64_t qId); /** * the main query execution function, including query on both table and multiple tables, @@ -38,7 +38,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableInfo* pQueryTableM * @param qinfo * @return */ -bool qTableQuery(qinfo_t qinfo, uint64_t *qId); +bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -48,7 +48,7 @@ bool qTableQuery(qinfo_t qinfo, uint64_t *qId); * @param qinfo * @return */ -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext); +int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext); /** * @@ -60,41 +60,41 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex * @param contLen payload length * @return */ -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); +int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); /** * return the transporter context (RPC) * @param qinfo * @return */ -void* qGetResultRetrieveMsg(qinfo_t qinfo); +void* qGetResultRetrieveMsg(qTaskInfo_t qinfo); /** * kill the ongoing query and free the query handle and corresponding resources automatically * @param qinfo qhandle * @return */ -int32_t qKillQuery(qinfo_t qinfo); +int32_t qKillTask(qTaskInfo_t qinfo); /** * return whether query is completed or not * @param qinfo * @return */ -int32_t qIsQueryCompleted(qinfo_t qinfo); +int32_t qIsQueryCompleted(qTaskInfo_t qinfo); /** * destroy query info structure * @param qHandle */ -void qDestroyQueryInfo(qinfo_t qHandle); +void qDestroyTask(qTaskInfo_t qHandle); /** * Get the queried table uid * @param qHandle * @return */ -int64_t qGetQueriedTableUid(qinfo_t qHandle); +int64_t qGetQueriedTableUid(qTaskInfo_t qHandle); /** * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks. @@ -121,7 +121,7 @@ int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGro * @param type operation type: ADD|DROP * @return */ -int32_t qUpdateQueriedTableIdList(qinfo_t qinfo, int64_t uid, int32_t type); +int32_t qUpdateQueriedTableIdList(qTaskInfo_t qinfo, int64_t uid, int32_t type); //================================================================================================ // query handle management @@ -130,13 +130,13 @@ int32_t qUpdateQueriedTableIdList(qinfo_t qinfo, int64_t uid, int32_t type); * @param vgId * @return */ -void* qOpenQueryMgmt(int32_t vgId); +void* qOpenTaskMgmt(int32_t vgId); /** * broadcast the close information and wait for all query stop. * @param pExecutor */ -void qQueryMgmtNotifyClosed(void* pExecutor); +void qTaskMgmtNotifyClosing(void* pExecutor); /** * Re-open the query handle management module when opening the vnode again. @@ -148,7 +148,7 @@ void qQueryMgmtReOpen(void *pExecutor); * Close query mgmt and clean up resources. * @param pExecutor */ -void qCleanupQueryMgmt(void* pExecutor); +void qCleanupTaskMgmt(void* pExecutor); /** * Add the query into the query mgmt object @@ -157,7 +157,7 @@ void qCleanupQueryMgmt(void* pExecutor); * @param qInfo * @return */ -void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo); +void** qRegisterTask(void* pMgmt, uint64_t qId, void *qInfo); /** * acquire the query handle according to the key from query mgmt object. @@ -165,7 +165,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo); * @param key * @return */ -void** qAcquireQInfo(void* pMgmt, uint64_t key); +void** qAcquireTask(void* pMgmt, uint64_t key); /** * release the query handle and decrease the reference count in cache @@ -174,7 +174,7 @@ void** qAcquireQInfo(void* pMgmt, uint64_t key); * @param freeHandle * @return */ -void** qReleaseQInfo(void* pMgmt, void* pQInfo); +void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle); /** * De-register the query handle from the management module and free it immediately. diff --git a/include/libs/function/function.h b/include/libs/function/function.h index d7360a81bc..bf2937a220 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -89,7 +89,7 @@ enum { }; enum { - MASTER_SCAN = 0x0u, + MAIN_SCAN = 0x0u, REVERSE_SCAN = 0x1u, REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan MERGE_STAGE = 0x20u, @@ -183,7 +183,6 @@ typedef struct tExprNode { struct {// function node char functionName[FUNCTIONS_NAME_MAX_LENGTH]; -// int32_t functionId; int32_t num; // Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 9e3cfbb0b0..c5da68f0a6 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -23,6 +23,7 @@ extern "C" { #include "query.h" #include "tmsg.h" #include "tarray.h" +#include "trpc.h" #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 @@ -155,7 +156,7 @@ int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** // @subplan subplan to be schedule // @templateId templateId of a group of datasource subplans of this @subplan // @ep one execution location of this group of datasource subplans -int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); +void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); int32_t qExplainQuery(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, char** str); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 92bbb074f1..3d5c74d093 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -104,55 +104,6 @@ typedef struct STableMetaOutput { STableMeta *tbMeta; } STableMetaOutput; -typedef struct SDataBuf { - void *pData; - uint32_t len; -} SDataBuf; - -typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code); -typedef int32_t (*__async_exec_fn_t)(void* param); - -typedef struct SMsgSendInfo { - __async_send_cb_fn_t fp; //async callback function - void *param; - uint64_t requestId; - uint64_t requestObjRefId; - int32_t msgType; - SDataBuf msgInfo; -} SMsgSendInfo; - -typedef struct SQueryNodeAddr{ - int32_t nodeId; //vgId or qnodeId - int8_t inUse; - int8_t numOfEps; - SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; -} SQueryNodeAddr; - -bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); - -int32_t initTaskQueue(); -int32_t cleanupTaskQueue(); - -/** - * - * @param execFn The asynchronously execution function - * @param execParam The parameters of the execFn - * @param code The response code during execution the execFn - * @return - */ -int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); - -/** - * Asynchronously send message to server, after the response received, the callback will be incured. - * - * @param pTransporter - * @param epSet - * @param pTransporterId - * @param pInfo - * @return - */ -int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); - const SSchema* tGetTbnameColumnSchema(); void initQueryModuleMsgHandle(); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index b88d45fbd6..e28098dfbf 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -84,6 +84,55 @@ void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp) int rpcReportProgress(void *pConn, char *pCont, int contLen); void rpcCancelRequest(int64_t rid); +typedef struct SDataBuf { + void *pData; + uint32_t len; +} SDataBuf; + +typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code); +typedef int32_t (*__async_exec_fn_t)(void* param); + +typedef struct SMsgSendInfo { + __async_send_cb_fn_t fp; //async callback function + void *param; + uint64_t requestId; + uint64_t requestObjRefId; + int32_t msgType; + SDataBuf msgInfo; +} SMsgSendInfo; + +typedef struct SQueryNodeAddr{ + int32_t nodeId; //vgId or qnodeId + int8_t inUse; + int8_t numOfEps; + SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; +} SQueryNodeAddr; + +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); + +int32_t initTaskQueue(); +int32_t cleanupTaskQueue(); + +/** + * + * @param execFn The asynchronously execution function + * @param execParam The parameters of the execFn + * @param code The response code during execution the execFn + * @return + */ +int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); + +/** + * Asynchronously send message to server, after the response received, the callback will be incured. + * + * @param pTransporter + * @param epSet + * @param pTransporterId + * @param pInfo + * @return + */ +int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 56f05ec0bf..80241405a6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -277,9 +277,12 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_BNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0452) #define TSDB_CODE_DND_BNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0453) #define TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0454) -#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0460) -#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0461) -#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0462) +#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0460) +#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0461) +#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0462) +#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0463) +#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0464) +#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0465) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") diff --git a/include/util/thash.h b/include/util/thash.h index 06a4ae1664..3a614a73a6 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -214,6 +214,24 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p); int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); +/** + * Get the corresponding key information for a given data in hash table, using memcpy + * @param data + * @param dst + * @return + */ +static FORCE_INLINE int32_t taosHashCopyKey(void *data, void* dst) { + if (NULL == data || NULL == dst) { + return -1; + } + + SHashNode * node = GET_HASH_PNODE(data); + void* key = GET_HASH_NODE_KEY(node); + memcpy(dst, key, node->keyLen); + + return 0; +} + /** * Get the corresponding data length for a given data in hash table * @param data diff --git a/source/client/inc/clientHb.h b/source/client/inc/clientHb.h index 73adb41308..7bc4311b29 100644 --- a/source/client/inc/clientHb.h +++ b/source/client/inc/clientHb.h @@ -18,36 +18,61 @@ #include "thash.h" #include "tmsg.h" +#define HEARTBEAT_INTERVAL 1500 // ms + typedef enum { - mq = 0, - // type can be added here + HEARTBEAT_TYPE_MQ = 0, + // types can be added here // HEARTBEAT_TYPE_MAX } EHbType; typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq); -typedef int32_t (*FGetConnInfo)(SClientHbKey connKey, void* param); + +typedef struct SAppHbMgr { + // statistics + int32_t reportCnt; + int32_t connKeyCnt; + int64_t reportBytes; // not implemented + int64_t startTime; + // ctl + SRWLatch lock; // lock is used in serialization + // connection + void* transporter; + SEpSet epSet; + // info + SHashObj* activeInfo; // hash + SHashObj* getInfoFuncs; // hash +} SAppHbMgr; typedef struct SClientHbMgr { - int8_t inited; - int32_t reportInterval; // unit ms - int32_t stats; - SRWLatch lock; - SHashObj* activeInfo; // hash - SHashObj* getInfoFuncs; // hash - FHbRspHandle handle[HEARTBEAT_TYPE_MAX]; - // input queue + int8_t inited; + // ctl + int8_t threadStop; + pthread_t thread; + pthread_mutex_t lock; // used when app init and cleanup + SArray* appHbMgrs; // SArray one for each cluster + FHbRspHandle handle[HEARTBEAT_TYPE_MAX]; } SClientHbMgr; -static SClientHbMgr clientHbMgr = {0}; +// TODO: embed param into function +// return type: SArray +typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param); +// global, called by mgmt int hbMgrInit(); void hbMgrCleanUp(); -int hbHandleRsp(void* hbMsg); +int hbHandleRsp(SClientHbBatchRsp* hbRsp); +// cluster level +SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet); +void appHbMgrCleanup(SAppHbMgr* pAppHbMgr); -int hbRegisterConn(SClientHbKey connKey, FGetConnInfo func); +// conn level +int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func); +void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); +int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); -int hbAddConnInfo(SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); - +// mq +void hbMgrInitMqHbRspHandle(); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7daa1204d0..9bbd62c1d9 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -14,64 +14,214 @@ */ #include "clientHb.h" +#include "trpc.h" -static int32_t mqHbRspHandle(SClientHbRsp* pReq) { +static SClientHbMgr clientHbMgr = {0}; + +static int32_t hbCreateThread(); +static void hbStopThread(); + +static int32_t hbMqHbRspHandle(SClientHbRsp* pReq) { return 0; } -uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { +void hbMgrInitMqHbRspHandle() { + clientHbMgr.handle[HEARTBEAT_TYPE_MQ] = hbMqHbRspHandle; +} + +static FORCE_INLINE void hbMgrInitHandle() { + // init all handle + hbMgrInitMqHbRspHandle(); +} + +SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { + SClientHbBatchReq* pReq = malloc(sizeof(SClientHbBatchReq)); + if (pReq == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); + pReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); + + void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); + while (pIter != NULL) { + taosArrayPush(pReq->reqs, pIter); + SClientHbReq* pOneReq = pIter; + taosHashClear(pOneReq->info); + + pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); + } + + pIter = taosHashIterate(pAppHbMgr->getInfoFuncs, NULL); + while (pIter != NULL) { + FGetConnInfo getConnInfoFp = (FGetConnInfo)pIter; + SClientHbKey connKey; + taosHashCopyKey(pIter, &connKey); + getConnInfoFp(connKey, NULL); + + pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); + } + + return pReq; +} + +static void* hbThreadFunc(void* param) { + setThreadName("hb"); + while (1) { + int8_t threadStop = atomic_load_8(&clientHbMgr.threadStop); + if(threadStop) { + break; + } + + int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); + for(int i = 0; i < sz; i++) { + SAppHbMgr* pAppHbMgr = taosArrayGet(clientHbMgr.appHbMgrs, i); + SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr); + void* reqStr = NULL; + int tlen = tSerializeSClientHbBatchReq(&reqStr, pReq); + SMsgSendInfo info; + /*info.fp = hbHandleRsp;*/ + + int64_t transporterId = 0; + asyncSendMsgToServer(pAppHbMgr->transporter, &pAppHbMgr->epSet, &transporterId, &info); + tFreeClientHbBatchReq(pReq); + + atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); + taosMsleep(HEARTBEAT_INTERVAL); + } + } + return NULL; +} + +static int32_t hbCreateThread() { + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pthread_attr_destroy(&thAttr); return 0; } -static void hbMgrInitMqHbFunc() { - clientHbMgr.handle[mq] = mqHbRspHandle; +static void hbStopThread() { + atomic_store_8(&clientHbMgr.threadStop, 1); +} + +SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet) { + SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); + if (pAppHbMgr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + // init stat + pAppHbMgr->startTime = taosGetTimestampMs(); + + // init connection info + pAppHbMgr->transporter = transporter; + pAppHbMgr->epSet = epSet; + + // init hash info + pAppHbMgr->activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq; + // init getInfoFunc + pAppHbMgr->getInfoFuncs = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + + taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); + return pAppHbMgr; +} + +void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { + pthread_mutex_lock(&clientHbMgr.lock); + + int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); + for (int i = 0; i < sz; i++) { + SAppHbMgr* pTarget = taosArrayGet(clientHbMgr.appHbMgrs, i); + if (pAppHbMgr == pTarget) { + taosHashCleanup(pTarget->activeInfo); + taosHashCleanup(pTarget->getInfoFuncs); + } + } + + pthread_mutex_unlock(&clientHbMgr.lock); } int hbMgrInit() { - //init once + // init once int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); if (old == 1) return 0; - //init config - clientHbMgr.reportInterval = 1500; + clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void*)); + pthread_mutex_init(&clientHbMgr.lock, NULL); - //init stat - clientHbMgr.stats = 0; - - //init lock - taosInitRWLatch(&clientHbMgr.lock); + // init handle funcs + hbMgrInitHandle(); - //init handle funcs - hbMgrInitMqHbFunc(); + // init backgroud thread + hbCreateThread(); - //init hash info - clientHbMgr.activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - //init getInfoFunc - clientHbMgr.getInfoFuncs = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); return 0; } void hbMgrCleanUp() { + // destroy all appHbMgr int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0); if (old == 0) return; - taosHashCleanup(clientHbMgr.activeInfo); - taosHashCleanup(clientHbMgr.getInfoFuncs); + taosArrayDestroy(clientHbMgr.appHbMgrs); + } -int hbRegisterConn(SClientHbKey connKey, FGetConnInfo func) { - +int hbHandleRsp(SClientHbBatchRsp* hbRsp) { + int64_t reqId = hbRsp->reqId; + int64_t rspId = hbRsp->rspId; + + SArray* rsps = hbRsp->rsps; + int32_t sz = taosArrayGetSize(rsps); + for (int i = 0; i < sz; i++) { + SClientHbRsp* pRsp = taosArrayGet(rsps, i); + if (pRsp->connKey.hbType < HEARTBEAT_TYPE_MAX) { + clientHbMgr.handle[pRsp->connKey.hbType](pRsp); + } else { + // discard rsp + } + } return 0; } -int hbAddConnInfo(SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) { - //lock +int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func) { + // init hash in activeinfo + void* data = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + if (data != NULL) { + return 0; + } + SClientHbReq hbReq; + hbReq.connKey = connKey; + hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); + // init hash + if (func != NULL) { + taosHashPut(pAppHbMgr->getInfoFuncs, &connKey, sizeof(SClientHbKey), func, sizeof(FGetConnInfo)); + } + + atomic_add_fetch_32(&pAppHbMgr->connKeyCnt, 1); + return 0; +} + +void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) { + taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + taosHashRemove(pAppHbMgr->getInfoFuncs, &connKey, sizeof(SClientHbKey)); + atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); +} + +int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) { + // find req by connection id + SClientHbReq* pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + ASSERT(pReq != NULL); + + taosHashPut(pReq->info, key, keyLen, value, valueLen); - //find req by connection id - SClientHbReq* data = taosHashGet(clientHbMgr.activeInfo, &connKey, sizeof(SClientHbKey)); - ASSERT(data != NULL); - taosHashPut(data->info, key, keyLen, value, valueLen); - - //unlock return 0; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a6b04624d7..d18142cebf 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -305,6 +305,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { } else { CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag, &pJob), _return); + pRequest->code = terrno; + return pRequest; } _return: diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c99ca151fc..53f59c7d57 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -89,17 +89,17 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { int tlen = 0; tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); - void *pIter = NULL; - void *data; - SKlv klv; - data = taosHashIterate(pReq->info, pIter); - while (data != NULL) { - taosHashGetKey(data, &klv.key, (size_t *)&klv.keyLen); - klv.valueLen = taosHashGetDataLen(data); - klv.value = data; - taosEncodeSKlv(buf, &klv); + int kvNum = taosHashGetSize(pReq->info); + tlen += taosEncodeFixedI32(buf, kvNum); + SKv kv; + void* pIter = taosHashIterate(pReq->info, pIter); + while (pIter != NULL) { + taosHashGetKey(pIter, &kv.key, (size_t *)&kv.keyLen); + kv.valueLen = taosHashGetDataLen(pIter); + kv.value = pIter; + tlen += taosEncodeSKv(buf, &kv); - data = taosHashIterate(pReq->info, pIter); + pIter = taosHashIterate(pReq->info, pIter); } return tlen; } @@ -109,16 +109,27 @@ void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) { buf = taosDecodeSClientHbKey(buf, &pReq->connKey); // TODO: error handling - if (pReq->info == NULL) { - pReq->info = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + int kvNum; + taosDecodeFixedI32(buf, &kvNum); + pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + for(int i = 0; i < kvNum; i++) { + SKv kv; + buf = taosDecodeSKv(buf, &kv); + taosHashPut(pReq->info, kv.key, kv.keyLen, kv.value, kv.valueLen); } - SKlv klv; - buf = taosDecodeSKlv(buf, &klv); - taosHashPut(pReq->info, klv.key, klv.keyLen, klv.value, klv.valueLen); return buf; } +int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq) { + int tlen = 0; + return tlen; +} + +void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq) { + return buf; +} + int tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int tlen = 0; diff --git a/source/dnode/mgmt/impl/inc/dndVnodes.h b/source/dnode/mgmt/impl/inc/dndVnodes.h index bf5f0122c1..b5fae62959 100644 --- a/source/dnode/mgmt/impl/inc/dndVnodes.h +++ b/source/dnode/mgmt/impl/inc/dndVnodes.h @@ -29,12 +29,12 @@ void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); +int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 61b220158f..15be59a419 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -42,18 +42,13 @@ static SBnode *dndAcquireBnode(SDnode *pDnode) { } static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) { + if (pBnode == NULL) return; + SBnodeMgmt *pMgmt = &pDnode->bmgmt; - int32_t refCount = 0; - taosRLockLatch(&pMgmt->latch); - if (pBnode != NULL) { - refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - } + int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - - if (pBnode != NULL) { - dTrace("release bnode, refCount:%d", refCount); - } + dTrace("release bnode, refCount:%d", refCount); } static int32_t dndReadBnodeFile(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index ae37967b3d..6c23af7f00 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -43,18 +43,13 @@ static SMnode *dndAcquireMnode(SDnode *pDnode) { } static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) { + if (pMnode == NULL) return; + SMnodeMgmt *pMgmt = &pDnode->mmgmt; - int32_t refCount = 0; - taosRLockLatch(&pMgmt->latch); - if (pMnode != NULL) { - refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - } + int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - - if (pMnode != NULL) { - dTrace("release mnode, refCount:%d", refCount); - } + dTrace("release mnode, refCount:%d", refCount); } static int32_t dndReadMnodeFile(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 3deee93e29..9d2f623c45 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -42,18 +42,13 @@ static SQnode *dndAcquireQnode(SDnode *pDnode) { } static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode) { + if (pQnode == NULL) return; + SQnodeMgmt *pMgmt = &pDnode->qmgmt; - int32_t refCount = 0; - taosRLockLatch(&pMgmt->latch); - if (pQnode != NULL) { - refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - } + int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - - if (pQnode != NULL) { - dTrace("release qnode, refCount:%d", refCount); - } + dTrace("release qnode, refCount:%d", refCount); } static int32_t dndReadQnodeFile(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index ab4e38bfb2..00435d4c3e 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -42,18 +42,13 @@ static SSnode *dndAcquireSnode(SDnode *pDnode) { } static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) { + if (pSnode == NULL) return; + SSnodeMgmt *pMgmt = &pDnode->smgmt; - int32_t refCount = 0; - taosRLockLatch(&pMgmt->latch); - if (pSnode != NULL) { - refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - } + int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - - if (pSnode != NULL) { - dTrace("release snode, refCount:%d", refCount); - } + dTrace("release snode, refCount:%d", refCount); } static int32_t dndReadSnodeFile(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 6427ab080a..9a6d3cb20b 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -40,7 +40,7 @@ typedef struct { STaosQueue *pSyncQ; STaosQueue *pApplyQ; STaosQueue *pQueryQ; - STaosQueue* pFetchQ; + STaosQueue *pFetchQ; } SVnodeObj; typedef struct { @@ -53,22 +53,8 @@ typedef struct { SWrapperCfg *pCfgs; } SVnodeThread; -static int32_t dndInitVnodeReadWorker(SDnode *pDnode); -static int32_t dndInitVnodeWriteWorker(SDnode *pDnode); -static int32_t dndInitVnodeSyncWorker(SDnode *pDnode); -static void dndCleanupVnodeReadWorker(SDnode *pDnode); -static void dndCleanupVnodeWriteWorker(SDnode *pDnode); -static void dndCleanupVnodeSyncWorker(SDnode *pDnode); -static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); +static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); @@ -117,11 +103,9 @@ static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { if (pVnode == NULL) return; SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosRLockLatch(&pMgmt->latch); int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); } @@ -134,7 +118,7 @@ static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { } pVnode->vgId = pCfg->vgId; - pVnode->refCount = 1; + pVnode->refCount = 0; pVnode->dropped = 0; pVnode->accessState = TSDB_VN_ALL_ACCCESS; pVnode->pImpl = pImpl; @@ -148,23 +132,8 @@ static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { return -1; } - if (dndAllocVnodeQueryQueue(pDnode, pVnode) != 0) { - return -1; - } - - if (dndAllocVnodeFetchQueue(pDnode, pVnode) != 0) { - return -1; - } - - if (dndAllocVnodeWriteQueue(pDnode, pVnode) != 0) { - return -1; - } - - if (dndAllocVnodeApplyQueue(pDnode, pVnode) != 0) { - return -1; - } - - if (dndAllocVnodeSyncQueue(pDnode, pVnode) != 0) { + if (dndAllocVnodeQueue(pDnode, pVnode) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -192,15 +161,12 @@ static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) { while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); - dndFreeVnodeQueryQueue(pDnode, pVnode); - dndFreeVnodeFetchQueue(pDnode, pVnode); - dndFreeVnodeWriteQueue(pDnode, pVnode); - dndFreeVnodeApplyQueue(pDnode, pVnode); - dndFreeVnodeSyncQueue(pDnode, pVnode); - + dndFreeVnodeQueue(pDnode, pVnode); vnodeClose(pVnode->pImpl); pVnode->pImpl = NULL; + dDebug("vgId:%d, vnode is closed", pVnode->vgId); + free(pVnode->path); free(pVnode->db); free(pVnode); @@ -274,59 +240,57 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ } int32_t vnodesNum = cJSON_GetArraySize(vnodes); - if (vnodesNum <= 0) { - dError("failed to read %s since vnodes size:%d invalid", file, vnodesNum); - goto PRASE_VNODE_OVER; + if (vnodesNum > 0) { + pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); + if (pCfgs == NULL) { + dError("failed to read %s since out of memory", file); + goto PRASE_VNODE_OVER; + } + + for (int32_t i = 0; i < vnodesNum; ++i) { + cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + SWrapperCfg *pCfg = &pCfgs[i]; + + cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); + if (!vgId || vgId->type != cJSON_Number) { + dError("failed to read %s since vgId not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgId = vgId->valueint; + snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId); + + cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dropped = dropped->valueint; + + cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); + if (!vgVersion || vgVersion->type != cJSON_Number) { + dError("failed to read %s since vgVersion not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgVersion = vgVersion->valueint; + + cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); + if (!dbUid || dbUid->type != cJSON_String) { + dError("failed to read %s since dbUid not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dbUid = atoll(dbUid->valuestring); + + cJSON *db = cJSON_GetObjectItem(vnode, "db"); + if (!db || db->type != cJSON_String) { + dError("failed to read %s since db not found", file); + goto PRASE_VNODE_OVER; + } + tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN); + } + + *ppCfgs = pCfgs; } - pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); - if (pCfgs == NULL) { - dError("failed to read %s since out of memory", file); - goto PRASE_VNODE_OVER; - } - - for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON *vnode = cJSON_GetArrayItem(vnodes, i); - SWrapperCfg *pCfg = &pCfgs[i]; - - cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_Number) { - dError("failed to read %s since vgId not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->vgId = vgId->valueint; - snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId); - - cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->dropped = dropped->valueint; - - cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); - if (!vgVersion || vgVersion->type != cJSON_Number) { - dError("failed to read %s since vgVersion not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->vgVersion = vgVersion->valueint; - - cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); - if (!dbUid || dbUid->type != cJSON_String) { - dError("failed to read %s since dbUid not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->dbUid = atoll(dbUid->valuestring); - - cJSON *db = cJSON_GetObjectItem(vnode, "db"); - if (!db || db->type != cJSON_String) { - dError("failed to read %s since db not found", file); - goto PRASE_VNODE_OVER; - } - tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN); - } - - *ppCfgs = pCfgs; *numOfVnodes = vnodesNum; code = 0; dInfo("succcessed to read file %s", file); @@ -412,7 +376,10 @@ static void *dnodeOpenVnodeFunc(void *param) { pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - SVnode *pImpl = vnodeOpen(pCfg->path, NULL); + SVnodeCfg vnodeCfg = {0}; + vnodeCfg.vgId = pCfg->vgId; + + SVnode *pImpl = vnodeOpen(pCfg->path, &vnodeCfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; @@ -508,7 +475,6 @@ static void dndCloseVnodes(SDnode *pDnode) { SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); for (int32_t i = 0; i < numOfVnodes; ++i) { - dndReleaseVnode(pDnode, pVnodes[i]); dndCloseVnode(pDnode, pVnodes[i]); } @@ -524,8 +490,8 @@ static void dndCloseVnodes(SDnode *pDnode) { dInfo("total vnodes:%d are all closed", numOfVnodes); } -static SCreateVnodeMsg *dndParseCreateVnodeReq(SRpcMsg *rpcMsg) { - SCreateVnodeMsg *pCreate = rpcMsg->pCont; +static SCreateVnodeReq *dndParseCreateVnodeReq(SRpcMsg *pReq) { + SCreateVnodeReq *pCreate = pReq->pCont; pCreate->vgId = htonl(pCreate->vgId); pCreate->dnodeId = htonl(pCreate->dnodeId); pCreate->dbUid = htobe64(pCreate->dbUid); @@ -549,7 +515,8 @@ static SCreateVnodeMsg *dndParseCreateVnodeReq(SRpcMsg *rpcMsg) { return pCreate; } -static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) { +static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { + pCfg->vgId = pCreate->vgId; pCfg->wsize = pCreate->cacheBlockSize; pCfg->ssize = pCreate->cacheBlockSize; pCfg->wsize = pCreate->cacheBlockSize; @@ -572,7 +539,7 @@ static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) { pCfg->walCfg.vgId = pCreate->vgId; } -static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWrapperCfg *pCfg) { +static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN); pCfg->dbUid = pCreate->dbUid; pCfg->dropped = 0; @@ -581,20 +548,20 @@ static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWra pCfg->vgVersion = pCreate->vgVersion; } -static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) { - SDropVnodeMsg *pDrop = rpcMsg->pCont; +static SDropVnodeReq *dndParseDropVnodeReq(SRpcMsg *pReq) { + SDropVnodeReq *pDrop = pReq->pCont; pDrop->vgId = htonl(pDrop->vgId); return pDrop; } -static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) { - SAuthVnodeMsg *pAuth = rpcMsg->pCont; +static SAuthVnodeReq *dndParseAuthVnodeReq(SRpcMsg *pReq) { + SAuthVnodeReq *pAuth = pReq->pCont; pAuth->vgId = htonl(pAuth->vgId); return pAuth; } -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SCreateVnodeMsg *pCreate = dndParseCreateVnodeReq(rpcMsg); +int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SCreateVnodeReq *pCreate = dndParseCreateVnodeReq(pReq); dDebug("vgId:%d, create vnode req is received", pCreate->vgId); SVnodeCfg vnodeCfg = {0}; @@ -605,18 +572,21 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId); if (pVnode != NULL) { - dDebug("vgId:%d, already exist, return success", pCreate->vgId); + dDebug("vgId:%d, already exist", pCreate->vgId); dndReleaseVnode(pDnode, pVnode); - return 0; + terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; + return -1; } - SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/); + SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { + dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); return -1; } int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl); if (code != 0) { + dError("vgId:%d, failed to open vnode since %s", pCreate->vgId, terrstr()); vnodeClose(pImpl); vnodeDestroy(wrapperCfg.path); terrno = code; @@ -634,23 +604,20 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { return 0; } -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SAlterVnodeMsg *pAlter = (SAlterVnodeMsg *)dndParseCreateVnodeReq(rpcMsg); +int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SAlterVnodeReq *pAlter = (SAlterVnodeReq *)dndParseCreateVnodeReq(pReq); dDebug("vgId:%d, alter vnode req is received", pAlter->vgId); SVnodeCfg vnodeCfg = {0}; dndGenerateVnodeCfg(pAlter, &vnodeCfg); - SWrapperCfg wrapperCfg = {0}; - dndGenerateWrapperCfg(pDnode, pAlter, &wrapperCfg); - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pAlter->vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); - return terrno; + return -1; } - if (wrapperCfg.vgVersion == pVnode->vgVersion) { + if (pAlter->vgVersion == pVnode->vgVersion) { dndReleaseVnode(pDnode, pVnode); dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", pAlter->vgId); return 0; @@ -659,11 +626,11 @@ int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { dError("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); - return terrno; + return -1; } int32_t oldVersion = pVnode->vgVersion; - pVnode->vgVersion = wrapperCfg.vgVersion; + pVnode->vgVersion = pAlter->vgVersion; int32_t code = dndWriteVnodesToFile(pDnode); if (code != 0) { pVnode->vgVersion = oldVersion; @@ -673,8 +640,8 @@ int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { return code; } -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SDropVnodeMsg *pDrop = vnodeParseDropVnodeReq(rpcMsg); +int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SDropVnodeReq *pDrop = dndParseDropVnodeReq(pReq); int32_t vgId = pDrop->vgId; dDebug("vgId:%d, drop vnode req is received", vgId); @@ -682,16 +649,17 @@ int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); - return 0; + terrno = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; + return -1; } pVnode->dropped = 1; if (dndWriteVnodesToFile(pDnode) != 0) { pVnode->dropped = 0; - return terrno; + dndReleaseVnode(pDnode, pVnode); + return -1; } - dndReleaseVnode(pDnode, pVnode); dndCloseVnode(pDnode, pVnode); vnodeClose(pVnode->pImpl); vnodeDestroy(pVnode->path); @@ -700,17 +668,16 @@ int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { return 0; } -int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg); +int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SAuthVnodeReq *pAuth = (SAuthVnodeReq *)dndParseAuthVnodeReq(pReq); - int32_t code = 0; int32_t vgId = pAuth->vgId; dDebug("vgId:%d, auth vnode req is received", vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to auth since %s", vgId, terrstr()); - return terrno; + return -1; } pVnode->accessState = pAuth->accessState; @@ -718,30 +685,30 @@ int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { return 0; } -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg); +int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SSyncVnodeReq *pSync = (SSyncVnodeReq *)dndParseDropVnodeReq(pReq); - int32_t vgId = pAuth->vgId; - dDebug("vgId:%d, auth vnode req is received", vgId); + int32_t vgId = pSync->vgId; + dDebug("vgId:%d, sync vnode req is received", vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { - dDebug("vgId:%d, failed to auth since %s", vgId, terrstr()); - return terrno; + dDebug("vgId:%d, failed to sync since %s", vgId, terrstr()); + return -1; } if (vnodeSync(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to auth vnode since %s", vgId, terrstr()); + dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); - return terrno; + return -1; } dndReleaseVnode(pDnode, pVnode); return 0; } -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SCompactVnodeMsg *pCompact = (SCompactVnodeMsg *)vnodeParseDropVnodeReq(rpcMsg); +int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SCompactVnodeReq *pCompact = (SCompactVnodeReq *)dndParseDropVnodeReq(pReq); int32_t vgId = pCompact->vgId; dDebug("vgId:%d, compact vnode req is received", vgId); @@ -749,13 +716,13 @@ int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); - return terrno; + return -1; } if (vnodeCompact(pVnode->pImpl) != 0) { dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); - return terrno; + return -1; } dndReleaseVnode(pDnode, pVnode); @@ -814,6 +781,7 @@ static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_ for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); + // todo SRpcMsg *pRsp = NULL; (void)vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); } @@ -825,6 +793,7 @@ static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); + // todo SRpcMsg *pRsp = NULL; (void)vnodeProcessSyncReq(pVnode->pImpl, pMsg, &pRsp); } @@ -848,21 +817,25 @@ static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg) } if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); + if (pRpcMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; + rpcSendResponse(&rsp); + } rpcFreeCont(pRpcMsg->pCont); } } static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { - SMsgHead *pHead = (SMsgHead *)pMsg->pCont; + SMsgHead *pHead = pMsg->pCont; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); if (pVnode == NULL) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); } @@ -903,193 +876,96 @@ void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - return -1; - } + if (pVnode == NULL) return -1; int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg); dndReleaseVnode(pDnode, pVnode); return code; } -static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue); - if (pVnode->pQueryQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - pVnode->pQueryQ = NULL; -} - -static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue); - if (pVnode->pFetchQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); - pVnode->pFetchQ = NULL; -} - -static int32_t dndInitVnodeReadWorker(SDnode *pDnode) { +static int32_t dndInitVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; int32_t maxFetchThreads = 4; - float threadsForQuery = MAX(pDnode->opt.numOfCores * pDnode->opt.ratioOfQueryCores, 1); + int32_t minFetchThreads = MIN(maxFetchThreads, pDnode->opt.numOfCores); + int32_t minQueryThreads = MAX((int32_t)(pDnode->opt.numOfCores * pDnode->opt.ratioOfQueryCores), 1); + int32_t maxQueryThreads = minQueryThreads; + int32_t maxWriteThreads = MAX(pDnode->opt.numOfCores, 1); + int32_t maxSyncThreads = MAX(pDnode->opt.numOfCores / 2, 1); SWorkerPool *pPool = &pMgmt->queryPool; pPool->name = "vnode-query"; - pPool->min = (int32_t)threadsForQuery; - pPool->max = pPool->min; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + pPool->min = minQueryThreads; + pPool->max = maxQueryThreads; + if (tWorkerInit(pPool) != 0) return -1; pPool = &pMgmt->fetchPool; pPool->name = "vnode-fetch"; - pPool->min = MIN(maxFetchThreads, pDnode->opt.numOfCores); - pPool->max = pPool->min; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + pPool->min = minFetchThreads; + pPool->max = maxFetchThreads; + if (tWorkerInit(pPool) != 0) return -1; - dDebug("vnode read worker is initialized"); + SMWorkerPool *pMPool = &pMgmt->writePool; + pMPool->name = "vnode-write"; + pMPool->max = maxWriteThreads; + if (tMWorkerInit(pMPool) != 0) return -1; + + pMPool = &pMgmt->syncPool; + pMPool->name = "vnode-sync"; + pMPool->max = maxSyncThreads; + if (tMWorkerInit(pMPool) != 0) return -1; + + dDebug("vnode workers is initialized"); return 0; } -static void dndCleanupVnodeReadWorker(SDnode *pDnode) { +static void dndCleanupVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; tWorkerCleanup(&pMgmt->fetchPool); tWorkerCleanup(&pMgmt->queryPool); - dDebug("vnode close worker is initialized"); -} - -static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue); - if (pVnode->pWriteQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); - pVnode->pWriteQ = NULL; -} - -static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue); - if (pVnode->pApplyQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); - pVnode->pApplyQ = NULL; -} - -static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SMWorkerPool *pPool = &pMgmt->writePool; - pPool->name = "vnode-write"; - pPool->max = pDnode->opt.numOfCores; - if (tMWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("vnode write worker is initialized"); - return 0; -} - -static void dndCleanupVnodeWriteWorker(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; tMWorkerCleanup(&pMgmt->writePool); - dDebug("vnode write worker is closed"); -} - -static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); - if (pVnode->pSyncQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); - pVnode->pSyncQ = NULL; -} - -static int32_t dndInitVnodeSyncWorker(SDnode *pDnode) { - int32_t maxThreads = pDnode->opt.numOfCores / 2; - if (maxThreads < 1) maxThreads = 1; - - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SMWorkerPool *pPool = &pMgmt->syncPool; - pPool->name = "vnode-sync"; - pPool->max = maxThreads; - if (tMWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("vnode sync worker is initialized"); - return 0; -} - -static void dndCleanupVnodeSyncWorker(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; tMWorkerCleanup(&pMgmt->syncPool); - dDebug("vnode sync worker is closed"); + dDebug("vnode workers is closed"); +} + +static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + + pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue); + pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue); + pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); + pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue); + pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue); + + if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || + pVnode->pQueryQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); + tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); + tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); + tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); + pVnode->pWriteQ = NULL; + pVnode->pApplyQ = NULL; + pVnode->pSyncQ = NULL; + pVnode->pFetchQ = NULL; + pVnode->pQueryQ = NULL; } int32_t dndInitVnodes(SDnode *pDnode) { dInfo("dnode-vnodes start to init"); - if (dndInitVnodeReadWorker(pDnode) != 0) { - dError("failed to init vnodes read worker since %s", terrstr()); - return -1; - } - - if (dndInitVnodeWriteWorker(pDnode) != 0) { - dError("failed to init vnodes write worker since %s", terrstr()); - return -1; - } - - if (dndInitVnodeSyncWorker(pDnode) != 0) { - dError("failed to init vnodes sync worker since %s", terrstr()); + if (dndInitVnodeWorkers(pDnode) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + dError("failed to init vnode workers since %s", terrstr()); return -1; } @@ -1105,9 +981,7 @@ int32_t dndInitVnodes(SDnode *pDnode) { void dndCleanupVnodes(SDnode *pDnode) { dInfo("dnode-vnodes start to clean up"); dndCloseVnodes(pDnode); - dndCleanupVnodeReadWorker(pDnode); - dndCleanupVnodeWriteWorker(pDnode); - dndCleanupVnodeSyncWorker(pDnode); + dndCleanupVnodeWorkers(pDnode); dInfo("dnode-vnodes is cleaned up"); } diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 8f721dae9a..ce93a14d3f 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -4,8 +4,5 @@ add_subdirectory(qnode) add_subdirectory(bnode) add_subdirectory(snode) add_subdirectory(mnode) -add_subdirectory(db) -add_subdirectory(stb) -add_subdirectory(vgroup) - +add_subdirectory(vnode) add_subdirectory(sut) diff --git a/source/dnode/mgmt/impl/test/db/CMakeLists.txt b/source/dnode/mgmt/impl/test/db/CMakeLists.txt deleted file mode 100644 index cb9f1600fc..0000000000 --- a/source/dnode/mgmt/impl/test/db/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(. DB_SRC) -add_executable(dnode_test_db ${DB_SRC}) -target_link_libraries( - dnode_test_db - PUBLIC sut -) - -add_test( - NAME dnode_test_db - COMMAND dnode_test_db -) diff --git a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp b/source/dnode/mgmt/impl/test/mnode/dmnode.cpp index 841d00d14d..edaecef49e 100644 --- a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/impl/test/mnode/dmnode.cpp @@ -13,7 +13,7 @@ class DndTestMnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_mnode", 9113); } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_mnode", 9114); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mgmt/impl/test/snode/dsnode.cpp b/source/dnode/mgmt/impl/test/snode/dsnode.cpp index f51f4a0268..019aa1cbbc 100644 --- a/source/dnode/mgmt/impl/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/impl/test/snode/dsnode.cpp @@ -13,7 +13,7 @@ class DndTestSnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_snode", 9112); } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_snode", 9113); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt b/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt deleted file mode 100644 index b864b0593c..0000000000 --- a/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(. VGROUP_SRC) -add_executable(dnode_test_vgroup ${VGROUP_SRC}) -target_link_libraries( - dnode_test_vgroup - PUBLIC sut -) - -add_test( - NAME dnode_test_vgroup - COMMAND dnode_test_vgroup -) diff --git a/source/dnode/mgmt/impl/test/vnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/vnode/CMakeLists.txt new file mode 100644 index 0000000000..6fb8bb4ba4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/vnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. VNODE_SRC) +add_executable(dnode_test_vnode ${VNODE_SRC}) +target_link_libraries( + dnode_test_vnode + PUBLIC sut +) + +add_test( + NAME dnode_test_vnode + COMMAND dnode_test_vnode +) diff --git a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp similarity index 73% rename from source/dnode/mgmt/impl/test/vgroup/vgroup.cpp rename to source/dnode/mgmt/impl/test/vnode/vnode.cpp index 7fa3b4ab61..6b042c3ce6 100644 --- a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -1,7 +1,7 @@ /** * @file db.cpp * @author slguan (slguan@taosdata.com) - * @brief DNODE module vgroup-msg tests + * @brief DNODE module vnode tests * @version 0.1 * @date 2021-12-20 * @@ -11,9 +11,9 @@ #include "sut.h" -class DndTestVgroup : public ::testing::Test { +class DndTestVnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_vgroup", 9150); } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_vnode", 9115); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -23,14 +23,14 @@ class DndTestVgroup : public ::testing::Test { void TearDown() override {} }; -Testbase DndTestVgroup::test; +Testbase DndTestVnode::test; -TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { +TEST_F(DndTestVnode, 01_Create_Restart_Drop_Vnode) { { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SCreateVnodeMsg); + int32_t contLen = sizeof(SCreateVnodeReq); - SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(contLen); + SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen); pReq->vgId = htonl(2); pReq->dnodeId = htonl(1); strcpy(pReq->db, "1.d1"); @@ -57,20 +57,25 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { for (int r = 0; r < pReq->replica; ++r) { SReplica* pReplica = &pReq->replicas[r]; pReplica->id = htonl(1); - pReplica->port = htons(9150); + pReplica->port = htons(9527); } SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); + if (i == 0) { + ASSERT_EQ(pRsp->code, 0); + test.Restart(); + } else { + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED); + } } } { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SAlterVnodeMsg); + int32_t contLen = sizeof(SAlterVnodeReq); - SAlterVnodeMsg* pReq = (SAlterVnodeMsg*)rpcMallocCont(contLen); + SAlterVnodeReq* pReq = (SAlterVnodeReq*)rpcMallocCont(contLen); pReq->vgId = htonl(2); pReq->dnodeId = htonl(1); strcpy(pReq->db, "1.d1"); @@ -97,7 +102,7 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { for (int r = 0; r < pReq->replica; ++r) { SReplica* pReplica = &pReq->replicas[r]; pReplica->id = htonl(1); - pReplica->port = htons(9150); + pReplica->port = htons(9527); } SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_VNODE, pReq, contLen); @@ -108,9 +113,9 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SDropVnodeMsg); + int32_t contLen = sizeof(SDropVnodeReq); - SDropVnodeMsg* pReq = (SDropVnodeMsg*)rpcMallocCont(contLen); + SDropVnodeReq* pReq = (SDropVnodeReq*)rpcMallocCont(contLen); pReq->vgId = htonl(2); pReq->dnodeId = htonl(1); strcpy(pReq->db, "1.d1"); @@ -118,12 +123,17 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropVnodeMsg); + rpcMsg.contLen = sizeof(SDropVnodeReq); rpcMsg.msgType = TDMT_DND_DROP_VNODE; SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_VNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); + if (i == 0) { + ASSERT_EQ(pRsp->code, 0); + test.Restart(); + } else { + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_NOT_DEPLOYED); + } } } } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 6d391450b7..6ab11aa1b4 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -31,8 +31,8 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); -SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); -SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); +SCreateVnodeReq *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); +SDropVnodeReq *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 945a6523ef..aa87eb43a1 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -177,7 +177,7 @@ static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { } static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { - mTrace("acct:%s, perform update action, old_row:%p new_row:%p", pOld->acct, pOld, pNew); + mTrace("acct:%s, perform update action, old row:%p new row:%p", pOld->acct, pOld, pNew); pOld->updateTime = pNew->updateTime; pOld->status = pNew->status; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 6325bbb6fd..dbe925992d 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -155,7 +155,7 @@ static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj) { } static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew) { - mTrace("bnode:%d, perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + mTrace("bnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 8cb98a148f..92b9017627 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -135,7 +135,7 @@ static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster) { } static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOld, SClusterObj *pNew) { - mTrace("cluster:%" PRId64 ", perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + mTrace("cluster:%" PRId64 ", perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 251cbb9217..1a5ea53d3c 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -28,15 +28,15 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); -static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb); -static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg); -static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg); -static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg); -static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg); -static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg); -static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew); +static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq); +static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq); +static int32_t mndProcessDropDbReq(SMnodeMsg *pReq); +static int32_t mndProcessUseDbReq(SMnodeMsg *pReq); +static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq); +static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq); +static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); int32_t mndInitDb(SMnode *pMnode) { @@ -48,12 +48,12 @@ int32_t mndInitDb(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndDbActionUpdate, .deleteFp = (SdbDeleteFp)mndDbActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbMsg); - mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbMsg); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_DB, mndProcessDropDbMsg); - mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbMsg); - mndSetMsgHandle(pMnode, TDMT_MND_SYNC_DB, mndProcessSyncDbMsg); - mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbReq); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_DB, mndProcessDropDbReq); + mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq); + mndSetMsgHandle(pMnode, TDMT_MND_SYNC_DB, mndProcessSyncDbReq); + mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetDbMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); @@ -182,12 +182,12 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { return 0; } -static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { - mTrace("db:%s, perform update action, old_row:%p new_row:%p", pOldDb->name, pOldDb, pNewDb); - pOldDb->updateTime = pNewDb->updateTime; - pOldDb->cfgVersion = pNewDb->cfgVersion; - pOldDb->vgVersion = pNewDb->vgVersion; - memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); +static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { + mTrace("db:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + pOld->updateTime = pNew->updateTime; + pOld->cfgVersion = pNew->cfgVersion; + pOld->vgVersion = pNew->vgVersion; + memcpy(&pOld->cfg, &pNew->cfg, sizeof(SDbCfg)); return 0; } @@ -331,14 +331,15 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) return -1; + SCreateVnodeReq *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup); + if (pReq == NULL) return -1; - action.pCont = pMsg; - action.contLen = sizeof(SCreateVnodeMsg); + action.pCont = pReq; + action.contLen = sizeof(SCreateVnodeReq); action.msgType = TDMT_DND_CREATE_VNODE; + action.acceptableCode = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } } @@ -360,14 +361,15 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) return -1; + SDropVnodeReq *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup); + if (pReq == NULL) return -1; - action.pCont = pMsg; - action.contLen = sizeof(SDropVnodeMsg); + action.pCont = pReq; + action.contLen = sizeof(SDropVnodeReq); action.msgType = TDMT_DND_DROP_VNODE; + action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } } @@ -376,7 +378,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) { +static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pReq, SCreateDbMsg *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; memcpy(dbObj.name, pCreate->db, TSDB_DB_FNAME_LEN); memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); @@ -425,43 +427,17 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat } int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - goto CREATE_DB_OVER; - } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); + if (pTrans == NULL) goto CREATE_DB_OVER; mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); - if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto CREATE_DB_OVER; - } - - if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { - mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); - goto CREATE_DB_OVER; - } - - if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto CREATE_DB_OVER; - } - - if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto CREATE_DB_OVER; - } - - if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto CREATE_DB_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - goto CREATE_DB_OVER; - } + if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; + if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; + if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; + if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; + if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_DB_OVER; code = 0; @@ -471,9 +447,9 @@ CREATE_DB_OVER: return code; } -static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont; +static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SCreateDbMsg *pCreate = pReq->rpcMsg.pCont; pCreate->numOfVgroups = htonl(pCreate->numOfVgroups); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); @@ -502,13 +478,13 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { } } - SUserObj *pOperUser = mndAcquireUser(pMnode, pMsg->user); + SUserObj *pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } - int32_t code = mndCreateDb(pMnode, pMsg, pCreate, pOperUser); + int32_t code = mndCreateDb(pMnode, pReq, pCreate, pOperUser); mndReleaseUser(pMnode, pOperUser); if (code != 0) { @@ -565,8 +541,8 @@ static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { return terrno; } -static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { - SSdbRaw *pRedoRaw = mndDbActionEncode(pOldDb); +static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { + SSdbRaw *pRedoRaw = mndDbActionEncode(pOld); if (pRedoRaw == NULL) return -1; if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_UPDATING) != 0) return -1; @@ -574,8 +550,8 @@ static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pO return 0; } -static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { - SSdbRaw *pCommitRaw = mndDbActionEncode(pNewDb); +static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { + SSdbRaw *pCommitRaw = mndDbActionEncode(pNew); if (pCommitRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; @@ -593,14 +569,14 @@ static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SAlterVnodeMsg *pMsg = (SAlterVnodeMsg *)mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) return -1; + SAlterVnodeReq *pReq = (SAlterVnodeReq *)mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup); + if (pReq == NULL) return -1; - action.pCont = pMsg; - action.contLen = sizeof(SAlterVnodeMsg); + action.pCont = pReq; + action.contLen = sizeof(SAlterVnodeReq); action.msgType = TDMT_DND_ALTER_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } } @@ -608,7 +584,7 @@ static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { +static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -617,8 +593,8 @@ static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; - if (pVgroup->dbUid == pNewDb->uid) { - if (mndBuildUpdateVgroupAction(pMnode, pTrans, pNewDb, pVgroup) != 0) { + if (pVgroup->dbUid == pNew->uid) { + if (mndBuildUpdateVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -631,27 +607,27 @@ static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) { +static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pReq->rpcMsg); if (pTrans == NULL) { - mError("db:%s, failed to update since %s", pOldDb->name, terrstr()); + mError("db:%s, failed to update since %s", pOld->name, terrstr()); return terrno; } - mDebug("trans:%d, used to update db:%s", pTrans->id, pOldDb->name); + mDebug("trans:%d, used to update db:%s", pTrans->id, pOld->name); - if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) { mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); goto UPDATE_DB_OVER; } - if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) { mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); goto UPDATE_DB_OVER; } - if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) { + if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) { mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); goto UPDATE_DB_OVER; } @@ -668,9 +644,9 @@ UPDATE_DB_OVER: return code; } -static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont; +static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SAlterDbMsg *pAlter = pReq->rpcMsg.pCont; pAlter->totalBlocks = htonl(pAlter->totalBlocks); pAlter->daysToKeep0 = htonl(pAlter->daysToKeep0); pAlter->daysToKeep1 = htonl(pAlter->daysToKeep1); @@ -697,7 +673,7 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); - code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj); + code = mndUpdateDb(pMnode, pReq, pDb, &dbObj); mndReleaseDb(pMnode, pDb); if (code != 0) { @@ -757,14 +733,15 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) return -1; + SDropVnodeReq *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup); + if (pReq == NULL) return -1; - action.pCont = pMsg; - action.contLen = sizeof(SCreateVnodeMsg); + action.pCont = pReq; + action.contLen = sizeof(SCreateVnodeReq); action.msgType = TDMT_DND_DROP_VNODE; + action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } } @@ -795,35 +772,17 @@ static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { +static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("db:%s, failed to drop since %s", pDb->name, terrstr()); - return -1; - } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pReq->rpcMsg); + if (pTrans == NULL) goto DROP_DB_OVER; mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); - if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto DROP_DB_OVER; - } - - if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto DROP_DB_OVER; - } - - if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto DROP_DB_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - goto DROP_DB_OVER; - } + if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; + if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; + if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_DB_OVER; code = 0; @@ -832,9 +791,9 @@ DROP_DB_OVER: return code; } -static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SDropDbMsg *pDrop = pMsg->rpcMsg.pCont; +static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SDropDbMsg *pDrop = pReq->rpcMsg.pCont; mDebug("db:%s, start to drop", pDrop->db); @@ -850,7 +809,7 @@ static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { } } - int32_t code = mndDropDb(pMnode, pMsg, pDb); + int32_t code = mndDropDb(pMnode, pReq, pDb); mndReleaseDb(pMnode, pDb); if (code != 0) { @@ -861,16 +820,16 @@ static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; - SUseDbMsg *pUse = pMsg->rpcMsg.pCont; + SUseDbMsg *pUse = pReq->rpcMsg.pCont; pUse->vgVersion = htonl(pUse->vgVersion); SDbObj *pDb = mndAcquireDb(pMnode, pUse->db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - mError("db:%s, failed to process use db msg since %s", pUse->db, terrstr()); + mError("db:%s, failed to process use db req since %s", pUse->db, terrstr()); return -1; } @@ -922,19 +881,19 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { pRsp->vgNum = htonl(vindex); pRsp->hashMethod = pDb->hashMethod; - pMsg->pCont = pRsp; - pMsg->contLen = contLen; + pReq->pCont = pRsp; + pReq->contLen = contLen; mndReleaseDb(pMnode, pDb); return 0; } -static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SSyncDbMsg *pSync = pMsg->rpcMsg.pCont; +static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SSyncDbMsg *pSync = pReq->rpcMsg.pCont; SDbObj *pDb = mndAcquireDb(pMnode, pSync->db); if (pDb == NULL) { - mError("db:%s, failed to process sync db msg since %s", pSync->db, terrstr()); + mError("db:%s, failed to process sync db req since %s", pSync->db, terrstr()); return -1; } @@ -942,12 +901,12 @@ static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont; +static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SCompactDbMsg *pCompact = pReq->rpcMsg.pCont; SDbObj *pDb = mndAcquireDb(pMnode, pCompact->db); if (pDb == NULL) { - mError("db:%s, failed to process compact db msg since %s", pCompact->db, terrstr()); + mError("db:%s, failed to process compact db req since %s", pCompact->db, terrstr()); return -1; } @@ -955,8 +914,8 @@ static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -1096,8 +1055,8 @@ char *mnGetDbStr(char *src) { return pos; } -static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SDbObj *pDb = NULL; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d94078cfe1..1b62a47c91 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -183,7 +183,7 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { } static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew) { - mTrace("dnode:%d, perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + mTrace("dnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; return 0; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index f7a39ce9e5..1c809a9b9a 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -152,7 +152,7 @@ static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { } static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc) { - mTrace("func:%s, perform update action, old_row:%p new_row:%p", pOldFunc->name, pOldFunc, pNewFunc); + mTrace("func:%s, perform update action, old row:%p new row:%p", pOldFunc->name, pOldFunc, pNewFunc); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index c14d1f51f8..2c09b29872 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -208,7 +208,7 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { } static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) { - mTrace("mnode:%d, perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; return 0; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index a223890e8c..88cade08ed 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -155,7 +155,7 @@ static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) { } static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew) { - mTrace("qnode:%d, perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + mTrace("qnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; return 0; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 1b66bedbcc..dabd1f0142 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -155,7 +155,7 @@ static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj) { } static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew) { - mTrace("snode:%d, perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + mTrace("snode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; return 0; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d6b6a07de0..b01c291ce5 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -178,7 +178,7 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { } static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb) { - mTrace("stb:%s, perform update action, old_row:%p new_row:%p", pOldStb->name, pOldStb, pNewStb); + mTrace("stb:%s, perform update action, old row:%p new row:%p", pOldStb->name, pOldStb, pNewStb); atomic_exchange_32(&pOldStb->updateTime, pNewStb->updateTime); atomic_exchange_32(&pOldStb->version, pNewStb->version); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index f030398144..591367c519 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -91,14 +91,15 @@ static int32_t mndRestoreWal(SMnode *pMnode) { if (sdbWriteFile(pSdb) != 0) { goto WAL_RESTORE_OVER; } - } - if (walBeginSnapshot(pWal, sdbVer) < 0) { - goto WAL_RESTORE_OVER; - } + if (walBeginSnapshot(pWal, sdbVer) < 0) { + goto WAL_RESTORE_OVER; + } + + if (walEndSnapshot(pWal) < 0) { + goto WAL_RESTORE_OVER; + } - if (walEndSnapshot(pWal) < 0) { - goto WAL_RESTORE_OVER; } code = 0; @@ -181,4 +182,4 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) { bool mndIsMaster(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; return pMgmt->state == TAOS_SYNC_STATE_LEADER; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index e73b34e688..8c15398215 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -192,7 +192,7 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { } static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { - mTrace("user:%s, perform update action, old_row:%p new_row:%p", pOld->user, pOld, pNew); + mTrace("user:%s, perform update action, old row:%p new row:%p", pOld->user, pOld, pNew); memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); pOld->updateTime = pNew->updateTime; return 0; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index bd17c6d150..c5c58a075d 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -165,7 +165,7 @@ static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup) { } static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { - mTrace("vgId:%d, perform update action, old_row:%p new_row:%p", pOldVgroup->vgId, pOldVgroup, pNewVgroup); + mTrace("vgId:%d, perform update action, old row:%p new row:%p", pOldVgroup->vgId, pOldVgroup, pNewVgroup); pOldVgroup->updateTime = pNewVgroup->updateTime; pOldVgroup->version = pNewVgroup->version; pOldVgroup->hashBegin = pNewVgroup->hashBegin; @@ -189,8 +189,8 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { sdbRelease(pSdb, pVgroup); } -SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SCreateVnodeMsg *pCreate = calloc(1, sizeof(SCreateVnodeMsg)); +SCreateVnodeReq *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { + SCreateVnodeReq *pCreate = calloc(1, sizeof(SCreateVnodeReq)); if (pCreate == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -248,8 +248,8 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb return pCreate; } -SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SDropVnodeMsg *pDrop = calloc(1, sizeof(SDropVnodeMsg)); +SDropVnodeReq *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { + SDropVnodeReq *pDrop = calloc(1, sizeof(SDropVnodeReq)); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 4d37bb368d..3ca35d58a7 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -10,3 +10,5 @@ add_subdirectory(show) add_subdirectory(profile) add_subdirectory(dnode) add_subdirectory(mnode) +add_subdirectory(db) +add_subdirectory(stb) diff --git a/source/dnode/mnode/impl/test/db/CMakeLists.txt b/source/dnode/mnode/impl/test/db/CMakeLists.txt new file mode 100644 index 0000000000..f0abdf152c --- /dev/null +++ b/source/dnode/mnode/impl/test/db/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. DB_SRC) +add_executable(mnode_test_db ${DB_SRC}) +target_link_libraries( + mnode_test_db + PUBLIC sut +) + +add_test( + NAME mnode_test_db + COMMAND mnode_test_db +) diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp similarity index 89% rename from source/dnode/mgmt/impl/test/db/db.cpp rename to source/dnode/mnode/impl/test/db/db.cpp index 3a69ae2305..fd860c87d6 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -1,31 +1,42 @@ /** * @file db.cpp * @author slguan (slguan@taosdata.com) - * @brief DNODE module db-msg tests - * @version 0.1 - * @date 2021-12-15 + * @brief MNODE module db tests + * @version 1.0 + * @date 2022-01-11 * - * @copyright Copyright (c) 2021 + * @copyright Copyright (c) 2022 * */ #include "sut.h" -class DndTestDb : public ::testing::Test { +class MndTestDb : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_db", 9040); } - static void TearDownTestSuite() { test.Cleanup(); } + static void SetUpTestSuite() { + test.Init("/tmp/mnode_test_db", 9030); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9030"; - static Testbase test; + server2.Start("/tmp/mnode_test_db2", fqdn, 9031, firstEp); + } + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; public: void SetUp() override {} void TearDown() override {} }; -Testbase DndTestDb::test; +Testbase MndTestDb::test; +TestServer MndTestDb::server2; -TEST_F(DndTestDb, 01_ShowDb) { +TEST_F(MndTestDb, 01_ShowDb) { test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); CHECK_META("show databases", 18); CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); @@ -51,7 +62,7 @@ TEST_F(DndTestDb, 01_ShowDb) { EXPECT_EQ(test.GetShowRows(), 0); } -TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { +TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { int32_t contLen = sizeof(SCreateDbMsg); @@ -149,7 +160,7 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups - CheckInt32(0); + CheckInt32(0); // tables CheckInt16(1); // replica CheckInt16(2); // quorum CheckInt16(10); // days @@ -177,7 +188,7 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups - CheckInt32(0); + CheckInt32(0); // tables CheckInt16(1); // replica CheckInt16(2); // quorum CheckInt16(10); // days @@ -211,7 +222,7 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { EXPECT_EQ(test.GetShowRows(), 0); } -TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { +TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { { int32_t contLen = sizeof(SCreateDbMsg); @@ -281,7 +292,7 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_EQ(pInfo->numOfEps, 1); SEpAddrMsg* pAddr = &pInfo->epAddr[0]; pAddr->port = htons(pAddr->port); - EXPECT_EQ(pAddr->port, 9040); + EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } @@ -297,8 +308,19 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_EQ(pInfo->numOfEps, 1); SEpAddrMsg* pAddr = &pInfo->epAddr[0]; pAddr->port = htons(pAddr->port); - EXPECT_EQ(pAddr->port, 9040); + EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } } + + { + int32_t contLen = sizeof(SDropDbMsg); + + SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(contLen); + strcpy(pReq->db, "1.d2"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } } diff --git a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt b/source/dnode/mnode/impl/test/stb/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/stb/CMakeLists.txt rename to source/dnode/mnode/impl/test/stb/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp similarity index 95% rename from source/dnode/mgmt/impl/test/stb/stb.cpp rename to source/dnode/mnode/impl/test/stb/stb.cpp index d3362c7a9b..55cc030122 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -1,19 +1,19 @@ /** * @file stb.cpp * @author slguan (slguan@taosdata.com) - * @brief DNODE module db-msg tests - * @version 0.1 - * @date 2021-12-17 + * @brief MNODE module stb tests + * @version 1.0 + * @date 2022-01-12 * - * @copyright Copyright (c) 2021 + * @copyright Copyright (c) 2022 * */ #include "sut.h" -class DndTestStb : public ::testing::Test { +class MndTestStb : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_stb", 9101); } + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_stb", 9034); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -23,9 +23,9 @@ class DndTestStb : public ::testing::Test { void TearDown() override {} }; -Testbase DndTestStb::test; +Testbase MndTestStb::test; -TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { +TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { int32_t contLen = sizeof(SCreateDbMsg); diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 88175646ae..995bed6e0b 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -24,9 +24,9 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; // Set default options - if (pVnodeCfg == NULL) { + //if (pVnodeCfg == NULL) { pVnodeCfg = &defaultVnodeOptions; - } + //} // Validate options if (vnodeValidateOptions(pVnodeCfg) < 0) { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 7e910d5674..2c1bf71638 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -88,37 +88,37 @@ typedef struct SResultRowPool { SArray* pData; // SArray } SResultRowPool; -struct SQueryAttr; -struct SQueryRuntimeEnv; +struct STaskAttr; +struct STaskRuntimeEnv; struct SUdfInfo; -int32_t getOutputInterResultBufSize(struct SQueryAttr* pQueryAttr); +int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr); -size_t getResultRowSize(struct SQueryRuntimeEnv* pRuntimeEnv); +size_t getResultRowSize(struct STaskRuntimeEnv* pRuntimeEnv); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -void resetResultRowInfo(struct SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); +void resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); int32_t initResultRow(SResultRow *pResultRow); void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); -void clearResultRow(struct SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); +void clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr); void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols); -int32_t getRowNumForMultioutput(struct SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable); +int32_t getRowNumForMultioutput(struct STaskAttr* pQueryAttr, bool topBottomQuery, bool stable); static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); return pResultRowInfo->pResult[slot]; } -static FORCE_INLINE char* getPosInResultPage(struct SQueryAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, +static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); @@ -155,7 +155,7 @@ bool hasRemainData(SGroupResInfo* pGroupResInfo); bool incNextGroup(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); -int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset); +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct STaskRuntimeEnv *pRuntimeEnv, int32_t* offset); int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 907fb4d2bf..a268215d3d 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -21,13 +21,14 @@ #include "tvariant.h" #include "thash.h" -//#include "parser.h" #include "executil.h" #include "taosdef.h" #include "tarray.h" #include "tfilter.h" #include "tlockfree.h" #include "tpagedfile.h" +#include "planner.h" + struct SColumnFilterElem; @@ -65,7 +66,6 @@ enum { QUERY_OVER = 0x4u, }; - typedef struct SResultRowCell { uint64_t groupId; SResultRow *pRow; @@ -100,7 +100,7 @@ typedef struct STableQueryInfo { TSKEY lastKey; int32_t groupIndex; // group id in table list SVariant tag; - STimeWindow win; + STimeWindow win; // todo remove it later STSCursor cur; void* pTable; // for retrieve the page id list SResultRowInfo resInfo; @@ -128,31 +128,34 @@ typedef struct { int64_t sumRunTimes; } SOperatorProfResult; -typedef struct SQueryCostInfo { - uint64_t loadStatisTime; - uint64_t loadFileBlockTime; - uint64_t loadDataInCacheTime; - uint64_t loadStatisSize; - uint64_t loadFileBlockSize; - uint64_t loadDataInCacheSize; - - uint64_t loadDataTime; - uint64_t totalRows; - uint64_t totalCheckedRows; - uint32_t totalBlocks; - uint32_t loadBlocks; - uint32_t loadBlockStatis; - uint32_t discardBlocks; - uint64_t elapsedTime; - uint64_t firstStageMergeTime; - uint64_t winInfoSize; - uint64_t tableInfoSize; - uint64_t hashSize; - uint64_t numOfTimeWindows; +typedef struct STaskCostInfo { + int64_t start; + int64_t end; - SArray* queryProfEvents; //SArray - SHashObj* operatorProfResults; //map -} SQueryCostInfo; + uint64_t loadStatisTime; + uint64_t loadFileBlockTime; + uint64_t loadDataInCacheTime; + uint64_t loadStatisSize; + uint64_t loadFileBlockSize; + uint64_t loadDataInCacheSize; + + uint64_t loadDataTime; + uint64_t totalRows; + uint64_t totalCheckedRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockStatis; + uint32_t discardBlocks; + uint64_t elapsedTime; + uint64_t firstStageMergeTime; + uint64_t winInfoSize; + uint64_t tableInfoSize; + uint64_t hashSize; + uint64_t numOfTimeWindows; + + SArray *queryProfEvents; //SArray + SHashObj *operatorProfResults; //map +} STaskCostInfo; typedef struct { int64_t vgroupLimit; @@ -166,7 +169,7 @@ typedef struct { // The basic query information extracted from the SQueryInfo tree to support the // execution of query in a data node. -typedef struct SQueryAttr { +typedef struct STaskAttr { SLimit limit; SLimit slimit; @@ -229,16 +232,40 @@ typedef struct SQueryAttr { STableGroupInfo tableGroupInfo; // table list SArray int32_t vgId; SArray *pUdfInfo; // no need to free -} SQueryAttr; +} STaskAttr; typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); struct SOperatorInfo; -typedef struct SQueryRuntimeEnv { +typedef struct STaskIdInfo { + uint64_t queryId; // this is also a request id + uint64_t subplanId; + uint64_t templateId; + uint64_t taskId; // this is a subplan id +} STaskIdInfo; + +typedef struct STaskInfo { + STaskIdInfo id; + char *content; + uint32_t status; + STimeWindow window; + STaskCostInfo cost; + int64_t owner; // if it is in execution + + STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure + pthread_mutex_t lock; // used to synchronize the rsp/query threads +// tsem_t ready; +// int32_t dataReady; // denote if query result is ready or not +// void* rspContext; // response context + char *sql; // query sql string + jmp_buf env; +} STaskInfo; + +typedef struct STaskRuntimeEnv { jmp_buf env; - SQueryAttr* pQueryAttr; + STaskAttr* pQueryAttr; uint32_t status; // query status void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not @@ -271,7 +298,7 @@ typedef struct SQueryRuntimeEnv { SRspResultInfo resultInfo; SHashObj *pTableRetrieveTsMap; struct SUdfInfo *pUdfInfo; -} SQueryRuntimeEnv; +} STaskRuntimeEnv; enum { OP_IN_EXECUTING = 1, @@ -287,10 +314,11 @@ typedef struct SOperatorInfo { char *name; // name, used to show the query execution plan void *info; // extension attribution SExprInfo *pExpr; - SQueryRuntimeEnv *pRuntimeEnv; + STaskRuntimeEnv *pRuntimeEnv; + STaskInfo *pTaskInfo; - struct SOperatorInfo **upstream; // upstream pointer list - int32_t numOfUpstream; // number of upstream. The value is always ONE expect for join operator + struct SOperatorInfo **pDownstream; // downstram pointer list + int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator __operator_fn_t exec; __optr_cleanup_fn_t cleanup; } SOperatorInfo; @@ -312,8 +340,8 @@ typedef struct SQInfo { int32_t code; // error code to returned to client int64_t owner; // if it is in execution - SQueryRuntimeEnv runtimeEnv; - SQueryAttr query; + STaskRuntimeEnv runtimeEnv; + STaskAttr query; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads @@ -322,10 +350,10 @@ typedef struct SQInfo { void* rspContext; // response context int64_t startExecTs; // start to exec timestamp char* sql; // query sql string - SQueryCostInfo summary; + STaskCostInfo summary; } SQInfo; -typedef struct SQueryParam { +typedef struct STaskParam { char *sql; char *tagCond; char *colCond; @@ -345,7 +373,7 @@ typedef struct SQueryParam { int32_t tableScanOperator; SArray *pOperator; struct SUdfInfo *pUdfInfo; -} SQueryParam; +} STaskParam; typedef struct STableScanInfo { void *pQueryHandle; @@ -366,9 +394,12 @@ typedef struct STableScanInfo { SSDataBlock block; int32_t numOfOutput; int64_t elapsedTime; - int32_t tableIndex; - int32_t prevGroupId; // previous table group id + + int32_t prevGroupId; // previous table group id + + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + STimeWindow window; } STableScanInfo; typedef struct STagScanInfo { @@ -512,34 +543,34 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); -SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime); +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); -SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); -SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, +SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); +SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); +SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows, void* merger); -SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); -SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); -SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, +SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); +SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); +SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo* createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); +SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); @@ -561,8 +592,8 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); -void freeParam(SQueryParam *param); -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); +void freeParam(STaskParam *param); +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, STaskParam* param); int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo); @@ -575,13 +606,13 @@ SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pCo SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo); -int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, +int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start, int32_t prevResultLen, void* merger); -int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId); +int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId); void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); -STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); +STableQueryInfo *createTableQueryInfo(STaskAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); @@ -590,9 +621,9 @@ bool isQueryKilled(SQInfo *pQInfo); int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo *pQInfo); bool doBuildResCheck(SQInfo* pQInfo); -void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status); +void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); -bool onlyQueryTags(SQueryAttr* pQueryAttr); +bool onlyQueryTags(STaskAttr* pQueryAttr); void destroyUdfInfo(struct SUdfInfo* pUdfInfo); bool isValidQInfo(void *param); @@ -607,8 +638,8 @@ void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code); void calculateOperatorProfResults(SQInfo* pQInfo); void queryCostStatis(SQInfo *pQInfo); -void freeQInfo(SQInfo *pQInfo); -void freeQueryAttr(SQueryAttr *pQuery); +void doDestroyTask(SQInfo *pQInfo); +void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ac91f906c7..e8ecffb72c 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -30,7 +30,7 @@ typedef struct SCompSupporter { int32_t order; } SCompSupporter; -int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable) { +int32_t getRowNumForMultioutput(STaskAttr* pQueryAttr, bool topBottomQuery, bool stable) { if (pQueryAttr && (!stable)) { for (int16_t i = 0; i < pQueryAttr->numOfOutput; ++i) { // if (pQueryAttr->pExpr1[i].base. == FUNCTION_TOP || pQueryAttr->pExpr1[i].base.functionId == FUNCTION_BOTTOM) { @@ -42,7 +42,7 @@ int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, boo return 1; } -int32_t getOutputInterResultBufSize(SQueryAttr* pQueryAttr) { +int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { int32_t size = 0; for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { @@ -86,7 +86,7 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { tfree(pResultRowInfo->pResult); } -void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { +void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0) { return; } @@ -136,7 +136,7 @@ void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { getResultRow(pResultRowInfo, slot)->closed = true; } -void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_t type) { +void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_t type) { if (pResultRow == NULL) { return; } @@ -174,8 +174,8 @@ struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, return NULL; } -size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +size_t getResultRowSize(STaskRuntimeEnv* pRuntimeEnv) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; return 0; // return (pQueryAttr->numOfOutput * sizeof(SResultRowEntryInfo)) + pQueryAttr->interBufSize + sizeof(SResultRow); } @@ -393,8 +393,8 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); } -static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) { int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId; @@ -488,7 +488,7 @@ int32_t tsDescOrder(const void* p1, const void* p2) { } } -void orderTheResultRows(SQueryRuntimeEnv* pRuntimeEnv) { +void orderTheResultRows(STaskRuntimeEnv* pRuntimeEnv) { __compar_fn_t fn = NULL; if (pRuntimeEnv->pQueryAttr->order.order == TSDB_ORDER_ASC) { fn = tsAscOrder; @@ -499,7 +499,7 @@ void orderTheResultRows(SQueryRuntimeEnv* pRuntimeEnv) { taosArraySort(pRuntimeEnv->pResultRowArrayList, fn); } -static int32_t mergeIntoGroupResultImplRv(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, uint64_t groupId, int32_t* rowCellInfoOffset) { +static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, uint64_t groupId, int32_t* rowCellInfoOffset) { if (!pGroupResInfo->ordered) { orderTheResultRows(pRuntimeEnv); pGroupResInfo->ordered = true; @@ -528,7 +528,7 @@ static int32_t mergeIntoGroupResultImplRv(SQueryRuntimeEnv *pRuntimeEnv, SGroupR return TSDB_CODE_SUCCESS; } -static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList, +static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList, int32_t* rowCellInfoOffset) { bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQueryAttr); @@ -630,7 +630,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEn return code; } -int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t* offset) { +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRuntimeEnv, int32_t* offset) { int64_t st = taosGetTimestampUs(); while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c new file mode 100644 index 0000000000..b98b7fef5c --- /dev/null +++ b/source/libs/executor/src/executorMain.c @@ -0,0 +1,579 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tcache.h" +#include "tglobal.h" +#include "tmsg.h" +#include "exception.h" + +#include "thash.h" +#include "executorimpl.h" +#include "executor.h" +#include "tlosertree.h" +#include "ttypes.h" +#include "query.h" + +typedef struct STaskMgmt { + pthread_mutex_t lock; + SCacheObj *qinfoPool; // query handle pool + int32_t vgId; + bool closed; +} STaskMgmt; + +static void taskMgmtKillTaskFn(void* handle, void* param1) { + void** fp = (void**)handle; + qKillTask(*fp); +} + +static void freeqinfoFn(void *qhandle) { + void** handle = qhandle; + if (handle == NULL || *handle == NULL) { + return; + } + + qKillTask(*handle); + qDestroyTask(*handle); +} + +void freeParam(STaskParam *param) { + tfree(param->sql); + tfree(param->tagCond); + tfree(param->tbnameCond); + tfree(param->pTableIdList); + taosArrayDestroy(param->pOperator); + tfree(param->pExprs); + tfree(param->pSecExprs); + + tfree(param->pExpr); + tfree(param->pSecExpr); + + tfree(param->pGroupColIndex); + tfree(param->pTagColumnInfo); + tfree(param->pGroupbyExpr); + tfree(param->prevResult); +} + +// todo parse json to get the operator tree. + +int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryMsg, qTaskInfo_t* pTaskInfo, uint64_t taskId) { + assert(pQueryMsg != NULL && tsdb != NULL); + + int32_t code = TSDB_CODE_SUCCESS; +#if 0 + STaskParam param = {0}; + code = convertQueryMsg(pQueryMsg, ¶m); + if (code != TSDB_CODE_SUCCESS) { + goto _over; + } + + if (pQueryMsg->numOfTables <= 0) { + qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _over; + } + + if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { + qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _over; + } + + SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; + if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExpr, param.pTagColumnInfo, + pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + + if (param.pSecExpr != NULL) { + if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } + + if (param.colCond != NULL) { + if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, ¶m.pFilters)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } + + param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); + if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { + goto _over; + } + + bool isSTableQuery = false; + STableGroupInfo tableGroupInfo = {0}; + int64_t st = taosGetTimestampUs(); + + if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { + STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); + + qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); + if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { + isSTableQuery = true; + + // also note there's possibility that only one table in the super table + if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { + STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); + + // group by normal column, do not pass the group by condition to tsdb to group table into different group + int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; + if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { + numOfGroupByCols = 0; + } + + qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid); + code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, + pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); + + if (code != TSDB_CODE_SUCCESS) { + qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); + goto _over; + } + } else { + code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _over; + } + + qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); + } + + int64_t el = taosGetTimestampUs() - st; + qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); + } else { + assert(0); + } + + code = checkForQueryBuf(tableGroupInfo.numOfTables); + if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort + goto _over; + } + + assert(pQueryMsg->stableQuery == isSTableQuery); + (*pTaskInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, + param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo); + + param.sql = NULL; + param.pExprs = NULL; + param.pSecExprs = NULL; + param.pGroupbyExpr = NULL; + param.pTagColumnInfo = NULL; + param.pFilters = NULL; + + if ((*pTaskInfo) == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _over; + } + param.pUdfInfo = NULL; + + code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pTaskInfo, ¶m, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL); + + _over: + if (param.pGroupbyExpr != NULL) { + taosArrayDestroy(param.pGroupbyExpr->columnInfo); + } + + tfree(param.colCond); + + destroyUdfInfo(param.pUdfInfo); + + taosArrayDestroy(param.pTableIdList); + param.pTableIdList = NULL; + + freeParam(¶m); + + for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { + SColumnInfo* column = pQueryMsg->tableCols + i; + freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); + } + + filterFreeInfo(param.pFilters); + + //pTaskInfo already freed in initQInfo, but *pTaskInfo may not pointer to null; + if (code != TSDB_CODE_SUCCESS) { + *pTaskInfo = NULL; + } +#endif + + // if failed to add ref for all tables in this query, abort current query + return code; +} + +#ifdef TEST_IMPL +// wait moment +int waitMoment(SQInfo* pQInfo){ + if(pQInfo->sql) { + int ms = 0; + char* pcnt = strstr(pQInfo->sql, " count(*)"); + if(pcnt) return 0; + + char* pos = strstr(pQInfo->sql, " t_"); + if(pos){ + pos += 3; + ms = atoi(pos); + while(*pos >= '0' && *pos <= '9'){ + pos ++; + } + char unit_char = *pos; + if(unit_char == 'h'){ + ms *= 3600*1000; + } else if(unit_char == 'm'){ + ms *= 60*1000; + } else if(unit_char == 's'){ + ms *= 1000; + } + } + if(ms == 0) return 0; + printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); + + if(ms < 1000) { + taosMsleep(ms); + } else { + int used_ms = 0; + while(used_ms < ms) { + taosMsleep(1000); + used_ms += 1000; + if(isQueryKilled(pQInfo)){ + printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); + break; + } + } + } + } + return 1; +} +#endif + +bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId) { + SQInfo *pQInfo = (SQInfo *)qinfo; + assert(pQInfo && pQInfo->signature == pQInfo); + int64_t threadId = taosGetSelfPthreadId(); + + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { + qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); + pQInfo->code = TSDB_CODE_QRY_IN_EXEC; + return false; + } + + *qId = pQInfo->qId; + if(pQInfo->startExecTs == 0) + pQInfo->startExecTs = taosGetTimestampMs(); + + if (isQueryKilled(pQInfo)) { + qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId); + return doBuildResCheck(pQInfo); + } + + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { + qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId); +// setTaskStatus(pRuntimeEnv, QUERY_COMPLETED); + return doBuildResCheck(pQInfo); + } + + // error occurs, record the error code and return to client + int32_t ret = setjmp(pQInfo->runtimeEnv.env); + if (ret != TSDB_CODE_SUCCESS) { + publishQueryAbortEvent(pQInfo, ret); + pQInfo->code = ret; + qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); + return doBuildResCheck(pQInfo); + } + + qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId); + + bool newgroup = false; + publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + + int64_t st = taosGetTimestampUs(); + pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup); + pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); +#ifdef TEST_IMPL + waitMoment(pQInfo); +#endif + publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC); + pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv); + + if (isQueryKilled(pQInfo)) { + qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId); + } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { + qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, + pRuntimeEnv->resultInfo.total); + } else { + qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId, + GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); + } + + return doBuildResCheck(pQInfo); +} + +int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + qError("QInfo invalid qhandle"); + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + *buildRes = false; + if (IS_QUERY_KILLED(pQInfo)) { + qDebug("QInfo:0x%"PRIx64" query is killed, code:0x%08x", pQInfo->qId, pQInfo->code); + return pQInfo->code; + } + + int32_t code = TSDB_CODE_SUCCESS; + + if (tsRetrieveBlockingModel) { + pQInfo->rspContext = pRspContext; + tsem_wait(&pQInfo->ready); + *buildRes = true; + code = pQInfo->code; + } else { + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + + pthread_mutex_lock(&pQInfo->lock); + + assert(pQInfo->rspContext == NULL); + if (pQInfo->dataReady == QUERY_RESULT_READY) { + *buildRes = true; + qDebug("QInfo:0x%"PRIx64" retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo->qId, pQueryAttr->resultRowSize, + GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code)); + } else { + *buildRes = false; + qDebug("QInfo:0x%"PRIx64" retrieve req set query return result after paused", pQInfo->qId); + pQInfo->rspContext = pRspContext; + assert(pQInfo->rspContext != NULL); + } + + code = pQInfo->code; + pthread_mutex_unlock(&pQInfo->lock); + } + + return code; +} + +void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) { + SQInfo* pQInfo = (SQInfo*) qinfo; + assert(pQInfo != NULL); + + return pQInfo->rspContext; +} + +int32_t qKillTask(qTaskInfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("QInfo:0x%"PRIx64" query killed", pQInfo->qId); + setQueryKilled(pQInfo); + + // Wait for the query executing thread being stopped/ + // Once the query is stopped, the owner of qHandle will be cleared immediately. + while (pQInfo->owner != 0) { + taosMsleep(100); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); +} + +void qDestroyTask(qTaskInfo_t qHandle) { + SQInfo* pQInfo = (SQInfo*) qHandle; + if (!isValidQInfo(pQInfo)) { + return; + } + + qDebug("QInfo:0x%"PRIx64" query completed", pQInfo->qId); + queryCostStatis(pQInfo); // print the query cost summary + doDestroyTask(pQInfo); +} + +void* qOpenTaskMgmt(int32_t vgId) { + const int32_t refreshHandleInterval = 30; // every 30 seconds, refresh handle pool + + char cacheName[128] = {0}; + sprintf(cacheName, "qhandle_%d", vgId); + + STaskMgmt* pTaskMgmt = calloc(1, sizeof(STaskMgmt)); + if (pTaskMgmt == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pTaskMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshHandleInterval, true, freeqinfoFn, cacheName); + pTaskMgmt->closed = false; + pTaskMgmt->vgId = vgId; + + pthread_mutex_init(&pTaskMgmt->lock, NULL); + + qDebug("vgId:%d, open queryTaskMgmt success", vgId); + return pTaskMgmt; +} + +void qTaskMgmtNotifyClosing(void* pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + STaskMgmt* pQueryMgmt = pQMgmt; + qInfo("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); + + pthread_mutex_lock(&pQueryMgmt->lock); + pQueryMgmt->closed = true; + pthread_mutex_unlock(&pQueryMgmt->lock); + + taosCacheRefresh(pQueryMgmt->qinfoPool, taskMgmtKillTaskFn, NULL); +} + +void qQueryMgmtReOpen(void *pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + STaskMgmt *pQueryMgmt = pQMgmt; + qInfo("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId); + + pthread_mutex_lock(&pQueryMgmt->lock); + pQueryMgmt->closed = false; + pthread_mutex_unlock(&pQueryMgmt->lock); +} + +void qCleanupTaskMgmt(void* pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + STaskMgmt* pQueryMgmt = pQMgmt; + int32_t vgId = pQueryMgmt->vgId; + + assert(pQueryMgmt->closed); + + SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool; + pQueryMgmt->qinfoPool = NULL; + + taosCacheCleanup(pqinfoPool); + pthread_mutex_destroy(&pQueryMgmt->lock); + tfree(pQueryMgmt); + + qDebug("vgId:%d, queryMgmt cleanup completed", vgId); +} + +void** qRegisterTask(void* pMgmt, uint64_t qId, void *qInfo) { + if (pMgmt == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + STaskMgmt *pQueryMgmt = pMgmt; + if (pQueryMgmt->qinfoPool == NULL) { + qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into qMgmt, since qMgmt is closed", qId, (void*)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + pthread_mutex_lock(&pQueryMgmt->lock); + if (pQueryMgmt->closed) { + pthread_mutex_unlock(&pQueryMgmt->lock); + qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into cache, since qMgmt is colsing", qId, (void*)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } else { + void** handle = taosCachePut(pQueryMgmt->qinfoPool, &qId, sizeof(qId), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), + (getMaximumIdleDurationSec()*1000)); + pthread_mutex_unlock(&pQueryMgmt->lock); + + return handle; + } +} + +void** qAcquireTask(void* pMgmt, uint64_t _key) { + STaskMgmt *pQueryMgmt = pMgmt; + + if (pQueryMgmt->closed) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + if (pQueryMgmt->qinfoPool == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; + return NULL; + } + + void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &_key, sizeof(_key)); + if (handle == NULL || *handle == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; + return NULL; + } else { + return handle; + } +} + +void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle) { + STaskMgmt *pQueryMgmt = pMgmt; + if (pQueryMgmt->qinfoPool == NULL) { + return NULL; + } + + taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); + return 0; +} + +#if 0 +//kill by qid +int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) { + int32_t error = TSDB_CODE_SUCCESS; + void** handle = qAcquireTask(pMgmt, qId); + if(handle == NULL) return terrno; + + SQInfo* pQInfo = (SQInfo*)(*handle); + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId); + setQueryKilled(pQInfo); + + // wait query stop + int32_t loop = 0; + while (pQInfo->owner != 0) { + taosMsleep(waitMs); + if(loop++ > waitCount){ + error = TSDB_CODE_FAILED; + break; + } + } + + qReleaseTask(pMgmt, (void **)&handle, true); + return error; +} + +#endif \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f119627c69..1d2740f0e0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -12,24 +12,25 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include #include "os.h" #include "tmsg.h" #include "tglobal.h" #include "ttime.h" - #include "exception.h" + +#include "../../../../contrib/cJson/cJSON.h" #include "executorimpl.h" -#include "thash.h" #include "function.h" #include "tcompare.h" #include "tcompression.h" +#include "thash.h" #include "ttypes.h" +#include "query.h" -#define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) +#define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) #define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN) -#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) +#define SET_MAIN_SCAN_FLAG(runtime) ((runtime)->scanFlag = MAIN_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) @@ -41,11 +42,6 @@ #define MULTI_KEY_DELIM "-" -#define TIME_WINDOW_COPY(_dst, _src) do {\ - (_dst).skey = (_src).skey;\ - (_dst).ekey = (_src).ekey;\ -} while (0) - enum { TS_JOIN_TS_EQUAL = 0, TS_JOIN_TS_NOT_EQUALS = 1, @@ -131,40 +127,16 @@ do { \ } \ } while (0) -uint64_t queryHandleId = 0; - int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } -int64_t genQueryId(void) { - int64_t uid = 0; - int64_t did = 0;//tsDnodeId; - - uid = did << 54; - - int64_t pid = ((int64_t)taosGetPId()) & 0x3FF; - - uid |= pid << 44; - - int64_t ts = taosGetTimestampMs() & 0x1FFFFFFFF; - - uid |= ts << 11; - - int64_t sid = atomic_add_fetch_64(&queryHandleId, 1) & 0x7FF; - - uid |= sid; - -// //qDebug("gen qid:0x%"PRIx64, uid); - - return uid; -} static int32_t getExprFunctionId(SExprInfo *pExprInfo) { assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); return 0; } -static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { +static void getNextTimeWindow(STaskAttr* pQueryAttr, STimeWindow* tw) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); if (pQueryAttr->interval.intervalUnit != 'n' && pQueryAttr->interval.intervalUnit != 'y') { tw->skey += pQueryAttr->interval.sliding * factor; @@ -198,28 +170,28 @@ static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setResultOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, +static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset); -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); -static bool hasMainOutput(SQueryAttr *pQueryAttr); +static bool hasMainOutput(STaskAttr *pQueryAttr); static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); -static int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo); +static int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); -//static STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win); +//static STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win); static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); -static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr); +static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); @@ -232,32 +204,34 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); +void setTaskStatus(STaskInfo *pTaskInfo, int8_t status); + static void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; - if (pOperator->pRuntimeEnv != NULL) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + if (pOperator->pTaskInfo != NULL) { + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); } } -static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); +static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); +static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); -static void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); -static void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo); -static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); -static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); -static void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); -static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, +static void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); +static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo); +static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); +static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); +static void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); -SArray* getOrderCheckColumns(SQueryAttr* pQuery); +SArray* getOrderCheckColumns(STaskAttr* pQuery); typedef struct SRowCompSupporter { - SQueryRuntimeEnv *pRuntimeEnv; + STaskRuntimeEnv *pRuntimeEnv; int16_t dataOffset; __compar_fn_t comFunc; } SRowCompSupporter; @@ -267,7 +241,7 @@ static int compareRowData(const void *a, const void *b, const void *userData) { const SResultRow *pRow2 = (const SResultRow *)b; SRowCompSupporter *supporter = (SRowCompSupporter *)userData; - SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; SFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId); SFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); @@ -279,7 +253,7 @@ static int compareRowData(const void *a, const void *b, const void *userData) { return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; } -static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) { +static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) { SArray *columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); size_t size = taosArrayGetSize(columnOrderList); taosArrayDestroy(columnOrderList); @@ -375,7 +349,7 @@ static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput // return (numOfSelectivity > 0 && hasTags); } -static bool isProjQuery(SQueryAttr *pQueryAttr) { +static bool isProjQuery(STaskAttr *pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functId = getExprFunctionId(&pQueryAttr->pExpr1[i]); if (functId != FUNCTION_PRJ && functId != FUNCTION_TAGPRJ) { @@ -398,7 +372,7 @@ static bool hasNull(SColIndex* pColIndex, SColumnDataAgg *pStatis) { return true; } -static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntimeEnv* pRuntimeEnv) { +static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, STaskRuntimeEnv* pRuntimeEnv) { // more than the capacity, reallocate the resources if (pResultRowInfo->size < pResultRowInfo->capacity) { return; @@ -424,7 +398,7 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntim pResultRowInfo->capacity = (int32_t)newCapacity; } -static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, +static bool chkResultRowFromKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { bool existed = false; SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); @@ -462,7 +436,7 @@ static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *p } -static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, +static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) { bool existed = false; SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tableGroupId); @@ -536,7 +510,7 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult return pResultRowInfo->pResult[pResultRowInfo->curPos]; } -static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { +static void getInitialStartTimeWindow(STaskAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { if (QUERY_IS_ASC_QUERY(pQueryAttr)) { getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); } else { @@ -561,7 +535,7 @@ static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWin } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { +static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, STaskAttr *pQueryAttr) { STimeWindow w = {0}; if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value @@ -609,7 +583,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t } // get the correct time window according to the handled timestamp -static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { +static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, STaskAttr *pQueryAttr) { STimeWindow w = {0}; if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value @@ -680,14 +654,14 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } -static bool chkWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, +static bool chkWindowOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } -static int32_t setResultOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, +static int32_t setResultOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); @@ -816,7 +790,7 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } } -static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQueryAttr* pQueryAttr, TSKEY lastKey) { +static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, STaskAttr* pQueryAttr, TSKEY lastKey) { bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); if ((lastKey > pQueryAttr->window.ekey && ascQuery) || (lastKey < pQueryAttr->window.ekey && (!ascQuery))) { closeAllResultRows(pResultRowInfo); @@ -827,10 +801,10 @@ static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQuer } } -static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, +static int32_t getNumOfRowsInTimeWindow(STaskRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; STableQueryInfo* item = pRuntimeEnv->current; int32_t num = -1; @@ -867,9 +841,9 @@ static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBloc return num; } -static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, +static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; bool hasAggregates = pCtx[0].isAggSet; for (int32_t k = 0; k < numOfOutput; ++k) { @@ -904,7 +878,7 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx } } -static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, +static int32_t getNextQualifiedWindow(STaskAttr* pQueryAttr, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, TSKEY* primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { getNextTimeWindow(pQueryAttr, pNext); @@ -983,7 +957,7 @@ static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow* pNext return startPos; } -static FORCE_INLINE TSKEY reviseWindowEkey(SQueryAttr *pQueryAttr, STimeWindow *pWindow) { +static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr *pQueryAttr, STimeWindow *pWindow) { TSKEY ekey = -1; if (QUERY_IS_ASC_QUERY(pQueryAttr)) { ekey = pWindow->ekey; @@ -1012,20 +986,20 @@ static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, in } } -static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, +static void saveDataBlockLastRow(STaskRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, int32_t rowIndex) { if (pDataBlock == NULL) { return; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t k = 0; k < pQueryAttr->numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); } } -static TSKEY getStartTsKey(SQueryAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { +static TSKEY getStartTsKey(STaskAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { TSKEY ts = TSKEY_INITIAL_VAL; bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); @@ -1126,7 +1100,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, } static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { @@ -1136,8 +1110,8 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction } } -static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +static void projectApplyFunctions(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].startTs = pQueryAttr->window.skey; @@ -1161,7 +1135,7 @@ static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; SExprInfo* pExpr = pOperator->pExpr; SQLFunctionCtx* pCtx = pInfo->pCtx; @@ -1226,8 +1200,8 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); @@ -1257,8 +1231,8 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLF static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { - SQueryRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfOutput = pOperatorInfo->numOfOutput; TSKEY actualEndKey = tsCols[endRowIndex]; @@ -1289,8 +1263,8 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFun static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; if (!pQueryAttr->timeWindowInterpo) { return; } @@ -1340,9 +1314,9 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; int32_t numOfOutput = pOperatorInfo->numOfOutput; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); @@ -1361,7 +1335,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); SResultRow* pResult = NULL; int32_t ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, @@ -1450,9 +1424,9 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; int32_t numOfOutput = pOperatorInfo->numOfOutput; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); @@ -1469,7 +1443,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); SResultRow* pResult = NULL; int32_t forwardStep = 0; @@ -1525,12 +1499,12 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; STableQueryInfo* item = pRuntimeEnv->current; SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int16_t bytes = pColInfoData->info.bytes; int16_t type = pColInfoData->info.type; @@ -1607,13 +1581,13 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; STableQueryInfo* item = pRuntimeEnv->current; // primary timestamp column SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); SOptrBasicInfo* pBInfo = &pInfo->binfo; int64_t gap = pOperator->pRuntimeEnv->pQueryAttr->sw.gap; @@ -1692,7 +1666,7 @@ static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { } } -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { +static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; @@ -1746,9 +1720,9 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD return -1; } -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // in case of timestamp column, always generated results. int32_t functionId = pCtx->functionId; @@ -1843,9 +1817,9 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { return TSDB_CODE_SUCCESS; } -static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, +static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); if (pFuncCtx == NULL) { @@ -1965,9 +1939,9 @@ static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { return NULL; } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { +static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->pQueryAttr = pQueryAttr; @@ -2019,30 +1993,30 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf // case OP_MultiTableTimeInterval: { // pRuntimeEnv->proot = // createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // break; // } // case OP_AllMultiTableTimeInterval: { // pRuntimeEnv->proot = // createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // break; // } // case OP_TimeWindow: { // pRuntimeEnv->proot = // createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput && opType != OP_Join) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } // case OP_AllTimeWindow: { // pRuntimeEnv->proot = // createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput && opType != OP_Join) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } @@ -2050,34 +2024,34 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf // pRuntimeEnv->proot = // createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); // -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } // case OP_SessionWindow: { // pRuntimeEnv->proot = // createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } // case OP_MultiTableAggregate: { // pRuntimeEnv->proot = // createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // break; // } // case OP_Aggregate: { // pRuntimeEnv->proot = // createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); // -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput && opType != OP_Join) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } @@ -2099,9 +2073,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf // // case OP_StateWindow: { // pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } @@ -2187,8 +2161,8 @@ _clean: return TSDB_CODE_QRY_OUT_OF_MEMORY; } -static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); pRuntimeEnv->pQueryHandle = NULL; @@ -2197,7 +2171,7 @@ static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { // assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); } -static void destroyTsComp(SQueryRuntimeEnv *pRuntimeEnv, SQueryAttr *pQueryAttr) { +static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) { if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) { SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); if (pColInfoData) { @@ -2210,8 +2184,8 @@ static void destroyTsComp(SQueryRuntimeEnv *pRuntimeEnv, SQueryAttr *pQueryAttr) } } -static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); @@ -2271,7 +2245,7 @@ bool isQueryKilled(SQInfo *pQInfo) { void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} -//static bool isFixedOutputQuery(SQueryAttr* pQueryAttr) { +//static bool isFixedOutputQuery(STaskAttr* pQueryAttr) { // if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { // return false; // } @@ -2297,7 +2271,7 @@ void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELL //} // todo refactor with isLastRowQuery -//bool isPointInterpoQuery(SQueryAttr *pQueryAttr) { +//bool isPointInterpoQuery(STaskAttr *pQueryAttr) { // for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { // int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; // if (functionId == FUNCTION_INTERP) { @@ -2308,7 +2282,7 @@ void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELL // return false; //} -static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { +static bool isFirstLastRowQuery(STaskAttr *pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionID = getExprFunctionId(&pQueryAttr->pExpr1[i]); if (functionID == FUNCTION_LAST_ROW) { @@ -2319,7 +2293,7 @@ static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { return false; } -static bool isCachedLastQuery(SQueryAttr *pQueryAttr) { +static bool isCachedLastQuery(STaskAttr *pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); if (functionId == FUNCTION_LAST || functionId == FUNCTION_LAST_DST) { @@ -2354,7 +2328,7 @@ static bool isCachedLastQuery(SQueryAttr *pQueryAttr) { * The following 4 kinds of query are treated as the tags query * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query */ -bool onlyQueryTags(SQueryAttr* pQueryAttr) { +bool onlyQueryTags(STaskAttr* pQueryAttr) { for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { SExprInfo* pExprInfo = &pQueryAttr->pExpr1[i]; @@ -2373,7 +2347,7 @@ bool onlyQueryTags(SQueryAttr* pQueryAttr) { ///////////////////////////////////////////////////////////////////////////////////////////// -void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { +void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { assert(key >= keyFirst && key <= keyLast && pQueryAttr->interval.sliding <= pQueryAttr->interval.interval); win->skey = taosTimeTruncate(key, &pQueryAttr->interval, pQueryAttr->precision); @@ -2394,7 +2368,7 @@ void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFir /* * todo add more parameters to check soon.. */ -bool colIdCheck(SQueryAttr *pQueryAttr, uint64_t qId) { +bool colIdCheck(STaskAttr *pQueryAttr, uint64_t qId) { // load data column information is incorrect for (int32_t i = 0; i < pQueryAttr->numOfCols - 1; ++i) { if (pQueryAttr->tableCols[i].colId == pQueryAttr->tableCols[i + 1].colId) { @@ -2408,7 +2382,7 @@ bool colIdCheck(SQueryAttr *pQueryAttr, uint64_t qId) { // todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which // the scan order is not matter -static bool onlyOneQueryType(SQueryAttr *pQueryAttr, int32_t functId, int32_t functIdDst) { +static bool onlyOneQueryType(STaskAttr *pQueryAttr, int32_t functId, int32_t functIdDst) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); @@ -2425,13 +2399,13 @@ static bool onlyOneQueryType(SQueryAttr *pQueryAttr, int32_t functId, int32_t fu return true; } -static bool onlyFirstQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST); } +static bool onlyFirstQuery(STaskAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST); } -static bool onlyLastQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST); } +static bool onlyLastQuery(STaskAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST); } -static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); } +static bool notContainSessionOrStateWindow(STaskAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); } -static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { +static int32_t updateBlockLoadStatus(STaskAttr *pQuery, int32_t status) { bool hasFirstLastFunc = false; bool hasOtherFunc = false; @@ -2465,7 +2439,7 @@ static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { return status; } -static void doUpdateLastKey(SQueryAttr* pQueryAttr) { +static void doUpdateLastKey(STaskAttr* pQueryAttr) { STimeWindow* win = &pQueryAttr->window; size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); @@ -2485,7 +2459,7 @@ static void doUpdateLastKey(SQueryAttr* pQueryAttr) { } static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { - SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; // in case of point-interpolation query, use asc order scan char msg[] = "QInfo:0x%"PRIx64" scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 @@ -2580,8 +2554,8 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool } } -static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t MIN_ROWS_PER_PAGE = 4; *rowsize = (int32_t)(pQueryAttr->resultRowSize * getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); @@ -2596,8 +2570,8 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) -//static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { -// SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +//static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { +// STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // // if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { // return true; @@ -2606,7 +2580,7 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i // return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); //} -static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { +static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); @@ -2655,7 +2629,7 @@ static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlock return false; } -static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { +static int32_t doTSJoinFilter(STaskRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); #if defined(_DEBUG_VIEW) @@ -2781,7 +2755,7 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { } } -void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, +void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock, bool ascQuery) { int32_t numOfRows = pBlock->info.rows; @@ -2823,7 +2797,7 @@ void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInf tfree(p); } -void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { +void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { int32_t numOfRows = pBlock->info.rows; int8_t *p = NULL; @@ -2913,22 +2887,35 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi } } -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, - uint32_t* status) { - *status = BLK_DATA_NO_NEEDED; - pBlock->pDataBlock = NULL; - pBlock->pBlockAgg = NULL; - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int64_t groupId = pRuntimeEnv->current->groupIndex; - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - SQInfo* pQInfo = pRuntimeEnv->qinfo; - SQueryCostInfo* pCost = &pQInfo->summary; +int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { + STaskCostInfo* pCost = &pTaskInfo->cost; pCost->totalBlocks += 1; pCost->totalRows += pBlock->info.rows; + pCost->totalCheckedRows += pBlock->info.rows; + pCost->loadBlocks += 1; + +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + if (pBlock->pDataBlock == NULL) { + return terrno; + } +} + +int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { + *status = BLK_DATA_NO_NEEDED; + + pBlock->pDataBlock = NULL; + pBlock->pBlockAgg = NULL; + +// int64_t groupId = pRuntimeEnv->current->groupIndex; +// bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + STaskCostInfo* pCost = &pTaskInfo->cost; + + pCost->totalBlocks += 1; + pCost->totalRows += pBlock->info.rows; +#if 0 if (pRuntimeEnv->pTsBuf != NULL) { (*status) = BLK_DATA_ALL_NEEDED; @@ -2953,7 +2940,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa // Calculate all time windows that are overlapping or contain current data block. // If current data block is contained by all possible time window, do not load current data block. if (/*pQueryAttr->pFilters || */pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || - (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pQueryAttr, &pBlock->info))) { + (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pTaskInfo, &pBlock->info))) { (*status) = BLK_DATA_ALL_NEEDED; } @@ -2966,7 +2953,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { SResultRow* pResult = NULL; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); @@ -2995,7 +2982,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa } SDataBlockInfo* pBlockInfo = &pBlock->info; - *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); +// *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); if ((*status) == BLK_DATA_NO_NEEDED || (*status) == BLK_DATA_DISCARD) { //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, @@ -3022,7 +3009,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { SResultRow* pResult = NULL; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); @@ -3074,7 +3061,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa // filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); // } } - +#endif return TSDB_CODE_SUCCESS; } @@ -3184,10 +3171,10 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num } void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; SExprInfo *pExpr = pOperatorInfo->pExpr; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; SExprInfo* pExprInfo = &pExpr[0]; int32_t functionId = getExprFunctionId(pExprInfo); @@ -3242,7 +3229,7 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCt } } -void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { +void copyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { SGroupResInfo* pGroupResInfo = &pRuntimeEnv->groupResInfo; pBlock->info.rows = 0; @@ -3293,9 +3280,8 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) } } -static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - +static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) { +#if 0 int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); for(int32_t i = 0; i < numOfGroups; ++i) { SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); @@ -3314,6 +3300,8 @@ static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { // assert(pCheckInfo->pTable == pTableKeyInfo->pTable); } } +#endif + } void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { @@ -3337,7 +3325,7 @@ int32_t initResultRow(SResultRow *pResultRow) { * +------------+-------------------------------------------+-------------------------------------------+ * offset[0] offset[1] offset[2] */ -void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { +void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { SQLFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; @@ -3461,40 +3449,38 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { } } -void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status) { +void setTaskStatus(STaskInfo *pTaskInfo, int8_t status) { if (status == QUERY_NOT_COMPLETED) { - pRuntimeEnv->status = status; + pTaskInfo->status = status; } else { // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pRuntimeEnv, QUERY_NOT_COMPLETED); - pRuntimeEnv->status |= status; + CLEAR_QUERY_STATUS(pTaskInfo, QUERY_NOT_COMPLETED); + pTaskInfo->status |= status; } } -static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - if (pRuntimeEnv->pTsBuf) { - SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } +static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +// if (pRuntimeEnv->pTsBuf) { +// SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); +// bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); +// assert(ret); +// } // reverse order time range - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + SWAP(pTableScanInfo->window.skey, pTableScanInfo->window.ekey, TSKEY); - SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + SET_REVERSE_SCAN_FLAG(pTableScanInfo); +// setTaskStatus(pTableScanInfo, QUERY_NOT_COMPLETED); switchCtxOrder(pCtx, numOfOutput); - SWITCH_ORDER(pQueryAttr->order.order); - setupQueryRangeForReverseScan(pRuntimeEnv); + SWITCH_ORDER(pTableScanInfo->order); + setupQueryRangeForReverseScan(pTableScanInfo); } void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfOutput = pOperator->numOfOutput; if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { @@ -3539,7 +3525,7 @@ void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResult } } -static bool hasMainOutput(SQueryAttr *pQueryAttr) { +static bool hasMainOutput(STaskAttr *pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); @@ -3551,7 +3537,7 @@ static bool hasMainOutput(SQueryAttr *pQueryAttr) { return false; } -STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { +STableQueryInfo *createTableQueryInfo(STaskAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { STableQueryInfo *pTableQueryInfo = buf; pTableQueryInfo->win = win; @@ -3602,7 +3588,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3635,7 +3621,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe } } -void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, +void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { // for simple group by query without interval, all the tables belong to one group result. int64_t uid = 0; @@ -3659,7 +3645,7 @@ void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pRe setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); } -void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, +void setExecutionContext(STaskRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, TSKEY nextKey) { STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; @@ -3675,7 +3661,7 @@ void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, i pRuntimeEnv->prevGroupId = tableGroupId; } -void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3698,8 +3684,8 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLF } } -void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; SSqlExpr* pExpr = &pExprInfo->base; // if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && @@ -3723,8 +3709,8 @@ void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExpr // } } -int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; assert(pRuntimeEnv->pTsBuf != NULL); @@ -3766,9 +3752,9 @@ int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, } // TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. -void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { +void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { #if 0 - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfExprs = pQueryAttr->numOfOutput; for(int32_t i = 0; i < numOfExprs; ++i) { @@ -3801,8 +3787,8 @@ void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx #endif } -void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 int32_t numOfExprs = pQueryAttr->numOfOutput; for(int32_t i = 0; i < numOfExprs; ++i) { @@ -3842,8 +3828,8 @@ void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunction * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there * is a previous result generated or not. */ -void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; SResultRowInfo *pResultRowInfo = &pTableQueryInfo->resInfo; @@ -3887,8 +3873,8 @@ void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { * @param result */ -static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfResult = pBlock->info.rows; // there are already exists result rows @@ -3946,7 +3932,7 @@ static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* return 0; } -static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { +static void toSSDataBlock(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); pBlock->info.rows = 0; @@ -3954,7 +3940,7 @@ static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRunti return; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t orderType = TSDB_ORDER_ASC;//(pQueryAttr->pGroupbyExpr != NULL) ? pQueryAttr->pGroupbyExpr->orderType : TSDB_ORDER_ASC; doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, orderType, pBlock); @@ -3969,9 +3955,9 @@ static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRunti } } -static void updateNumOfRowsInResultRows(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, +static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // update the number of result for each, only update the number of rows for the corresponding window result. if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { @@ -4000,8 +3986,8 @@ static int32_t compressQueryColData(SColumnInfoData *pColRes, int32_t numOfRows, } static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data, int8_t compressed, int32_t *compLen) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; SSDataBlock* pRes = pRuntimeEnv->outputBuf; @@ -4070,7 +4056,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data // Check if query is completed or not for stable query or normal table query respectively. if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { - setQueryStatus(pRuntimeEnv, QUERY_OVER); +// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } @@ -4187,8 +4173,8 @@ void calculateOperatorProfResults(SQInfo* pQInfo) { } void queryCostStatis(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryCostInfo *pSummary = &pQInfo->summary; + STaskRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + STaskCostInfo *pSummary = &pQInfo->summary; uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); @@ -4226,8 +4212,8 @@ void queryCostStatis(SQInfo *pQInfo) { } } -//static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +//static void updateOffsetVal(STaskRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); @@ -4262,8 +4248,8 @@ void queryCostStatis(SQInfo *pQInfo) { // pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); //} -//void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +//void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) { +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // // if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) { // return; @@ -4301,8 +4287,8 @@ void queryCostStatis(SQInfo *pQInfo) { // } //} -//static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +//static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; // // assert(pQueryAttr->limit.offset == 0); @@ -4352,8 +4338,8 @@ void queryCostStatis(SQInfo *pQInfo) { // return true; //} -//static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +//static bool skipTimeInterval(STaskRuntimeEnv *pRuntimeEnv, TSKEY* start) { +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // assert(*start <= pRuntimeEnv->current->lastKey); // } else { @@ -4463,18 +4449,18 @@ void queryCostStatis(SQInfo *pQInfo) { //} void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { - if (p->upstream == NULL) { - assert(p->numOfUpstream == 0); + if (p->pDownstream == NULL) { + assert(p->numOfDownstream == 0); } - p->upstream = realloc(p->upstream, POINTER_BYTES * (p->numOfUpstream + 1)); - p->upstream[p->numOfUpstream++] = pUpstream; + p->pDownstream = realloc(p->pDownstream, POINTER_BYTES * (p->numOfDownstream + 1)); + p->pDownstream[p->numOfDownstream++] = pUpstream; } static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); -static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 // TODO set the tags scan handle if (onlyQueryTags(pQueryAttr)) { @@ -4533,9 +4519,9 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, SArray* pOperator, void* param) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + STaskRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; pQueryAttr->tsdb = tsdb; if (tsdb != NULL) { @@ -4616,25 +4602,25 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr return code; } - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_NOT_COMPLETED); return TSDB_CODE_SUCCESS; } -static void doTableQueryInfoTimeWindowCheck(SQueryAttr* pQueryAttr, STableQueryInfo* pTableQueryInfo) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { +static void doTableQueryInfoTimeWindowCheck(STaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { + if (order == TSDB_ORDER_ASC) { assert( (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && - (pTableQueryInfo->win.skey >= pQueryAttr->window.skey && pTableQueryInfo->win.ekey <= pQueryAttr->window.ekey)); + (pTableQueryInfo->lastKey >= pTaskInfo->window.skey) && + (pTableQueryInfo->win.skey >= pTaskInfo->window.skey && pTableQueryInfo->win.ekey <= pTaskInfo->window.ekey)); } else { assert( (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && - (pTableQueryInfo->win.skey <= pQueryAttr->window.skey && pTableQueryInfo->win.ekey >= pQueryAttr->window.ekey)); + (pTableQueryInfo->lastKey <= pTaskInfo->window.skey) && + (pTableQueryInfo->win.skey <= pTaskInfo->window.skey && pTableQueryInfo->win.ekey >= pTaskInfo->window.ekey)); } } -//STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win) { +//STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win) { // STsdbQueryCond cond = { // .colList = pQueryAttr->tableCols, // .order = pQueryAttr->order.order, @@ -4676,7 +4662,7 @@ static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { // } //} -static void doCloseAllTimeWindow(SQueryRuntimeEnv* pRuntimeEnv) { +static void doCloseAllTimeWindow(STaskRuntimeEnv* pRuntimeEnv) { size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); for (int32_t i = 0; i < numOfGroup; ++i) { SArray* group = GET_TABLEGROUP(pRuntimeEnv, i); @@ -4693,44 +4679,37 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { SOperatorInfo *pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; + STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SSDataBlock *pBlock = &pTableScanInfo->block; - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - STableGroupInfo *pTableGroupInfo = &pOperator->pRuntimeEnv->tableqinfoGroupInfo; + STableGroupInfo *pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; *newgroup = false; -#if 0 - while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { + + while (/*tsdbNextDataBlock(pTableScanInfo->pQueryHandle)*/1) { if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); +// tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); // todo opt - if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { - STableQueryInfo** pTableQueryInfo = - (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); - if (pTableQueryInfo == NULL) { - break; - } - - pRuntimeEnv->current = *pTableQueryInfo; - doTableQueryInfoTimeWindowCheck(pQueryAttr, *pTableQueryInfo); - - if (pRuntimeEnv->enableGroupData) { - if(pTableScanInfo->prevGroupId != -1 && pTableScanInfo->prevGroupId != (*pTableQueryInfo)->groupIndex) { - *newgroup = true; - } - } - - pTableScanInfo->prevGroupId = (*pTableQueryInfo)->groupIndex; - } +// if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { +// STableQueryInfo** pTableQueryInfo = +// (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); +// if (pTableQueryInfo == NULL) { +// break; +// } +// +// pRuntimeEnv->current = *pTableQueryInfo; +// doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order); +// } // this function never returns error? uint32_t status; - int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); + int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); +// int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { longjmp(pOperator->pRuntimeEnv->env, code); } @@ -4742,7 +4721,6 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { return pBlock; } -#endif return NULL; } @@ -4750,9 +4728,8 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { static SSDataBlock* doTableScan(void* param, bool *newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; - STableScanInfo *pTableScanInfo = pOperator->info; - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STableScanInfo *pTableScanInfo = pOperator->info; + STaskInfo *pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; *newgroup = false; @@ -4775,14 +4752,14 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); // tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); - pRuntimeEnv->scanFlag = REPEAT_SCAN; - - if (pRuntimeEnv->pTsBuf) { - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } + setTaskStatus(pTaskInfo, QUERY_NOT_COMPLETED); + pTableScanInfo->scanFlag = REPEAT_SCAN; +// if (pTaskInfo->pTsBuf) { +// bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); +// assert(ret); +// } +// if (pResultRowInfo->size > 0) { pResultRowInfo->curPos = 0; } @@ -4792,17 +4769,15 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } SSDataBlock *p = NULL; + // todo refactor if (pTableScanInfo->reverseTimes > 0) { - setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); - + setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); // tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); //qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, // GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - pRuntimeEnv->scanFlag = REVERSE_SCAN; - pTableScanInfo->times = 1; pTableScanInfo->current = 0; pTableScanInfo->reverseTimes = 0; @@ -4867,31 +4842,31 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { } -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime) { - assert(repeatTime > 0); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime) { + assert(repeatTime > 0 && numOfOutput > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - pInfo->current = 0; -// pInfo->prevGroupId = -1; + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = 0; + pInfo->order = order; + pInfo->current = 0; + pInfo->scanFlag = MAIN_SCAN; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableScanOperator"; -// pOperator->operatorType = OP_TableScan; + pOperator->operatorType = OP_TableScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = NULL; pOperator->exec = doTableScan; return pOperator; } -SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); pInfo->pQueryHandle = pTsdbQueryHandle; @@ -4915,7 +4890,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeE return pOperator; } -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); pInfo->pQueryHandle = pTsdbQueryHandle; @@ -4998,7 +4973,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf } -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); @@ -5019,7 +4994,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime return pOptr; } -SArray* getOrderCheckColumns(SQueryAttr* pQuery) { +SArray* getOrderCheckColumns(STaskAttr* pQuery) { int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0: taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); SArray* pOrderColumns = NULL; @@ -5058,7 +5033,7 @@ SArray* getOrderCheckColumns(SQueryAttr* pQuery) { return pOrderColumns; } -SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { +SArray* getResultGroupCheckColumns(STaskAttr* pQuery) { int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); SArray* pOrderColumns = NULL; @@ -5109,7 +5084,7 @@ static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { tfree(pInfo->prevRow); } -SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, +SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp) { SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); @@ -5172,12 +5147,12 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, pOperator->exec = doGlobalAggregate; pOperator->cleanup = destroyGlobalAggOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, +SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, int32_t numOfRows, void *merger) { SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); @@ -5252,9 +5227,9 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { SSDataBlock* pBlock = NULL; while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); // start to flush data into disk and try do multiway merge sort if (pBlock == NULL) { @@ -5288,7 +5263,7 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; } -SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal) { +SOperatorInfo *createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal) { SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); { @@ -5321,7 +5296,7 @@ SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->cleanup = destroyOrderOperatorInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -5339,17 +5314,17 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5359,8 +5334,8 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); } -// if (upstream->operatorType == OP_DataBlocksOptScan) { -// STableScanInfo* pScanInfo = upstream->info; +// if (downstream->operatorType == OP_DataBlocksOptScan) { +// STableScanInfo* pScanInfo = downstream->info; // order = getTableScanOrder(pScanInfo); // } @@ -5386,7 +5361,7 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); @@ -5398,15 +5373,15 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { return pInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5414,8 +5389,8 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); -// if (upstream->operatorType == OP_DataBlocksOptScan) { -// STableScanInfo* pScanInfo = upstream->info; +// if (downstream->operatorType == OP_DataBlocksOptScan) { +// STableScanInfo* pScanInfo = downstream->info; // order = getTableScanOrder(pScanInfo); // } @@ -5455,7 +5430,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; SProjectOperatorInfo* pProjectInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; SOptrBasicInfo *pInfo = &pProjectInfo->binfo; SSDataBlock* pRes = pInfo->pRes; @@ -5492,16 +5467,16 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { while(1) { bool prevVal = *newgroup; - // The upstream exec may change the value of the newgroup, so use a local variable instead. - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + // The downstream exec may change the value of the newgroup, so use a local variable instead. + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { assert(*newgroup == false); *newgroup = prevVal; - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); break; } @@ -5547,13 +5522,13 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) { } SLimitOperatorInfo* pInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; SSDataBlock* pBlock = NULL; while (1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { doSetOperatorCompleted(pOperator); @@ -5599,12 +5574,12 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) { } SFilterOperatorInfo* pCondInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; while (1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock *pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock *pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5631,7 +5606,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { @@ -5641,16 +5616,16 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5669,7 +5644,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5690,7 +5665,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); @@ -5701,16 +5676,16 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5729,7 +5704,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5749,7 +5724,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { } STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { int64_t st = taosGetTimestampUs(); @@ -5765,15 +5740,15 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5792,7 +5767,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { @@ -5809,7 +5784,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { } STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); @@ -5820,15 +5795,15 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5847,7 +5822,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); int64_t st = taosGetTimestampUs(); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); @@ -5862,13 +5837,13 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { } static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; STableQueryInfo* item = pRuntimeEnv->current; SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); SOptrBasicInfo* pBInfo = &pInfo->binfo; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); int16_t bytes = pColInfoData->info.bytes; int16_t type = pColInfoData->info.type; @@ -5945,7 +5920,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { SStateWindowOperatorInfo* pWindowInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); @@ -5956,14 +5931,14 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { return pBInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5981,7 +5956,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); @@ -6004,7 +5979,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); @@ -6015,17 +5990,17 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { return pBInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; //pQueryAttr->order.order = TSDB_ORDER_ASC; int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; } @@ -6041,7 +6016,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); @@ -6062,7 +6037,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { SGroupbyOperatorInfo *pInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); @@ -6073,12 +6048,12 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { return pInfo->binfo.pRes; } - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; } @@ -6095,7 +6070,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pInfo->binfo.resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); @@ -6117,7 +6092,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { return pInfo->binfo.pRes; } -static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SQueryRuntimeEnv* pRuntimeEnv, bool* newgroup) { +static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); @@ -6130,7 +6105,7 @@ static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SQueryR *newgroup = true; } -static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, SQueryRuntimeEnv *pRuntimeEnv, bool *newgroup) { +static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, STaskRuntimeEnv *pRuntimeEnv, bool *newgroup) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { *newgroup = false; doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity, pInfo->p); @@ -6155,16 +6130,16 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { return NULL; } - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { return pInfo->pRes; } while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { assert(pBlock != NULL); @@ -6220,7 +6195,7 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { } // todo set the attribute of query scan count -static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { +static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr) { for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); if (functionId == FUNCTION_STDDEV || functionId == FUNCTION_PERCT) { @@ -6240,23 +6215,23 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { pOperator->cleanup(pOperator->info, pOperator->numOfOutput); } - if (pOperator->upstream != NULL) { - for(int32_t i = 0; i < pOperator->numOfUpstream; ++i) { - destroyOperatorInfo(pOperator->upstream[i]); + if (pOperator->pDownstream != NULL) { + for(int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + destroyOperatorInfo(pOperator->pDownstream[i]); } - tfree(pOperator->upstream); - pOperator->numOfUpstream = 0; + tfree(pOperator->pDownstream); + pOperator->numOfDownstream = 0; } tfree(pOperator->info); tfree(pOperator); } -SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfRows = (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); @@ -6265,7 +6240,7 @@ SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpera initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); pInfo->seed = rand(); - setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MASTER_SCAN); + setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MAIN_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; @@ -6279,7 +6254,7 @@ SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpera pOperator->exec = doAggregate; pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -6353,7 +6328,7 @@ static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = destroyOutputBuf(pInfo->pRes); } -SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); @@ -6374,12 +6349,12 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SO pOperator->exec = doSTableAggregate; pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); pInfo->seed = rand(); @@ -6390,7 +6365,7 @@ SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MASTER_SCAN); + setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "ProjectOperator"; @@ -6404,7 +6379,7 @@ SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->exec = doProjectOperation; pOperator->cleanup = destroyProjectOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -6442,7 +6417,7 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3 return 0; } -SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, +SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter) { SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); @@ -6462,12 +6437,12 @@ SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroyConditionOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { +SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream) { SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); pInfo->limit = pRuntimeEnv->pQueryAttr->limit.limit; @@ -6480,12 +6455,12 @@ SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->exec = doLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); @@ -6505,12 +6480,12 @@ SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOp pOperator->exec = doIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); @@ -6530,11 +6505,11 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, pOperator->exec = doAllIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); pInfo->colIndex = -1; pInfo->reptScan = false; @@ -6554,10 +6529,10 @@ SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpe pOperator->exec = doStateWindowAgg; pOperator->cleanup = destroyStateWindowOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); @@ -6579,11 +6554,11 @@ SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->exec = doSessionWindowAgg; pOperator->cleanup = destroySWindowOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); @@ -6603,11 +6578,11 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRunti pOperator->exec = doSTableIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); @@ -6627,20 +6602,20 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRu pOperator->exec = doAllSTableIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); pInfo->colIndex = -1; // group by column index pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); @@ -6660,17 +6635,17 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->exec = hashGroupbyAggregate; pOperator->cleanup = destroyGroupbyOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { +SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); pInfo->multigroupResult = multigroupResult; { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); STimeWindow w = TSWINDOW_INITIALIZER; @@ -6699,14 +6674,14 @@ SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->exec = doFill; pOperator->cleanup = destroySFillOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { +SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); pInfo->slimit = pQueryAttr->slimit; @@ -6747,7 +6722,7 @@ SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroySlimitOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -6758,7 +6733,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { return NULL; } - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; STagScanInfo *pInfo = pOperator->info; @@ -6770,7 +6745,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; assert(pQueryAttr->numOfOutput == 1); SExprInfo* pExprInfo = &pOperator->pExpr[0]; @@ -6874,7 +6849,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { } if (pOperator->status == OP_EXEC_DONE) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); } pRes->info.rows = count; @@ -6883,7 +6858,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { #endif } -SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); @@ -6968,9 +6943,9 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { SSDataBlock* pBlock = NULL; while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { doSetOperatorCompleted(pOperator); @@ -7021,7 +6996,7 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; } -SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); pInfo->totalBytes = 0; pInfo->buf = NULL; @@ -7045,7 +7020,7 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat pOperator->pExpr = pExpr; pOperator->cleanup = destroyDistinctOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -7199,6 +7174,92 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t return TSDB_CODE_SUCCESS; } +/** + * { + "Id": { + "QueryId": 20, + "TemplateId": 0, + "SubplanId": 0 + }, + "Node": { + "Name": "TableScan", + "InputSchema": [{ + "Type": 9, + "ColId": 1, + "Bytes": 8 + }, { + "Type": 4, + "ColId": 2, + "Bytes": 4 + }, { + "Type": 8, + "ColId": 3, + "Bytes": 20 + }], + "TableScan": { + "TableId": 1, + "TableType": 3, + "Flag": 0, + "Window": { + "StartKey": 0, + "EndKey": 0 + } + } + }, + "DataSink": { + "Name": "Dispatch", + "Dispatch": { + } + } +} + */ +int32_t parseTaskInfo(const char* msg, int32_t len) { + cJSON* pJson = cJSON_Parse(msg); + if (NULL == pJson) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pSub = cJSON_GetObjectItem(pJson, "ID"); + if (NULL != pSub) { + printf("Id : %s\n", pSub->valuestring); + } + + cJSON* pNode = cJSON_GetObjectItem(pJson, "Node"); + if (pNode == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pNodeName = cJSON_GetObjectItem(pNode, "name"); + if (pNodeName == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + printf("node name is: %s\n", pNodeName->valuestring); + + cJSON* pNodeSchema = cJSON_GetObjectItem(pNode, "InputSchema"); + if (pNodeSchema == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pOperator = cJSON_GetObjectItem(pNode, pNodeName->valuestring); + if (pOperator == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pTableId = cJSON_GetObjectItem(pOperator, "tableId"); + if (pTableId == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pTimeWindow = cJSON_GetObjectItem(pOperator, "window"); + if (pTimeWindow == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + + +} + /** * pQueryMsg->head has been converted before this function is called. * @@ -7207,7 +7268,7 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t * @param pExpr * @return */ -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, STaskParam* param) { int32_t code = TSDB_CODE_SUCCESS; // if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { @@ -7924,7 +7985,7 @@ void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFil return NULL; } -int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { +int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) { for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { // if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) { // pQueryAttr->numOfFilterCols++; @@ -7943,7 +8004,7 @@ int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { return TSDB_CODE_SUCCESS; } -static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { +static void doUpdateExprColumnIndex(STaskAttr *pQueryAttr) { assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL); for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) { @@ -7980,7 +8041,7 @@ static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { } } -void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo) { +void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo) { const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); // the minimum number of rows for projection query @@ -8026,7 +8087,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; - SQueryAttr* pQueryAttr = &pQInfo->query; + STaskAttr* pQueryAttr = &pQInfo->query; pQInfo->runtimeEnv.pQueryAttr = pQueryAttr; pQueryAttr->tableGroupInfo = *pTableGroupInfo; @@ -8145,7 +8206,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S pQueryAttr->window = pQueryMsg->window; updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; STimeWindow window = pQueryAttr->window; int32_t index = 0; @@ -8213,7 +8274,7 @@ _cleanup_qinfo: // filterFreeInfo(pFilters); _cleanup: - freeQInfo(pQInfo); + doDestroyTask(pQInfo); return NULL; } @@ -8231,14 +8292,14 @@ bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } -int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, +int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start, int32_t prevResultLen, void* merger) { int32_t code = TSDB_CODE_SUCCESS; - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; pRuntimeEnv->qinfo = pQInfo; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; STSBuf *pTsBuf = NULL; @@ -8271,7 +8332,7 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* (!QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.ekey > pQueryAttr->window.skey))) { //qDebug("QInfo:0x%"PRIx64" no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo->qId, pQueryAttr->window.skey, // pQueryAttr->window.ekey, pQueryAttr->order.order); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; // todo free memory return TSDB_CODE_SUCCESS; @@ -8279,7 +8340,7 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { //qDebug("QInfo:0x%"PRIx64" no table qualified for tag filter, abort query", pQInfo->qId); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; } @@ -8292,7 +8353,7 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* _error: // table query ref will be decrease during error handling - freeQInfo(pQInfo); + doDestroyTask(pQInfo); return code; } @@ -8373,20 +8434,20 @@ void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { return NULL; } -void freeQInfo(SQInfo *pQInfo) { +void doDestroyTask(SQInfo *pQInfo) { if (!isValidQInfo(pQInfo)) { return; } //qDebug("QInfo:0x%"PRIx64" start to free QInfo", pQInfo->qId); - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; freeQueryAttr(pQueryAttr); // tsdbDestroyTableGroup(&pQueryAttr->tableGroupInfo); @@ -8407,8 +8468,8 @@ void freeQInfo(SQInfo *pQInfo) { int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen) { // the remained number of retrieved rows, not the interpolated result - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; // load data from file to msg buffer if (pQueryAttr->tsCompQuery) { @@ -8445,7 +8506,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t // all data returned, set query over if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { - setQueryStatus(pRuntimeEnv, QUERY_OVER); +// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } else { doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data, compressed, compLen); @@ -8456,7 +8517,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t if (pQueryAttr->limit.limit > 0 && pQueryAttr->limit.limit == pRuntimeEnv->resultInfo.total) { //qDebug("QInfo:0x%"PRIx64" results limitation reached, limitation:%"PRId64, pQInfo->qId, pQueryAttr->limit.limit); - setQueryStatus(pRuntimeEnv, QUERY_OVER); +// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } return TSDB_CODE_SUCCESS; @@ -8535,8 +8596,8 @@ int32_t checkForQueryBuf(size_t numOfTables) { } bool checkNeedToCompressQueryCol(SQInfo *pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; SSDataBlock* pRes = pRuntimeEnv->outputBuf; @@ -8569,7 +8630,7 @@ void releaseQueryBuf(size_t numOfTables) { atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); } -void freeQueryAttr(SQueryAttr* pQueryAttr) { +void freeQueryAttr(STaskAttr* pQueryAttr) { if (pQueryAttr != NULL) { if (pQueryAttr->fillVal != NULL) { tfree(pQueryAttr->fillVal); diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 378af4c1d1..90ad1e15f4 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -19,6 +19,7 @@ #include "index.h" #include "index_fst.h" #include "taos.h" +#include "tchecksum.h" #include "thash.h" #include "tlog.h" diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index d7363f2f4c..1e0a88e17f 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -34,6 +34,7 @@ typedef struct WriterCtx { int (*read)(struct WriterCtx* ctx, uint8_t* buf, int len); int (*flush)(struct WriterCtx* ctx); int (*readFrom)(struct WriterCtx* ctx, uint8_t* buf, int len, int32_t offset); + int (*size)(struct WriterCtx* ctx); WriterType type; union { struct { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 04dac57d8f..19e9375491 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -34,7 +34,10 @@ void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); } -void indexCleanUp() { taosCleanUpScheduler(indexQhandle); } +void indexCleanUp() { + // refacto later + taosCleanUpScheduler(indexQhandle); +} static int uidCompare(const void* a, const void* b) { // add more version compare diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 8bc3776ed9..294c8192e8 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -21,7 +21,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later #define MEM_TERM_LIMIT 10 * 10000 -#define MEM_THRESHOLD 1024 * 1024 * 2 +#define MEM_THRESHOLD 1024 * 1024 #define MEM_ESTIMATE_RADIO 1.5 static void indexMemRef(MemTable* tbl); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index bfaeeaaa33..4f782cef26 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -935,7 +935,10 @@ Fst* fstCreate(FstSlice* slice) { uint32_t checkSum = 0; len -= sizeof(checkSum); taosDecodeFixedU32(buf + len, &checkSum); - + if (taosCheckChecksum(buf, len, checkSum)) { + // verify fst + return NULL; + } CompiledAddr rootAddr; len -= sizeof(rootAddr); taosDecodeFixedU64(buf + len, &rootAddr); diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 0763aae857..6db5555aa6 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -59,6 +59,13 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off } return nRead; } +static int writeCtxGetSize(WriterCtx* ctx) { + if (ctx->type == TFile && ctx->file.readOnly) { + // refactor later + return ctx->file.size; + } + return 0; +} static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { // taosFsyncFile(ctx->file.fd); @@ -109,6 +116,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int ctx->read = writeCtxDoRead; ctx->flush = writeCtxDoFlush; ctx->readFrom = writeCtxDoReadFrom; + ctx->size = writeCtxGetSize; ctx->offset = 0; ctx->limit = capacity; @@ -159,6 +167,8 @@ int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) int nWrite = ctx->write(ctx, buf, len); assert(nWrite == len); write->count += len; + + write->summer = taosCalcChecksum(write->summer, buf, len); return len; } int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) { @@ -169,7 +179,10 @@ int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) return nRead; } -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { return 0; } +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { + // opt + return write->summer; +} int fstCountingWriterFlush(FstCountingWriter* write) { WriterCtx* ctx = write->wrt; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 495c4d4477..4b76402560 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -21,8 +21,11 @@ p * #include "index_fst_counting_writer.h" #include "index_util.h" #include "taosdef.h" +#include "tcoding.h" #include "tcompare.h" +const static uint64_t tfileMagicNumber = 0xdb4775248b80fb57ull; + typedef struct TFileFstIter { FstStreamBuilder* fb; StreamWithState* st; @@ -40,9 +43,12 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds); static int tfileWriteHeader(TFileWriter* writer); static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset); static int tfileWriteData(TFileWriter* write, TFileValue* tval); +static int tfileWriteFooter(TFileWriter* write); +// handle file corrupt later static int tfileReaderLoadHeader(TFileReader* reader); static int tfileReaderLoadFst(TFileReader* reader); +static int tfileReaderVerify(TFileReader* reader); static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); static SArray* tfileGetFileList(const char* path); @@ -138,8 +144,14 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); if (reader == NULL) { return NULL; } - // T_REF_INC(reader); reader->ctx = ctx; + + if (0 != tfileReaderVerify(reader)) { + tfileReaderDestroy(reader); + indexError("invalid tfile, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); + return NULL; + } + // T_REF_INC(reader); if (0 != tfileReaderLoadHeader(reader)) { tfileReaderDestroy(reader); indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid, @@ -296,6 +308,8 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { fstBuilderFinish(tw->fb); fstBuilderDestroy(tw->fb); tw->fb = NULL; + + tfileWriteFooter(tw); return 0; } void tfileWriterClose(TFileWriter* tw) { @@ -502,6 +516,14 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) { } return 0; } +static int tfileWriteFooter(TFileWriter* write) { + char buf[sizeof(tfileMagicNumber) + 1] = {0}; + void* pBuf = (void*)buf; + taosEncodeFixedU64((void**)(void*)&pBuf, tfileMagicNumber); + int nwrite = write->ctx->write(write->ctx, buf, strlen(buf)); + assert(nwrite == sizeof(tfileMagicNumber)); + return nwrite; +} static int tfileReaderLoadHeader(TFileReader* reader) { // TODO simple tfile header later char buf[TFILE_HEADER_SIZE] = {0}; @@ -527,9 +549,14 @@ static int tfileReaderLoadFst(TFileReader* reader) { if (buf == NULL) { return -1; } WriterCtx* ctx = reader->ctx; - int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); - indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d ", nread, reader->header.fstOffset, ctx->file.buf, - ctx->file.size); + int size = ctx->size(ctx); + + int64_t ts = taosGetTimestampUs(); + int32_t nread = + ctx->readFrom(ctx, buf, size - reader->header.fstOffset - sizeof(tfileMagicNumber), reader->header.fstOffset); + int64_t cost = taosGetTimestampUs() - ts; + indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread, + reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost); // we assuse fst size less than FST_MAX_SIZE assert(nread > 0 && nread < FST_MAX_SIZE); @@ -558,6 +585,25 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* free(buf); return 0; } +static int tfileReaderVerify(TFileReader* reader) { + // just validate header and Footer, file corrupted also shuild be verified later + WriterCtx* ctx = reader->ctx; + + uint64_t tMagicNumber = 0; + + char buf[sizeof(tMagicNumber) + 1] = {0}; + int size = ctx->size(ctx); + + if (size < sizeof(tMagicNumber) || size <= sizeof(reader->header)) { + return -1; + } else if (ctx->readFrom(ctx, buf, sizeof(tMagicNumber), size - sizeof(tMagicNumber)) != sizeof(tMagicNumber)) { + return -1; + } + + taosDecodeFixedU64(buf, &tMagicNumber); + return tMagicNumber == tfileMagicNumber ? 0 : -1; +} + void tfileReaderRef(TFileReader* reader) { if (reader == NULL) { return; } int ref = T_REF_INC(reader); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 70671a5f3e..a2c0046f9a 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -1,4 +1,5 @@ +#include #include #include #include @@ -12,7 +13,6 @@ #include "index_tfile.h" #include "tskiplist.h" #include "tutil.h" - void* callback(void* s) { return s; } static std::string fileName = "/tmp/tindex.tindex"; @@ -293,7 +293,7 @@ void validateTFile(char* arg) { std::thread threads[NUM_OF_THREAD]; // std::vector threads; - TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1"); + TFileReader* reader = tfileReaderOpen(arg, 0, 20000000, "tag1"); for (int i = 0; i < NUM_OF_THREAD; i++) { threads[i] = std::thread(fst_get, reader->fst); @@ -306,13 +306,41 @@ void validateTFile(char* arg) { } tfCleanup(); } + +void iterTFileReader(char* path, char* ver) { + tfInit(); + + int version = atoi(ver); + TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1"); + Iterate* iter = tfileIteratorCreate(reader); + bool tn = iter ? iter->next(iter) : false; + int count = 0; + int termCount = 0; + while (tn == true) { + count++; + IterateValue* cv = iter->getValue(iter); + termCount += (int)taosArrayGetSize(cv->val); + printf("col val: %s, size: %d\n", cv->colVal, (int)taosArrayGetSize(cv->val)); + tn = iter->next(iter); + } + printf("total size: %d\n term count: %d\n", count, termCount); + + tfileIteratorDestroy(iter); + tfCleanup(); +} + int main(int argc, char* argv[]) { // tool to check all kind of fst test // if (argc > 1) { validateTFile(argv[1]); } + if (argc > 2) { + // opt + iterTFileReader(argv[1], argv[2]); + } // checkFstCheckIterator(); // checkFstLongTerm(); // checkFstPrefixSearch(); - checkMillonWriteAndReadOfFst(); + // checkMillonWriteAndReadOfFst(); + return 1; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 5438f88b76..4f3330b7b3 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -665,14 +665,19 @@ class IndexObj { size_t numOfTable = 100 * 10000) { std::string tColVal = colVal; size_t colValSize = tColVal.size(); + int skip = 100; + numOfTable /= skip; for (int i = 0; i < numOfTable; i++) { - tColVal[i % colValSize] = 'a' + i % 26; + for (int k = 0; k < 10 && k < colVal.size(); k++) { + // opt + tColVal[rand() % colValSize] = 'a' + k % 26; + } SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); - for (size_t i = 0; i < 10; i++) { - int ret = Put(terms, i); + for (size_t j = 0; j < skip; j++) { + int ret = Put(terms, j); assert(ret == 0); } indexMultiTermDestroy(terms); @@ -939,10 +944,11 @@ TEST_F(IndexEnv2, testIndex_read_performance) { TEST_F(IndexEnv2, testIndexMultiTag) { std::string path = "/tmp/multi_tag"; if (index->Init(path) != 0) {} - index->WriteMultiMillonData("tag1", "Hello", 100 * 10000); - index->WriteMultiMillonData("tag2", "Test", 100 * 10000); - index->WriteMultiMillonData("tag3", "Test", 100 * 10000); - index->WriteMultiMillonData("tag4", "Test", 100 * 10000); + int64_t st = taosGetTimestampUs(); + int32_t num = 1000 * 10000; + index->WriteMultiMillonData("tag1", "xxxxxxxxxxxxxxx", num); + std::cout << "numOfRow: " << num << "\ttime cost:" << taosGetTimestampUs() - st << std::endl; + // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000); } TEST_F(IndexEnv2, testLongComVal) { std::string path = "/tmp/long_colVal"; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 2a0f6b38eb..e2b8766700 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -112,6 +112,9 @@ public: int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { // todo vgInfo->vgId = 1; + vgInfo->numOfEps = 1; + vgInfo->epAddr[0].port = 6030; + strcpy(vgInfo->epAddr[0].fqdn, "node1"); return 0; } diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 6234dbe0ac..8d8c148fde 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,7 +8,8 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util catalog cjson parser transport function qcom + PRIVATE os util catalog cjson parser function qcom + PUBLIC transport ) if(${BUILD_TEST}) diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 63d2e9b855..2a50752c88 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -106,7 +106,7 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId); -int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); +void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); int32_t subPlanToString(const SSubplan *pPhyNode, char** str, int32_t* len); int32_t stringToSubplan(const char* str, SSubplan** subplan); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 547e3cc53c..a38d110d5f 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -75,24 +75,16 @@ int32_t dsinkNameToDsinkType(const char* name) { return DSINK_Unknown; } -static SDataSink* initDataSink(int32_t type, int32_t size) { - SDataSink* sink = (SDataSink*)validPointer(calloc(1, size)); - sink->info.type = type; - sink->info.name = dsinkTypeToDsinkName(type); - return sink; -} - -static SDataSink* createDataDispatcher(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { - SDataDispatcher* dispatcher = (SDataDispatcher*)initDataSink(DSINK_Dispatch, sizeof(SDataDispatcher)); - return (SDataSink*)dispatcher; -} - -static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks) { - SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter)); - inserter->numOfTables = pBlocks->numOfTables; - inserter->size = pBlocks->size; - SWAP(inserter->pData, pBlocks->pData, char*); - return (SDataSink*)inserter; +static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) { + dst->pSchema = malloc(sizeof(SSlotSchema) * src->numOfCols); + if (NULL == dst->pSchema) { + return false; + } + memcpy(dst->pSchema, src->pSchema, sizeof(SSlotSchema) * src->numOfCols); + dst->numOfCols = src->numOfCols; + dst->resultRowSize = src->resultRowSize; + dst->precision = src->precision; + return true; } static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { @@ -102,6 +94,10 @@ static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataB return false; } memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols); + dataBlockSchema->resultRowSize = 0; + for (int32_t i = 0; i < dataBlockSchema->numOfCols; ++i) { + dataBlockSchema->resultRowSize += dataBlockSchema->pSchema[i].bytes; + } return true; } @@ -120,13 +116,37 @@ static bool cloneExprArray(SArray** dst, SArray* src) { return (TSDB_CODE_SUCCESS == copyAllExprInfo(*dst, src, true) ? true : false); } +static SDataSink* initDataSink(int32_t type, int32_t size, const SPhyNode* pRoot) { + SDataSink* sink = (SDataSink*)validPointer(calloc(1, size)); + sink->info.type = type; + sink->info.name = dsinkTypeToDsinkName(type); + if (NULL !=pRoot && !copySchema(&sink->schema, &pRoot->targetSchema)) { + tfree(sink); + THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); + } + return sink; +} + +static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks, const SPhyNode* pRoot) { + SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter), pRoot); + inserter->numOfTables = pBlocks->numOfTables; + inserter->size = pBlocks->size; + SWAP(inserter->pData, pBlocks->pData, char*); + return (SDataSink*)inserter; +} + +static SDataSink* createDataDispatcher(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, const SPhyNode* pRoot) { + SDataDispatcher* dispatcher = (SDataDispatcher*)initDataSink(DSINK_Dispatch, sizeof(SDataDispatcher), pRoot); + return (SDataSink*)dispatcher; +} + static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { SPhyNode* node = (SPhyNode*)validPointer(calloc(1, size)); node->info.type = type; node->info.name = opTypeToOpName(type); if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { free(node); - return NULL; + THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); } return node; } @@ -149,7 +169,7 @@ static SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) { static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { // todo - return MASTER_SCAN; + return MAIN_SCAN; } static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { @@ -214,7 +234,7 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { static void vgroupInfoToEpSet(const SVgroupInfo* vg, SQueryNodeAddr* execNode) { execNode->nodeId = vg->vgId; - execNode->inUse = 0; // todo + execNode->inUse = vg->inUse; execNode->numOfEps = vg->numOfEps; for (int8_t i = 0; i < vg->numOfEps; ++i) { execNode->epAddr[i] = vg->epAddr[i]; @@ -240,7 +260,7 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod subplan->msgType = TDMT_VND_QUERY; vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execNode); subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); - subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode); + subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode, subplan->pNode); RECOVERY_CURRENT_SUBPLAN(pCxt); } return pCxt->nextId.templateId++; @@ -249,6 +269,7 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod static SPhyNode* createExchangeNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, uint64_t srcTemplateId) { SExchangePhyNode* node = (SExchangePhyNode*)initPhyNode(pPlanNode, OP_Exchange, sizeof(SExchangePhyNode)); node->srcTemplateId = srcTemplateId; + node->pSrcEndPoints = validPointer(taosArrayInit(TARRAY_MIN_SIZE, sizeof(SQueryNodeAddr))); return (SPhyNode*)node; } @@ -314,7 +335,7 @@ static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlan SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i); vgroupInfoToEpSet(&blocks->vg, &subplan->execNode); - subplan->pDataSink = createDataInserter(pCxt, blocks); + subplan->pDataSink = createDataInserter(pCxt, blocks, NULL); subplan->pNode = NULL; subplan->type = QUERY_TYPE_MODIFY; subplan->msgType = pPayload->msgType; @@ -333,7 +354,7 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { subplan->msgType = TDMT_VND_QUERY; subplan->pNode = createPhyNode(pCxt, pRoot); - subplan->pDataSink = createDataDispatcher(pCxt, pRoot); + subplan->pDataSink = createDataDispatcher(pCxt, pRoot, subplan->pNode); } // todo deal subquery } @@ -360,7 +381,24 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD return TSDB_CODE_SUCCESS; } -int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - //todo - return TSDB_CODE_SUCCESS; +void setExchangSourceNode(uint64_t templateId, SQueryNodeAddr* pEp, SPhyNode* pNode) { + if (NULL == pNode) { + return; + } + if (OP_Exchange == pNode->info.type) { + SExchangePhyNode* pExchange = (SExchangePhyNode*)pNode; + if (templateId == pExchange->srcTemplateId) { + taosArrayPush(pExchange->pSrcEndPoints, pEp); + } + } + if (pNode->pChildren != NULL) { + size_t size = taosArrayGetSize(pNode->pChildren); + for(int32_t i = 0; i < size; ++i) { + setExchangSourceNode(templateId, pEp, taosArrayGetP(pNode->pChildren, i)); + } + } +} + +void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* pEp) { + setExchangSourceNode(templateId, pEp, subplan->pNode); } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 20da1842cf..d67ff956b9 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -62,7 +62,7 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f return func(jObj, *obj); } -static bool addArray(cJSON* json, const char* name, FToJson func, const SArray* array) { +static bool addTarray(cJSON* json, const char* name, FToJson func, const SArray* array, bool isPoint) { size_t size = (NULL == array) ? 0 : taosArrayGetSize(array); if (size > 0) { cJSON* jArray = cJSON_AddArrayToObject(json, name); @@ -70,7 +70,7 @@ static bool addArray(cJSON* json, const char* name, FToJson func, const SArray* return false; } for (size_t i = 0; i < size; ++i) { - if (!addItem(jArray, func, taosArrayGetP(array, i))) { + if (!addItem(jArray, func, isPoint ? taosArrayGetP(array, i) : taosArrayGet(array, i))) { return false; } } @@ -78,11 +78,19 @@ static bool addArray(cJSON* json, const char* name, FToJson func, const SArray* return true; } -static bool fromArray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize) { +static bool addInlineArray(cJSON* json, const char* name, FToJson func, const SArray* array) { + return addTarray(json, name, func, array, false); +} + +static bool addArray(cJSON* json, const char* name, FToJson func, const SArray* array) { + return addTarray(json, name, func, array, true); +} + +static bool fromTarray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize, bool isPoint) { const cJSON* jArray = cJSON_GetObjectItem(json, name); int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); if (size > 0) { - *array = taosArrayInit(size, POINTER_BYTES); + *array = taosArrayInit(size, isPoint ? POINTER_BYTES : itemSize); if (NULL == *array) { return false; } @@ -92,11 +100,19 @@ static bool fromArray(const cJSON* json, const char* name, FFromJson func, SArra if (NULL == item || !func(cJSON_GetArrayItem(jArray, i), item)) { return false; } - taosArrayPush(*array, &item); + taosArrayPush(*array, isPoint ? &item : item); } return true; } +static bool fromInlineArray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize) { + return fromTarray(json, name, func, array, itemSize, false); +} + +static bool fromArray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize) { + return fromTarray(json, name, func, array, itemSize, true); +} + static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize, int32_t size) { if (size > 0) { cJSON* jArray = cJSON_AddArrayToObject(json, name); @@ -556,6 +572,32 @@ static bool epAddrFromJson(const cJSON* json, void* obj) { return true; } +static const char* jkNodeAddrId = "NodeId"; +static const char* jkNodeAddrInUse = "InUse"; +static const char* jkNodeAddrEpAddrs = "EpAddrs"; + +static bool nodeAddrToJson(const void* obj, cJSON* json) { + const SQueryNodeAddr* ep = (const SQueryNodeAddr*)obj; + bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, ep->nodeId); + if (res) { + res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, ep->inUse); + } + if (res) { + res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, ep->epAddr, ep->numOfEps, sizeof(SEpAddrMsg)); + } + return res; +} + +static bool nodeAddrFromJson(const cJSON* json, void* obj) { + SQueryNodeAddr* ep = (SQueryNodeAddr*)obj; + ep->nodeId = getNumber(json, jkNodeAddrId); + ep->inUse = getNumber(json, jkNodeAddrInUse); + int32_t numOfEps = 0; + bool res = fromRawArray(json, jkNodeAddrEpAddrs, nodeAddrFromJson, &ep->epAddr, sizeof(SEpAddrMsg), &numOfEps); + ep->numOfEps = numOfEps; + return res; +} + static const char* jkExchangeNodeSrcTemplateId = "SrcTemplateId"; static const char* jkExchangeNodeSrcEndPoints = "SrcEndPoints"; @@ -563,7 +605,7 @@ static bool exchangeNodeToJson(const void* obj, cJSON* json) { const SExchangePhyNode* exchange = (const SExchangePhyNode*)obj; bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId); if (res) { - res = addArray(json, jkExchangeNodeSrcEndPoints, epAddrToJson, exchange->pSrcEndPoints); + res = addInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrToJson, exchange->pSrcEndPoints); } return res; } @@ -571,7 +613,7 @@ static bool exchangeNodeToJson(const void* obj, cJSON* json) { static bool exchangeNodeFromJson(const cJSON* json, void* obj) { SExchangePhyNode* exchange = (SExchangePhyNode*)obj; exchange->srcTemplateId = getNumber(json, jkExchangeNodeSrcTemplateId); - return fromArray(json, jkExchangeNodeSrcEndPoints, epAddrFromJson, &exchange->pSrcEndPoints, sizeof(SEpAddrMsg)); + return fromInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrFromJson, &exchange->pSrcEndPoints, sizeof(SQueryNodeAddr)); } static bool specificPhyNodeToJson(const void* obj, cJSON* json) { @@ -803,7 +845,6 @@ static cJSON* subplanToJson(const SSubplan* subplan) { if (res) { res = addObject(jSubplan, jkSubplanDataSink, dataSinkToJson, subplan->pDataSink); } - if (!res) { cJSON_Delete(jSubplan); return NULL; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 54bddd0e3f..f80a631413 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -88,8 +88,8 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, return TSDB_CODE_SUCCESS; } -int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - return setSubplanExecutionNode(subplan, templateId, ep); +void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { + setSubplanExecutionNode(subplan, templateId, ep); } int32_t qSubPlanToString(const SSubplan *subplan, char** str, int32_t* len) { diff --git a/source/libs/qcom/test/CMakeLists.txt b/source/libs/qcom/test/CMakeLists.txt index 7adec3752a..e3a0e11a32 100644 --- a/source/libs/qcom/test/CMakeLists.txt +++ b/source/libs/qcom/test/CMakeLists.txt @@ -15,5 +15,5 @@ TARGET_INCLUDE_DIRECTORIES( TARGET_LINK_LIBRARIES( queryUtilTest - PUBLIC os util gtest qcom common + PUBLIC os util gtest qcom common transport ) diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp index ddf89c6272..8fc6b7e529 100644 --- a/source/libs/qcom/test/queryTest.cpp +++ b/source/libs/qcom/test/queryTest.cpp @@ -17,6 +17,7 @@ #include #include "tmsg.h" #include "query.h" +#include "trpc.h" #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" @@ -80,4 +81,4 @@ TEST(testCase, error_in_async_test) { taosAsyncExec(testPrintError, p, &code); usleep(1000); printf("Error code:%d after asynchronously exec function\n", code); -} \ No newline at end of file +} diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 4195376812..8923dc91cd 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -118,10 +118,10 @@ typedef struct SSchJob { #define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY) #define SCH_TASK_NO_NEED_DROP(task) ((task)->plan->type == QUERY_TYPE_MODIFY) -#define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) +#define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) -#define SCH_SET_JOB_STATUS(job, st) atomic_store_8(&(job)->status, st) +#define SCH_SET_JOB_STATUS(job, st) atomic_store_8(&(job)->status, st) #define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status) #define SCH_SET_JOB_TYPE(pAttr, type) (pAttr)->queryJob = ((type) != QUERY_TYPE_MODIFY) diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 871c95193f..7631593dd8 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -131,6 +131,7 @@ int walMetaDeserialize(SWal* pWal, const char* bytes); // seek section int walChangeFile(SWal* pWal, int64_t ver); +int walChangeFileToLast(SWal* pWal); // seek section end int64_t walGetSeq(); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 6164ec2baf..270a26bf80 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -184,6 +184,7 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { } taosArraySetSize(pArray, sz); pWal->fileInfoSet = pArray; + pWal->writeCur = sz - 1; cJSON_Delete(pRoot); return 0; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 189881c86d..d12acb52c6 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -232,7 +232,8 @@ static int32_t walCreateThread() { if (pthread_create(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { wError("failed to create wal thread since %s", strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } pthread_attr_destroy(&thAttr); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 39bf06cedd..975f232e3d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -244,6 +244,7 @@ int walRoll(SWal *pWal) { pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; + ASSERT(pWal->writeCur >= 0); pWal->lastRollSeq = walGetSeq(); return 0; @@ -253,6 +254,7 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { + terrno = TAOS_SYSTEM_ERROR(errno); // TODO truncate return -1; } @@ -286,7 +288,13 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i } /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ + ASSERT(pWal->writeCur >= 0); + pthread_mutex_lock(&pWal->mutex); + if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) { + walChangeFileToLast(pWal); + } + pWal->writeHead.head.version = index; int64_t offset = walGetCurFileOffset(pWal); @@ -308,6 +316,7 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } + code = walWriteIndex(pWal, index, offset); if (code != 0) { // TODO diff --git a/source/util/src/terror.c b/source/util/src/terror.c index eaaaf28297..3ea564722b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -277,9 +277,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_NOT_DEPLOYED, "Bnode not deployed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_INVALID_OPTION, "Bnode option invalid") TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_READ_FILE_ERROR, "Read bnode.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR, "Write bnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnode directories") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED, "Vnode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_NOT_DEPLOYED, "Vnode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_INVALID_OPTION, "Vnode option invalid") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_READ_FILE_ERROR, "Read vnodes.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR, "Write vnodes.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnodes") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, "Action in progress") diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 2f4e43b13f..2b013bfdd0 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -812,7 +812,9 @@ FORCE_INLINE int32_t taosHashGetKey(void *data, void** key, size_t* keyLen) { SHashNode * node = GET_HASH_PNODE(data); *key = GET_HASH_NODE_KEY(node); - *keyLen = node->keyLen; + if (keyLen) { + *keyLen = node->keyLen; + } return 0; }