From b9927eae8516b2a2eb0c62ed13c902aabcd5fe44 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 21 Dec 2021 17:18:39 +0800 Subject: [PATCH 01/14] [td-11818] support async exec task. --- include/libs/qcom/query.h | 22 +++++++++++--- source/client/inc/clientInt.h | 2 +- source/client/src/clientImpl.c | 3 +- source/client/src/clientMain.c | 13 ++------ source/client/src/tscEnv.c | 24 ++++----------- source/client/test/clientTests.cpp | 15 +++++++++ source/libs/parser/inc/parserUtil.h | 1 + source/libs/parser/src/parserUtil.c | 9 +++++- source/libs/qcom/CMakeLists.txt | 2 ++ source/libs/qcom/src/queryUtil.c | 47 ++++++++++++++++++++++++++++- 10 files changed, 100 insertions(+), 38 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 254d572149..70dcc94280 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -81,16 +81,28 @@ typedef struct STableMetaOutput { STableMeta *tbMeta; } STableMetaOutput; +typedef int32_t __async_exec_fn_t(void* param); + bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); +int32_t initTaskQueue(); +int32_t cleanupTaskQueue(); + +/** + * + * @param execFn The asynchronously execution function + * @param execParam The parameters of the execFn + * @param code The response code during execution the execFn + * @return + */ +int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); + +SSchema* tGetTbnameColumnSchema(); +void msgInit(); + extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); -SSchema* tGetTbnameColumnSchema(); -extern void msgInit(); - -extern int32_t qDebugFlag; - #define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) #define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) #define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index a1a9155d16..c1fd113da7 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -154,7 +154,7 @@ void destroyRequest(SRequestObj* pRequest); void taos_init_imp(void); int taos_options_imp(TSDB_OPTION option, const char *str); -void* openTransporter(const char *user, const char *auth); +void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index bab1f9cc9b..0acb74d4a9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -102,9 +102,8 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL) { SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); - p->mgmtEp = epSet; - p->pTransporter = openTransporter(user, secretEncrypt); + p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 8a75799ed7..dce04a869b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1,16 +1,11 @@ +#include "os.h" #include "clientInt.h" #include "clientLog.h" -#include "os.h" +#include "query.h" #include "taosmsg.h" -#include "tcache.h" -#include "tconfig.h" #include "tglobal.h" -#include "tnote.h" #include "tref.h" #include "trpc.h" -#include "tsched.h" -#include "ttime.h" -#include "ttimezone.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -44,9 +39,7 @@ void taos_cleanup(void) { tscReqRef = -1; taosCloseRef(id); - void* p = tscQhandle; - tscQhandle = NULL; - taosCleanUpScheduler(p); + cleanupTaskQueue(); id = tscConnRef; tscConnRef = -1; diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 182e330df7..b7a622c2e8 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -13,17 +13,17 @@ * along with this program. If not, see . */ -#include "clientInt.h" -#include "clientLog.h" #include "os.h" #include "taosmsg.h" +#include "query.h" +#include "clientInt.h" +#include "clientLog.h" #include "tcache.h" #include "tconfig.h" #include "tglobal.h" #include "tnote.h" #include "tref.h" #include "trpc.h" -#include "tsched.h" #include "ttime.h" #include "ttimezone.h" @@ -33,10 +33,8 @@ SAppInfo appInfo; int32_t tscReqRef = -1; int32_t tscConnRef = -1; -void *tscQhandle = NULL; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -int32_t tsNumOfThreads = 1; volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj* pRequest) { @@ -98,12 +96,12 @@ void closeTransporter(STscObj* pTscObj) { } // TODO refactor -void* openTransporter(const char *user, const char *auth) { +void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "TSC"; - rpcInit.numOfThreads = tsNumOfThreads; + rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; @@ -229,18 +227,8 @@ void taos_init_imp(void) { taosSetCoreDump(true); - double factor = 4.0; - int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + initTaskQueue(); - int32_t queueSize = tsMaxConnections * 2; - tscQhandle = taosInitScheduler(queueSize, numOfThreads, "tsc"); - if (NULL == tscQhandle) { - tscError("failed to init task queue"); - tscInitRes = -1; - return; - } - - tscDebug("client task queue is initialized, numOfThreads: %d", numOfThreads); tscConnRef = taosOpenRef(200, destroyTscObj); tscReqRef = taosOpenRef(40960, doDestroyRequest); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 1c8cd6a4b6..2e89bb45fa 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -148,3 +148,18 @@ TEST(testCase, create_db_Test) { taos_close(pConn); } + +TEST(testCase, create_stable_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index c588a34a40..ad43b9876a 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -67,6 +67,7 @@ int32_t getExprFunctionId(SExprInfo *pExprInfo); STableMeta* tableMetaDup(const STableMeta* pTableMeta); bool isDclSqlStatement(SSqlInfo* pSqlInfo); +bool isDdlSqlStatement(SSqlInfo* pSqlInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index b72bc06324..13434da057 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1613,7 +1613,14 @@ uint32_t convertRelationalOperator(SToken *pToken) { } bool isDclSqlStatement(SSqlInfo* pSqlInfo) { - return (pSqlInfo->type != TSDB_SQL_SELECT); + int32_t type = pSqlInfo->type; + return (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_CREATE_ACCT || type == TSDB_SQL_DROP_USER || + type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_SHOW); +} + +bool isDdlSqlStatement(SSqlInfo* pSqlInfo) { + int32_t type = pSqlInfo->type; + return (type == TSDB_SQL_CREATE_TABLE || type == TSDB_SQL_CREATE_DB); } #if 0 diff --git a/source/libs/qcom/CMakeLists.txt b/source/libs/qcom/CMakeLists.txt index 41cf1826bc..8e09f3d97a 100644 --- a/source/libs/qcom/CMakeLists.txt +++ b/source/libs/qcom/CMakeLists.txt @@ -10,3 +10,5 @@ target_link_libraries( qcom PRIVATE os util transport ) + +ADD_SUBDIRECTORY(test) diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 2a13b708ec..829f426c9d 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -1,5 +1,8 @@ #include "os.h" #include "taosmsg.h" +#include "query.h" +#include "tglobal.h" +#include "tsched.h" #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) #define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) @@ -75,4 +78,46 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTag } return true; -} \ No newline at end of file +} + +static void* pTaskQueue = NULL; + +int32_t initTaskQueue() { + double factor = 4.0; + int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + + int32_t queueSize = tsMaxConnections * 2; + pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); + if (NULL == pTaskQueue) { + qError("failed to init task queue"); + return -1; + } + + qDebug("task queue is initialized, numOfThreads: %d", numOfThreads); +} + +int32_t cleanupTaskQueue() { + taosCleanUpScheduler(pTaskQueue); +} + +static void execHelper(struct SSchedMsg* pSchedMsg) { + assert(pSchedMsg != NULL && pSchedMsg->ahandle != NULL); + + __async_exec_fn_t* execFn = (__async_exec_fn_t*) pSchedMsg->ahandle; + int32_t code = execFn(pSchedMsg->thandle); + if (code != 0 && pSchedMsg->msg != NULL) { + *(int32_t*) pSchedMsg->msg = code; + } +} + +int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) { + assert(execFn != NULL); + + SSchedMsg schedMsg = {0}; + schedMsg.fp = execHelper; + schedMsg.ahandle = execFn; + schedMsg.thandle = execParam; + schedMsg.msg = code; + + taosScheduleTask(pTaskQueue, &schedMsg); +} From 6beee204d825ef8a5fc0a2412f1064a1bb9ae615 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 21 Dec 2021 20:07:32 +0800 Subject: [PATCH 02/14] add tq query --- include/dnode/vnode/tq/tq.h | 29 +++-- include/dnode/vnode/vnode.h | 10 ++ include/libs/wal/wal.h | 5 +- include/util/ttimer.h | 2 + source/dnode/vnode/tq/CMakeLists.txt | 1 + source/dnode/vnode/tq/inc/tqInt.h | 1 + source/dnode/vnode/tq/src/tq.c | 182 ++++++++++++++++++++++----- source/libs/wal/src/walRead.c | 8 +- 8 files changed, 197 insertions(+), 41 deletions(-) diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 60a8c252c0..5eeaaa1011 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -22,6 +22,7 @@ #include "taoserror.h" #include "taosmsg.h" #include "tlist.h" +#include "trpc.h" #include "tutil.h" #ifdef __cplusplus @@ -54,6 +55,7 @@ typedef struct STqSetCurReq { typedef struct STqConsumeReq { STqMsgHead head; + int64_t blockingTime; // milisec STqAcks acks; } STqConsumeReq; @@ -107,6 +109,17 @@ typedef struct STqExec { struct STqExec* (*deserialize)(char*); } STqExec; +typedef struct STqRspHandle { + void* handle; + void* ahandle; +} STqRspHandle; + +typedef enum { + TQ_ITEM_READY, + TQ_ITEM_PROCESS, + TQ_ITEM_EMPTY +} STqItemStatus; + typedef struct STqBufferItem { int64_t offset; // executors are identical but not concurrent @@ -135,13 +148,13 @@ typedef struct STqListHandle { } STqList; typedef struct STqGroup { - int64_t clientId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - STqList* head; - SList* topicList; // SList - void* returnMsg; // SVReadMsg + int64_t clientId; + int64_t cgId; + void* ahandle; + int32_t topicNum; + //STqList* head; + SList* topicList; // SList + STqRspHandle rspHandle; } STqGroup; typedef struct STqQueryMsg { @@ -264,7 +277,7 @@ void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); -int tqConsume(STQ*, STqConsumeReq*); +int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); int tqSetCursor(STQ*, STqSetCurReq* pMsg); int tqBufferSetOffset(STqTopic*, int64_t offset); diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 8458ad9da3..a373828ebd 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -122,6 +122,16 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs); */ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +/** + * @brief Process a consume message. + * + * @param pVnode The vnode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int 0 for success, -1 for failure + */ +int vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); + /** * @brief Process the sync request * diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 89f24cf3a4..67d2009d3b 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -174,8 +174,11 @@ SWalReadHandle *walOpenReadHandle(SWal *); void walCloseReadHandle(SWalReadHandle *); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); +// deprecated +#if 0 int32_t walRead(SWal *, SWalHead **, int64_t ver); -// int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); +int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); +#endif // lifecycle check int64_t walGetFirstVer(SWal *); diff --git a/include/util/ttimer.h b/include/util/ttimer.h index 987d3f3cdc..89ec6cd8d9 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -16,6 +16,8 @@ #ifndef _TD_UTIL_TIMER_H #define _TD_UTIL_TIMER_H +#include "os.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/source/dnode/vnode/tq/CMakeLists.txt b/source/dnode/vnode/tq/CMakeLists.txt index 8d59c7b07a..7cb7499d64 100644 --- a/source/dnode/vnode/tq/CMakeLists.txt +++ b/source/dnode/vnode/tq/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common + PUBLIC transport ) if(${BUILD_TEST}) diff --git a/source/dnode/vnode/tq/inc/tqInt.h b/source/dnode/vnode/tq/inc/tqInt.h index 5685a29d03..107f5d5103 100644 --- a/source/dnode/vnode/tq/inc/tqInt.h +++ b/source/dnode/vnode/tq/inc/tqInt.h @@ -18,6 +18,7 @@ #include "tq.h" #include "tlog.h" +#include "trpc.h" #ifdef __cplusplus extern "C" { #endif diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index d8dfe4ddcf..972740183d 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -14,7 +14,9 @@ */ #include "tqInt.h" +#include "osSocket.h" #include "tqMetaStore.h" +#include "osAtomic.h" // static // read next version data @@ -51,16 +53,22 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemA } pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0); if (pTq->tqMeta == NULL) { - // TODO: free STQ + free(pTq); + allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); return NULL; } + return pTq; } + void tqClose(STQ* pTq) { // TODO } -static int tqProtoCheck(STqMsgHead* pMsg) { return pMsg->protoVer == 0; } +static int tqProtoCheck(STqMsgHead* pMsg) { + // TODO + return pMsg->protoVer == 0; +} static int tqAckOneTopic(STqTopic* pTopic, STqOneAck* pAck, STqQueryMsg** ppQuery) { // clean old item and move forward @@ -121,9 +129,15 @@ int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup // TODO return -1; } - *ppGroup = pGroup; memset(pGroup, 0, sizeof(STqGroup)); + pGroup->topicList = tdListNew(sizeof(STqTopic)); + if(pGroup->topicList == NULL) { + free(pGroup); + return -1; + } + *ppGroup = pGroup; + return 0; } @@ -152,46 +166,55 @@ int tqDropGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; } -static int tqFetch(STqGroup* pGroup, void** msg) { - STqList* head = pGroup->head; - STqList* node = head; +static int tqFetch(STqGroup* pGroup, STqConsumeRsp** pRsp) { + STqList* pHead = pGroup->head; + STqList* pNode = pHead; int totSize = 0; + int numOfMsgs = 0; // TODO: make it a macro - int sizeLimit = 4 * 1024; - STqMsgContent* buffer = malloc(sizeLimit); - if (buffer == NULL) { - // TODO:memory insufficient + int sizeLimit = 4 * 1024; + + void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + sizeLimit); + if (ptr == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } + *pRsp = ptr; + STqMsgContent* buffer = (*pRsp)->msgs; + // iterate the list to get msgs of all topics // until all topic iterated or msgs over sizeLimit - while (node->next) { - node = node->next; - STqTopic* topicHandle = &node->topic; - int idx = topicHandle->nextConsumeOffset % TQ_BUFFER_SIZE; - if (topicHandle->buffer[idx].content != NULL && topicHandle->buffer[idx].offset == topicHandle->nextConsumeOffset) { - totSize += topicHandle->buffer[idx].size; + while (pNode->next) { + pNode = pNode->next; + STqTopic* pTopic = &pNode->topic; + int idx = pTopic->nextConsumeOffset % TQ_BUFFER_SIZE; + if (pTopic->buffer[idx].content != NULL && pTopic->buffer[idx].offset == pTopic->nextConsumeOffset) { + totSize += pTopic->buffer[idx].size; if (totSize > sizeLimit) { - void* ptr = realloc(buffer, totSize); + void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + totSize); if (ptr == NULL) { - totSize -= topicHandle->buffer[idx].size; - // TODO:memory insufficient + totSize -= pTopic->buffer[idx].size; + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; // return msgs already copied break; } + *pRsp = ptr; + break; } - *((int64_t*)buffer) = topicHandle->topicId; + *((int64_t*)buffer) = pTopic->topicId; buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = topicHandle->buffer[idx].size; + *((int64_t*)buffer) = pTopic->buffer[idx].size; buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, topicHandle->buffer[idx].content, topicHandle->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, topicHandle->buffer[idx].size); + memcpy(buffer, pTopic->buffer[idx].content, pTopic->buffer[idx].size); + buffer = POINTER_SHIFT(buffer, pTopic->buffer[idx].size); + numOfMsgs++; if (totSize > sizeLimit) { break; } } } - return totSize; + (*pRsp)->bodySize = totSize; + return numOfMsgs; } STqGroup* tqGetGroup(STQ* pTq, int64_t clientId) { return tqHandleGet(pTq->tqMeta, clientId); } @@ -273,7 +296,22 @@ int tqSetCursor(STQ* pTq, STqSetCurReq* pMsg) { return 0; } -int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { +// temporary +int tqProcessCMsg(STQ* pTq, STqConsumeReq* pMsg, STqRspHandle* pRsp) { + int64_t clientId = pMsg->head.clientId; + STqGroup* pGroup = tqGetGroup(pTq, clientId); + if (pGroup == NULL) { + terrno = TSDB_CODE_TQ_GROUP_NOT_SET; + return -1; + } + pGroup->rspHandle.handle = pRsp->handle; + pGroup->rspHandle.ahandle = pRsp->ahandle; + + return 0; +} + +int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { + STqConsumeReq *pMsg = pReq->pCont; int64_t clientId = pMsg->head.clientId; STqGroup* pGroup = tqGetGroup(pTq, clientId); if (pGroup == NULL) { @@ -281,17 +319,103 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { return -1; } - STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; - int numOfMsgs = tqFetch(pGroup, (void**)&pRsp->msgs); + SList* topicList = pGroup->topicList; + + int totSize = 0; + int numOfMsgs = 0; + int sizeLimit = 4096; + + + STqConsumeRsp *pCsmRsp = (*pRsp)->pCont; + void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit); + if (ptr == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; + } + (*pRsp)->pCont = ptr; + + SListIter iter; + tdListInitIter(topicList, &iter, TD_LIST_FORWARD); + + STqMsgContent* buffer = NULL; + SArray* pArray = taosArrayInit(0, sizeof(void*)); + + SListNode *pn; + while((pn = tdListNext(&iter)) != NULL) { + STqTopic* pTopic = *(STqTopic**)pn->data; + int idx = pTopic->floatingCursor % TQ_BUFFER_SIZE; + STqMsgItem* pItem = &pTopic->buffer[idx]; + if (pItem->content != NULL && pItem->offset == pTopic->floatingCursor) { + if(pItem->status == TQ_ITEM_READY) { + //if has data + totSize += pTopic->buffer[idx].size; + if (totSize > sizeLimit) { + void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + totSize); + if (ptr == NULL) { + totSize -= pTopic->buffer[idx].size; + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + // return msgs already copied + break; + } + (*pRsp)->pCont = ptr; + break; + } + *((int64_t*)buffer) = htonll(pTopic->topicId); + buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); + *((int64_t*)buffer) = htonll(pTopic->buffer[idx].size); + buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); + memcpy(buffer, pTopic->buffer[idx].content, pTopic->buffer[idx].size); + buffer = POINTER_SHIFT(buffer, pTopic->buffer[idx].size); + numOfMsgs++; + if (totSize > sizeLimit) { + break; + } + } else if(pItem->status == TQ_ITEM_PROCESS) { + //if not have data but in process + + } else if(pItem->status == TQ_ITEM_EMPTY){ + //if not have data and not in process + int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_EMPTY, TQ_ITEM_PROCESS); + if(old != TQ_ITEM_EMPTY) { + continue; + } + pItem->offset = pTopic->floatingCursor; + taosArrayPush(pArray, &pItem); + } else { + ASSERT(0); + } + + } + } + + for(int i = 0; i < pArray->size; i++) { + STqMsgItem* pItem = taosArrayGet(pArray, i); + + void* raw; + //read from wal + //get msgType + //if submitblk + pItem->executor->assign(pItem->executor->runtimeEnv, raw); + SSDataBlock* content = pItem->executor->exec(pItem->executor->runtimeEnv); + pItem->content = content; + //if other type, send just put into buffer + pItem->content = raw; + + int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_PROCESS, TQ_ITEM_READY); + ASSERT(old == TQ_ITEM_PROCESS); + + } + if (numOfMsgs < 0) { return -1; } + if (numOfMsgs == 0) { // most recent data has been fetched // enable timer for blocking wait - // once new data written during wait time - // launch query and response + // once new data written when waiting, launch query and rsp + return -1; } // fetched a num of msgs, rpc response diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 48eb84b536..c80fb4eed8 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -170,6 +170,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { return 0; } +#if 0 int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int code; code = walSeekVer(pWal, ver); @@ -207,6 +208,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { return 0; } -/*int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {*/ -/*return 0;*/ -/*}*/ +int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) { +return 0; +} +#endif From 4cdcf1cc733e2624814501e5273288f9db2d82b2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 22 Dec 2021 00:39:11 -0500 Subject: [PATCH 03/14] TD-12416 insert statement plan --- include/libs/parser/parsenodes.h | 161 ++++++++++++++++++ include/libs/parser/parser.h | 127 +------------- include/libs/planner/planner.h | 27 +-- source/libs/parser/inc/dataBlockMgt.h | 2 +- source/libs/parser/src/dataBlockMgt.c | 3 - source/libs/parser/src/insertParser.c | 72 ++++---- source/libs/parser/src/parserUtil.c | 2 +- source/libs/parser/src/queryInfoUtil.c | 2 +- source/libs/parser/test/insertParserTest.cpp | 2 +- source/libs/parser/test/mockCatalog.cpp | 17 +- .../libs/parser/test/mockCatalogService.cpp | 19 ++- source/libs/parser/test/mockCatalogService.h | 5 +- source/libs/parser/test/plannerTest.cpp | 4 +- source/libs/planner/inc/plannerInt.h | 3 +- source/libs/planner/src/logicPlan.c | 50 ++++-- source/libs/planner/src/physicalPlan.c | 108 +++++++++--- source/libs/planner/src/physicalPlanJson.c | 2 +- source/libs/planner/src/planner.c | 4 +- 18 files changed, 380 insertions(+), 230 deletions(-) create mode 100644 include/libs/parser/parsenodes.h diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h new file mode 100644 index 0000000000..01104ab5d8 --- /dev/null +++ b/include/libs/parser/parsenodes.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_PARSENODES_H_ +#define _TD_PARSENODES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "catalog.h" +#include "common.h" +#include "function.h" +#include "tmsgtype.h" +#include "tname.h" +#include "tvariant.h" + +/* + * The first field of a node of any type is guaranteed to be the int16_t. + * Hence the type of any node can be gotten by casting it to SQueryNode. + */ +typedef struct SQueryNode { + int16_t type; +} SQueryNode; + +#define nodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type) + +typedef struct SField { + char name[TSDB_COL_NAME_LEN]; + uint8_t type; + int16_t bytes; +} SField; + +typedef struct SFieldInfo { + int16_t numOfOutput; // number of column in result + SField *final; + SArray *internalField; // SArray +} SFieldInfo; + +typedef struct SCond { + uint64_t uid; + int32_t len; // length of tag query condition data + char * cond; +} SCond; + +typedef struct SJoinNode { + uint64_t uid; + int16_t tagColId; + SArray* tsJoin; + SArray* tagJoin; +} SJoinNode; + +typedef struct SJoinInfo { + bool hasJoin; + SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; +} SJoinInfo; + +typedef struct STagCond { + int16_t relType; // relation between tbname list and query condition, including : TK_AND or TK_OR + SCond tbnameCond; // tbname query condition, only support tbname query condition on one table + SJoinInfo joinInfo; // join condition, only support two tables join currently + SArray *pCond; // for different table, the query condition must be seperated +} STagCond; + +typedef struct STableMetaInfo { + STableMeta *pTableMeta; // table meta, cached in client side and acquired by name + SVgroupsInfo *vgroupList; + SName name; + char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql + SArray *tagColList; // SArray, involved tag columns +} STableMetaInfo; + +typedef struct SColumnIndex { + int16_t tableIndex; + int16_t columnIndex; + int16_t type; // normal column/tag/ user input constant column +} SColumnIndex; + +// select statement +typedef struct SQueryStmtInfo { + int16_t command; // the command may be different for each subclause, so keep it seperately. + uint32_t type; // query/insert type + STimeWindow window; // the whole query time window + SInterval interval; // tumble time window + SSessionWindow sessionWindow; // session time window + SStateWindow stateWindow; // state window query + SGroupbyExpr groupbyExpr; // groupby tags info + SArray * colList; // SArray + SFieldInfo fieldsInfo; + SArray** exprList; // SArray + SLimit limit; + SLimit slimit; + STagCond tagCond; + SArray * colCond; + SArray * order; + int16_t numOfTables; + int16_t curTableIdx; + STableMetaInfo **pTableMetaInfo; + struct STSBuf *tsBuf; + + int16_t fillType; // final result fill type + int64_t * fillVal; // default value for fill + int32_t numOfFillVal; // fill value size + + char * msg; // pointer to the pCmd->payload to keep error message temporarily + int64_t clauseLimit; // limit for current sub clause + + int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit + + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX + int32_t bufLen; + char* buf; + SArray *pUdfInfo; + + struct SQueryStmtInfo *sibling; // sibling + struct SQueryStmtInfo *pDownstream; + SMultiFunctionsDesc info; + SArray *pUpstream; // SArray + int32_t havingFieldNum; + int32_t exprListLevelIndex; +} SQueryStmtInfo; + +typedef enum { + PAYLOAD_TYPE_KV = 0, + PAYLOAD_TYPE_RAW = 1, +} EPayloadType; + +typedef struct SVgDataBlocks { + SVgroupInfo vg; + int32_t numOfTables; // number of tables in current submit block + uint32_t size; + char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... +} SVgDataBlocks; + +typedef struct SInsertStmtInfo { + int16_t nodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + int8_t schemaAttache; // denote if submit block is built with table schema or not + uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* sql; // current sql statement position +} SInsertStmtInfo; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_PARSENODES_H_*/ diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index df2170423f..737d5c258c 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -20,108 +20,7 @@ extern "C" { #endif -#include "catalog.h" -#include "common.h" -#include "tname.h" -#include "tvariant.h" -#include "function.h" - -typedef struct SField { - char name[TSDB_COL_NAME_LEN]; - uint8_t type; - int16_t bytes; -} SField; - -typedef struct SFieldInfo { - int16_t numOfOutput; // number of column in result - SField *final; - SArray *internalField; // SArray -} SFieldInfo; - -typedef struct SCond { - uint64_t uid; - int32_t len; // length of tag query condition data - char * cond; -} SCond; - -typedef struct SJoinNode { - uint64_t uid; - int16_t tagColId; - SArray* tsJoin; - SArray* tagJoin; -} SJoinNode; - -typedef struct SJoinInfo { - bool hasJoin; - SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; -} SJoinInfo; - -typedef struct STagCond { - int16_t relType; // relation between tbname list and query condition, including : TK_AND or TK_OR - SCond tbnameCond; // tbname query condition, only support tbname query condition on one table - SJoinInfo joinInfo; // join condition, only support two tables join currently - SArray *pCond; // for different table, the query condition must be seperated -} STagCond; - -typedef struct STableMetaInfo { - STableMeta *pTableMeta; // table meta, cached in client side and acquired by name - SVgroupsInfo *vgroupList; - SName name; - char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql - SArray *tagColList; // SArray, involved tag columns -} STableMetaInfo; - -typedef struct SQueryStmtInfo { - int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert type - STimeWindow window; // the whole query time window - SInterval interval; // tumble time window - SSessionWindow sessionWindow; // session time window - SStateWindow stateWindow; // state window query - SGroupbyExpr groupbyExpr; // groupby tags info - SArray * colList; // SArray - SFieldInfo fieldsInfo; - SArray** exprList; // SArray - SLimit limit; - SLimit slimit; - STagCond tagCond; - SArray * colCond; - SArray * order; - int16_t numOfTables; - int16_t curTableIdx; - STableMetaInfo **pTableMetaInfo; - struct STSBuf *tsBuf; - - int16_t fillType; // final result fill type - int64_t * fillVal; // default value for fill - int32_t numOfFillVal; // fill value size - - char * msg; // pointer to the pCmd->payload to keep error message temporarily - int64_t clauseLimit; // limit for current sub clause - - int64_t prjOffset; // offset value in the original sql expression, only applied at client side - int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit - - int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX - int32_t bufLen; - char* buf; - SArray *pUdfInfo; - - struct SQueryStmtInfo *sibling; // sibling - struct SQueryStmtInfo *pDownstream; - SMultiFunctionsDesc info; - SArray *pUpstream; // SArray - int32_t havingFieldNum; - int32_t exprListLevelIndex; -} SQueryStmtInfo; - -typedef struct SColumnIndex { - int16_t tableIndex; - int16_t columnIndex; - int16_t type; // normal column/tag/ user input constant column -} SColumnIndex; - -struct SInsertStmtInfo; +#include "parsenodes.h" /** * True will be returned if the input sql string is insert, false otherwise. @@ -156,26 +55,6 @@ typedef struct SParseContext { */ int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); -typedef enum { - PAYLOAD_TYPE_KV = 0, - PAYLOAD_TYPE_RAW = 1, -} EPayloadType; - -typedef struct SVgDataBlocks { - int64_t vgId; // virtual group id - int32_t numOfTables; // number of tables in current submit block - uint32_t size; - char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... -} SVgDataBlocks; - -typedef struct SInsertStmtInfo { - SArray* pDataBlocks; // data block for each vgroup, SArray. - int8_t schemaAttache; // denote if submit block is built with table schema or not - uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* sql; // current sql statement position -} SInsertStmtInfo; - /** * Parse the insert sql statement. * @param pStr sql string @@ -211,9 +90,9 @@ typedef struct SSourceParam { SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); -int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo); +int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo); -STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); +STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex); SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex); SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 8b54b88b28..153cf0bb3e 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -25,6 +25,7 @@ extern "C" { #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 #define QUERY_TYPE_SCAN 3 +#define QUERY_TYPE_MODIFY 4 enum OPERATOR_TYPE_E { OP_Unknown, @@ -58,18 +59,17 @@ typedef struct SQueryNodeBasicInfo { typedef struct SDataSink { SQueryNodeBasicInfo info; - SDataBlockSchema schema; } SDataSink; typedef struct SDataDispatcher { SDataSink sink; - // todo } SDataDispatcher; typedef struct SDataInserter { SDataSink sink; - uint64_t uid; // unique id of the table - // todo data field + int32_t numOfTables; + uint32_t size; + char *pData; } SDataInserter; typedef struct SPhyNode { @@ -119,12 +119,13 @@ typedef struct SSubplanId { typedef struct SSubplan { SSubplanId id; // unique id of the subplan - int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN - int32_t level; // the execution level of current subplan, starting from 0. - SEpSet execEpSet; // for the scan sub plan, the optional execution node - SArray *pChildern; // the datasource subplan,from which to fetch the result - SArray *pParents; // the data destination subplan, get data from current subplan - SPhyNode *pNode; // physical plan of current subplan + int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN + int32_t level; // the execution level of current subplan, starting from 0. + SEpSet execEpSet; // for the scan sub plan, the optional execution node + SArray *pChildern; // the datasource subplan,from which to fetch the result + SArray *pParents; // the data destination subplan, get data from current subplan + SPhyNode *pNode; // physical plan of current subplan + SDataSink *pDataSink; // data of the subplan flow into the datasink } SSubplan; typedef struct SQueryDag { @@ -133,10 +134,12 @@ typedef struct SQueryDag { SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. } SQueryDag; +struct SQueryNode; + /** * Create the physical plan for the query, according to the AST. */ -int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); +int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule @@ -144,7 +147,7 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* // @ep one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep); -int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); +int32_t qExplainQuery(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, char** str); /** * Convert to subplan to string for the scheduler to send to the executor diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index fbd92b89f8..69dad4d9f4 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -78,7 +78,7 @@ typedef struct { typedef struct STableDataBlocks { int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not - int64_t vgId; // virtual group id + int32_t vgId; // virtual group id int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending int32_t numOfTables; // number of tables in current submit block int32_t rowSize; // row size for current table diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index d37ac74d21..2f3d5aab77 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -123,7 +123,6 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->nAllocSize = dataBuf->headerSize * 2; } - //dataBuf->pData = calloc(1, dataBuf->nAllocSize); dataBuf->pData = malloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { tfree(dataBuf); @@ -145,8 +144,6 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->tsSource = -1; dataBuf->vgId = dataBuf->pTableMeta->vgId; - // tNameAssign(&dataBuf->tableName, name); - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 2481896f09..9aba2e43ce 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -50,15 +50,16 @@ enum { }; typedef struct SInsertParseContext { - SParseContext* pComCxt; - const char* pSql; - SMsgBuf msg; - STableMeta* pTableMeta; - SParsedDataColInfo tags; - SKVRowBuilder tagsBuilder; - SHashObj* pTableBlockHashObj; - SArray* pTableDataBlocks; - SArray* pVgDataBlocks; + SParseContext* pComCxt; // input + const char* pSql; // input + SMsgBuf msg; // input + STableMeta* pTableMeta; // each table + SParsedDataColInfo tags; // each table + SKVRowBuilder tagsBuilder; // each table + SHashObj* pVgroupsHashObj; // global + SHashObj* pTableBlockHashObj; // global + SArray* pTableDataBlocks; // global + SArray* pVgDataBlocks; // global int32_t totalNum; SInsertStmtInfo* pOutput; } SInsertParseContext; @@ -173,10 +174,12 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { char tableName[TSDB_TABLE_NAME_LEN] = {0}; CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName)); CHECK_CODE(catalogGetTableMeta(pCxt->pComCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta)); + SVgroupInfo vg; + CHECK_CODE(catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); return TSDB_CODE_SUCCESS; } -// todo speedup by using hash list static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { while (start < end) { if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { @@ -187,16 +190,16 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS return -1; } -static void fillMsgHeader(SVgDataBlocks* dst) { - SMsgDesc* desc = (SMsgDesc*)dst->pData; +static void buildMsgHeader(SVgDataBlocks* blocks) { + SMsgDesc* desc = (SMsgDesc*)blocks->pData; desc->numOfVnodes = htonl(1); SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); - submit->header.vgId = htonl(dst->vgId); - submit->header.contLen = htonl(dst->size - sizeof(SMsgDesc)); + submit->header.vgId = htonl(blocks->vg.vgId); + submit->header.contLen = htonl(blocks->size - sizeof(SMsgDesc)); submit->length = submit->header.contLen; - submit->numOfBlocks = htonl(dst->numOfTables); + submit->numOfBlocks = htonl(blocks->numOfTables); SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); - int32_t numOfBlocks = dst->numOfTables; + int32_t numOfBlocks = blocks->numOfTables; while (numOfBlocks--) { int32_t dataLen = blk->dataLen; blk->uid = htobe64(blk->uid); @@ -222,11 +225,11 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - dst->vgId = src->vgId; + taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; dst->size = src->size; SWAP(dst->pData, src->pData, char*); - fillMsgHeader(dst); + buildMsgHeader(dst); taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); } return TSDB_CODE_SUCCESS; @@ -546,7 +549,7 @@ static FORCE_INLINE int32_t parseOneValue(SInsertParseContext* pCxt, SToken* pTo } case TSDB_DATA_TYPE_BINARY: { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z); } return func(pToken->z, pToken->n, param); @@ -644,9 +647,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, CHECK_CODE(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m)); } - destroyBoundColumnInfo(&pCxt->tags); SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); - tdDestroyKVRowBuilder(&pCxt->tagsBuilder); if (NULL == row) { return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); } @@ -799,13 +800,30 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da return TSDB_CODE_SUCCESS; } +static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { + tfree(pCxt->pTableMeta); + destroyBoundColumnInfo(&pCxt->tags); + tdDestroyKVRowBuilder(&pCxt->tagsBuilder); +} + +static void destroyInsertParseContext(SInsertParseContext* pCxt) { + destroyInsertParseContextForTable(pCxt); + taosHashCleanup(pCxt->pVgroupsHashObj); + taosHashCleanup(pCxt->pTableBlockHashObj); + destroyBlockArrayList(pCxt->pTableDataBlocks); + destroyBlockArrayList(pCxt->pVgDataBlocks); +} + // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] // [(field1_name, ...)] // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; static int32_t parseInsertBody(SInsertParseContext* pCxt) { + // for each table while (1) { + destroyInsertParseContextForTable(pCxt); + SToken sToken; // pSql -> tb_name ... NEXT_TOKEN(pCxt->pSql, sToken); @@ -867,15 +885,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildOutput(pCxt); } -static void destroyInsertParseContext(SInsertParseContext* pCxt) { - tfree(pCxt->pTableMeta); - destroyBoundColumnInfo(&pCxt->tags); - tdDestroyKVRowBuilder(&pCxt->tagsBuilder); - taosHashCleanup(pCxt->pTableBlockHashObj); - destroyBlockArrayList(pCxt->pTableDataBlocks); - destroyBlockArrayList(pCxt->pVgDataBlocks); -} - // INSERT INTO // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] @@ -888,12 +897,13 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { .pSql = pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, .pTableMeta = NULL, + .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .totalNum = 0, .pOutput = calloc(1, sizeof(SInsertStmtInfo)) }; - if (NULL == context.pTableBlockHashObj || NULL == context.pOutput) { + if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pOutput) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index b72bc06324..0239bf71dd 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1167,7 +1167,7 @@ void cleanupTagCond(STagCond* pTagCond) { * @param tableIndex denote the table index for join query, where more than one table exists * @return */ -STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex) { +STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex) { assert(pQueryInfo != NULL); if (pQueryInfo->pTableMetaInfo == NULL) { assert(pQueryInfo->numOfTables == 0); diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 502946675b..1ae0d9211a 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -354,7 +354,7 @@ bool tscHasColumnFilter(SQueryStmtInfo* pQueryInfo) { return false; } -int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo) { +int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo) { int32_t n = 10; int32_t level = 0; diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/insertParserTest.cpp index 433d189a45..29a2677d1b 100644 --- a/source/libs/parser/test/insertParserTest.cpp +++ b/source/libs/parser/test/insertParserTest.cpp @@ -69,7 +69,7 @@ protected: cout << "schemaAttache:" << (int32_t)res_->schemaAttache << ", payloadType:" << (int32_t)res_->payloadType << ", insertType:" << res_->insertType << ", numOfVgs:" << num << endl; for (size_t i = 0; i < num; ++i) { SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); - cout << "vgId:" << vg->vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; + cout << "vgId:" << vg->vg.vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; SMsgDesc* desc = (SMsgDesc*)(vg->pData); cout << "numOfVnodes:" << ntohl(desc->numOfVnodes) << endl; SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index fdcb6e0433..6d89733668 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -42,11 +42,15 @@ void generateTestST1(MockCatalogService* mcs) { } int32_t __catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { - return mockCatalogService->catalogGetHandle(clusterId, catalogHandle); + return 0; } int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { - return mockCatalogService->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); + return mockCatalogService->catalogGetTableMeta(pDBName, pTableName, pTableMeta); +} + +int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) { + return mockCatalogService->catalogGetTableHashVgroup(pDBName, pTableName, vgInfo); } void initMetaDataEnv() { @@ -55,6 +59,7 @@ void initMetaDataEnv() { static Stub stub; stub.set(catalogGetHandle, __catalogGetHandle); stub.set(catalogGetTableMeta, __catalogGetTableMeta); + stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup); { AddrAny any("libcatalog.so"); std::map result; @@ -71,6 +76,14 @@ void initMetaDataEnv() { stub.set(f.second, __catalogGetTableMeta); } } + { + AddrAny any("libcatalog.so"); + std::map result; + any.get_global_func_addr_dynsym("^catalogGetTableHashVgroup$", result); + for (const auto& f : result) { + stub.set(f.second, __catalogGetTableHashVgroup); + } + } } void generateMetaData() { diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index ad366e8f74..e234f82da9 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -90,11 +90,11 @@ public: MockCatalogServiceImpl() : id_(1) { } - int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { + int32_t catalogGetHandle() const { return 0; } - int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { + int32_t catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { std::unique_ptr table; int32_t code = copyTableSchemaMeta(pDBName, pTableName, &table); if (TSDB_CODE_SUCCESS != code) { @@ -104,6 +104,11 @@ public: return TSDB_CODE_SUCCESS; } + int32_t catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const { + // todo + return 0; + } + TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); meta_[db][tbname] = builder_->table(); @@ -270,10 +275,10 @@ std::shared_ptr MockCatalogService::getTableMeta(const std::strin return impl_->getTableMeta(db, tbname); } -int32_t MockCatalogService::catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { - return impl_->catalogGetHandle(clusterId, catalogHandle); +int32_t MockCatalogService::catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { + return impl_->catalogGetTableMeta(pDBName, pTableName, pTableMeta); } -int32_t MockCatalogService::catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { - return impl_->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); -} +int32_t MockCatalogService::catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const { + return impl_->catalogGetTableHashVgroup(pDBName, pTableName, vgInfo); +} \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index cd01db09cc..889a454397 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -57,9 +57,8 @@ public: void showTables() const; std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const; - // mock interface - int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const; - int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; + int32_t catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; + int32_t catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const; private: std::unique_ptr impl_; diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index a2078defda..f9b1722106 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -93,7 +93,7 @@ void generateLogicplan(const char* sql) { ASSERT_EQ(ret, 0); struct SQueryPlanNode* n = nullptr; - code = createQueryPlan(pQueryInfo, &n); + code = createQueryPlan((const SQueryNode*)pQueryInfo, &n); char* str = NULL; queryPlanToString(n, &str); @@ -156,7 +156,7 @@ TEST(testCase, planner_test) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); struct SQueryPlanNode* n = nullptr; - code = createQueryPlan(pQueryInfo, &n); + code = createQueryPlan((const SQueryNode*)pQueryInfo, &n); char* str = NULL; queryPlanToString(n, &str); diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index b1c4643f15..1bee95b8e5 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -40,6 +40,7 @@ extern "C" { #define QNODE_SESSIONWINDOW 12 #define QNODE_STATEWINDOW 13 #define QNODE_FILL 14 +#define QNODE_INSERT 15 typedef struct SQueryDistPlanNodeInfo { bool stableQuery; // super table query or not @@ -82,7 +83,7 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode); * @param pQueryNode * @return */ -int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode); +int32_t createQueryPlan(const SQueryNode* pNode, struct SQueryPlanNode** pQueryPlan); /** * Convert the query plan to string, in order to display it in the shell. diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index ce923314bd..136073aa60 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -29,7 +29,7 @@ typedef struct SJoinCond { SColumn *colCond[2]; } SJoinCond; -static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); +static SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo); static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len); @@ -37,16 +37,40 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { return 0; } -int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { - SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); +int32_t createInsertPlan(const SInsertStmtInfo* pInsert, SQueryPlanNode** pQueryPlan) { + *pQueryPlan = calloc(1, sizeof(SQueryPlanNode)); + SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES); + if (NULL == *pQueryPlan || NULL == blocks) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + (*pQueryPlan)->info.type = QNODE_INSERT; + taosArrayAddAll(blocks, pInsert->pDataBlocks); + (*pQueryPlan)->pExtInfo = blocks; + return TSDB_CODE_SUCCESS; +} + +int32_t createSelectPlan(const SQueryStmtInfo* pSelect, SQueryPlanNode** pQueryPlan) { + SArray* upstream = createQueryPlanImpl(pSelect); assert(taosArrayGetSize(upstream) == 1); - - *pQueryNode = taosArrayGetP(upstream, 0); - + *pQueryPlan = taosArrayGetP(upstream, 0); taosArrayDestroy(upstream); return TSDB_CODE_SUCCESS; } +int32_t createQueryPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { + switch (nodeType(pNode)) { + case TSDB_SQL_SELECT: { + return createSelectPlan((const SQueryStmtInfo*)pNode, pQueryPlan); + } + case TSDB_SQL_INSERT: + return createInsertPlan((const SInsertStmtInfo*)pNode, pQueryPlan); + default: + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { return 0; } @@ -62,7 +86,7 @@ void destroyQueryPlan(SQueryPlanNode* pQueryNode) { //====================================================================================================================== static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { + SExprInfo** pExpr, int32_t numOfOutput, const void* pExtInfo) { SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); pNode->info.type = type; @@ -123,7 +147,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla } case QNODE_FILL: { // todo !! - pNode->pExtInfo = pExtInfo; + pNode->pExtInfo = (void*)pExtInfo; break; } @@ -145,7 +169,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla return pNode; } -static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, +static SQueryPlanNode* doAddTableColumnNode(const SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, SArray* pExprs, SArray* tableCols) { if (pQueryInfo->info.onlyTagQuery) { int32_t num = (int32_t) taosArrayGetSize(pExprs); @@ -186,7 +210,7 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe return pNode; } -static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { +static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(const SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { // group by column not by tag size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); @@ -239,7 +263,7 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); SArray* p = pQueryInfo->exprList[0]; // top expression in select clause - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); + pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p->pData, taosArrayGetSize(p), pInfo); } if (pQueryInfo->order != NULL) { @@ -254,7 +278,7 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer return pNode; } -static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, +static SQueryPlanNode* doCreateQueryPlanForSingleTable(const SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, SArray* tableCols) { char name[TSDB_TABLE_FNAME_LEN] = {0}; tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); @@ -286,7 +310,7 @@ static bool isAllAggExpr(SArray* pList) { return true; } -SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { +SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { SArray* upstream = NULL; if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 0696a5b24e..9db9e059b3 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -14,6 +14,7 @@ */ #include "plannerInt.h" +#include "exception.h" #include "parser.h" #define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan @@ -33,6 +34,13 @@ static const char* gOpName[] = { #undef INCLUDE_AS_NAME }; +static void* vailidPointer(void* p) { + if (NULL == p) { + THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); + } + return p; +} + const char* opTypeToOpName(int32_t type) { return gOpName[type]; } @@ -46,6 +54,25 @@ int32_t opNameToOpType(const char* name) { return OP_Unknown; } +static SDataSink* initDataSink(int32_t type, int32_t size) { + SDataSink* sink = (SDataSink*)vailidPointer(calloc(1, size)); + sink->info.type = type; + return sink; +} + +static SDataSink* createDataDispatcher(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SDataDispatcher* dispatcher = (SDataDispatcher*)initDataSink(DSINK_Dispatch, sizeof(SDataDispatcher)); + return (SDataSink*)dispatcher; +} + +static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks) { + SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter)); + inserter->numOfTables = pBlocks->numOfTables; + inserter->size = pBlocks->size; + SWAP(inserter->pData, pBlocks->pData, char*); + return (SDataSink*)inserter; +} + static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { dataBlockSchema->numOfCols = pPlanNode->numOfCols; dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); @@ -72,10 +99,7 @@ static bool cloneExprArray(SArray** dst, SArray* src) { } static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { - SPhyNode* node = (SPhyNode*)calloc(1, size); - if (NULL == node) { - return NULL; - } + SPhyNode* node = (SPhyNode*)vailidPointer(calloc(1, size)); node->info.type = type; node->info.name = opTypeToOpName(type); if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { @@ -138,7 +162,7 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { - SSubplan* subplan = calloc(1, sizeof(SSubplan)); + SSubplan* subplan = vailidPointer(calloc(1, sizeof(SSubplan))); subplan->id = pCxt->nextId; ++(pCxt->nextId.subplanId); subplan->type = type; @@ -146,15 +170,15 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { if (NULL != pCxt->pCurrentSubplan) { subplan->level = pCxt->pCurrentSubplan->level + 1; if (NULL == pCxt->pCurrentSubplan->pChildern) { - pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + pCxt->pCurrentSubplan->pChildern = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); } taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan); - subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + subplan->pParents = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan); } SArray* currentLevel; if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) { - currentLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + currentLevel = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(pCxt->pDag->pSubplans, ¤tLevel); } else { currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level); @@ -164,7 +188,17 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { return subplan; } -static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { +static void vgroupInfoToEpSet(const SVgroupInfo* vg, SEpSet* epSet) { + epSet->inUse = 0; // todo + epSet->numOfEps = vg->numOfEps; + for (int8_t i = 0; i < vg->numOfEps; ++i) { + epSet->port[i] = vg->epAddr[i].port; + strcpy(epSet->fqdn[i], vg->epAddr[i].fqdn); + } + return; +} + +static void vgroupMsgToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { epSet->inUse = 0; // todo epSet->numOfEps = vg->numOfEps; for (int8_t i = 0; i < vg->numOfEps; ++i) { @@ -179,8 +213,9 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) { STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); - vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); + vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); + subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode); RECOVERY_CURRENT_SUBPLAN(pCxt); } return pCxt->nextId.templateId++; @@ -214,6 +249,9 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; + case QNODE_INSERT: + // Insert is not an operator in a physical plan. + break; default: assert(false); } @@ -229,26 +267,46 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { return node; } +static void splitInsertSubplan(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SArray* vgs = (SArray*)pPlanNode->pExtInfo; + size_t numOfVg = taosArrayGetSize(vgs); + for (int32_t i = 0; i < numOfVg; ++i) { + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY); + SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(vgs, i); + vgroupInfoToEpSet(&blocks->vg, &subplan->execEpSet); + subplan->pNode = NULL; + subplan->pDataSink = createDataInserter(pCxt, blocks); + } +} + static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { - SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); - ++(pCxt->nextId.templateId); - subplan->pNode = createPhyNode(pCxt, pRoot); + if (QNODE_INSERT == pRoot->info.type) { + splitInsertSubplan(pCxt, pRoot); + } else { + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); + ++(pCxt->nextId.templateId); + subplan->pNode = createPhyNode(pCxt, pRoot); + subplan->pDataSink = createDataDispatcher(pCxt, pRoot); + } // todo deal subquery } int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag) { - SPlanContext context = { - .pCatalog = pCatalog, - .pDag = calloc(1, sizeof(SQueryDag)), - .pCurrentSubplan = NULL, - .nextId = {0} // todo queryid - }; - if (NULL == context.pDag) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - context.pDag->pSubplans = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); - createSubplanByLevel(&context, pQueryNode); - *pDag = context.pDag; + TRY(TSDB_MAX_TAG_CONDITIONS) { + SPlanContext context = { + .pCatalog = pCatalog, + .pDag = vailidPointer(calloc(1, sizeof(SQueryDag))), + .pCurrentSubplan = NULL, + .nextId = {0} // todo queryid + }; + *pDag = context.pDag; + context.pDag->pSubplans = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + createSubplanByLevel(&context, pQueryNode); + } CATCH(code) { + CLEANUP_EXECUTE(); + terrno = code; + return TSDB_CODE_FAILED; + } END_TRY return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 2510797158..15c0e632a7 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -651,7 +651,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { static const char* jkPnodeName = "Name"; static const char* jkPnodeTargets = "Targets"; static const char* jkPnodeConditions = "Conditions"; -static const char* jkPnodeSchema = "Schema"; +static const char* jkPnodeSchema = "InputSchema"; static const char* jkPnodeChildren = "Children"; // The 'pParent' field do not need to be serialized. static bool phyNodeToJson(const void* obj, cJSON* jNode) { diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index e9a4591d4a..7722a7d363 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -24,9 +24,9 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { // todo } -int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag) { +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SEpSet* pEpSet, struct SQueryDag** pDag) { SQueryPlanNode* logicPlan; - int32_t code = createQueryPlan(pQueryInfo, &logicPlan); + int32_t code = createQueryPlan(pNode, &logicPlan); if (TSDB_CODE_SUCCESS != code) { destroyQueryPlan(logicPlan); return code; From 556788b6343f6972c6065fd111e4c6949a28b5a2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Dec 2021 23:12:11 +0800 Subject: [PATCH 04/14] update tindex reader --- source/libs/index/inc/index_tfile.h | 1 - source/libs/index/src/index_tfile.c | 82 +++++++++++++++++++---------- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index d711da1807..cf6341ce9c 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -47,7 +47,6 @@ typedef struct TFileHeader { typedef struct TFileCacheKey { uint64_t suid; uint8_t colType; - int32_t version; char* colName; int32_t nColName; } TFileCacheKey; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 6ac1a893a6..9acd9f6870 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -38,11 +38,14 @@ static int tfileWriteHeader(TFileWriter* writer); static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset); static int tfileWriteData(TFileWriter* write, TFileValue* tval); -static int tfileReadLoadHeader(TFileReader* reader); -static int tfileReadLoadFst(TFileReader* reader); -static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); +static int tfileReadLoadHeader(TFileReader* reader); +static int tfileReadLoadFst(TFileReader* reader); +static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); +static void tfileReadRef(TFileReader* reader); +static void tfileReadUnRef(TFileReader* reader); static int tfileGetFileList(const char* path, SArray* result); +static int tfileRmExpireFile(SArray* result); static void tfileDestroyFileName(void* elem); static int tfileCompare(const void* a, const void* b); static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version); @@ -58,6 +61,8 @@ TFileCache* tfileCacheCreate(const char* path) { SArray* files = taosArrayInit(4, sizeof(void*)); tfileGetFileList(path, files); taosArraySort(files, tfileCompare); + tfileRmExpireFile(files); + uint64_t suid; int32_t colId, version; for (size_t i = 0; i < taosArrayGetSize(files); i++) { @@ -66,29 +71,29 @@ TFileCache* tfileCacheCreate(const char* path) { indexInfo("try parse invalid file: %s, skip it", file); continue; } + WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64); if (wc == NULL) { indexError("failed to open index: %s", file); goto End; } + TFileReader* reader = tfileReaderCreate(wc); if (0 != tfileReadLoadHeader(reader)) { tfileReaderDestroy(reader); indexError("failed to load index header, index file: %s", file); goto End; } + if (0 != tfileReadLoadFst(reader)) { tfileReaderDestroy(reader); indexError("failed to load index fst, index file: %s", file); + goto End; } + tfileReadRef(reader); // loader fst and validate it - TFileHeader* header = &reader->header; - TFileCacheKey key = {.suid = header->suid, - .version = header->version, - .colName = header->colName, - .nColName = strlen(header->colName), - .colType = header->colType}; + TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; char buf[128] = {0}; tfileSerialCacheKey(&key, buf); @@ -110,7 +115,7 @@ void tfileCacheDestroy(TFileCache* tcache) { TFileReader* p = *reader; indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); - tfileReaderDestroy(p); + tfileReadUnRef(p); reader = taosHashIterate(tcache->tableCache, reader); } taosHashCleanup(tcache->tableCache); @@ -120,12 +125,17 @@ void tfileCacheDestroy(TFileCache* tcache) { TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { char buf[128] = {0}; tfileSerialCacheKey(key, buf); + TFileReader* reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); + tfileReadRef(reader); + return reader; } void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) { char buf[128] = {0}; tfileSerialCacheKey(key, buf); + tfileReadRef(reader); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); return; } @@ -147,25 +157,29 @@ void tfileReaderDestroy(TFileReader* reader) { } int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) { - SIndexTerm* term = query->term; + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; + int ret = -1; // refactor to callback later - if (query->qType == QUERY_TERM) { + if (qtype == QUERY_TERM) { uint64_t offset; FstSlice key = fstSliceCreate(term->colVal, term->nColVal); if (fstGet(reader->fst, &key, &offset)) { - return tfileReadLoadTableIds(reader, offset, result); + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName, term->colVal); + ret = tfileReadLoadTableIds(reader, offset, result); } else { - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found in tindex", term->suid, term->colName, term->colVal); + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, term->colVal); } - return 0; - } else if (query->qType == QUERY_PREFIX) { + fstSliceDestroy(&key); + } else if (qtype == QUERY_PREFIX) { // handle later // } else { // handle later } - return 0; + tfileReadUnRef(reader); + return ret; } TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { @@ -209,7 +223,6 @@ int tfileWriterPut(TFileWriter* tw, void* data) { int32_t tbsz = taosArrayGetSize(v->tableId); fstOffset += TF_TABLE_TATOAL_SIZE(tbsz); } - // check result or not tfileWriteFstOffset(tw, fstOffset); for (size_t i = 0; i < sz; i++) { @@ -237,6 +250,7 @@ int tfileWriterPut(TFileWriter* tw, void* data) { // write reversed data in buf to tindex tw->ctx->write(tw->ctx, buf, offset); } + tfree(buf); // write fst for (size_t i = 0; i < sz; i++) { @@ -244,11 +258,8 @@ int tfileWriterPut(TFileWriter* tw, void* data) { TFileValue* v = taosArrayGetP((SArray*)data, i); if (tfileWriteData(tw, v) == 0) { // - // } } - - tfree(buf); return 0; } void tfileWriterDestroy(TFileWriter* tw) { @@ -270,17 +281,21 @@ void IndexTFileDestroy(IndexTFile* tfile) { } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { - if (tfile == NULL) { return -1; } + int ret = -1; + if (tfile == NULL) { return ret; } IndexTFile* pTfile = (IndexTFile*)tfile; SIndexTerm* term = query->term; - TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName}; + TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; + TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - return tfileReaderSearch(reader, query, result); + ret = tfileReaderSearch(reader, query, result); + + return ret; } int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { - TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1}; + // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = + // 1}; return 0; } @@ -382,6 +397,15 @@ static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* re free(buf); return 0; } +static void tfileReadRef(TFileReader* reader) { + int ref = T_REF_INC(reader); + UNUSED(ref); +} + +static void tfileReadUnRef(TFileReader* reader) { + int ref = T_REF_DEC(reader); + if (ref == 0) { tfileReaderDestroy(reader); } +} static int tfileGetFileList(const char* path, SArray* result) { DIR* dir = opendir(path); @@ -397,6 +421,10 @@ static int tfileGetFileList(const char* path, SArray* result) { closedir(dir); return 0; } +static int tfileRmExpireFile(SArray* result) { + // TODO(yihao): remove expire tindex after restart + return 0; +} static void tfileDestroyFileName(void* elem) { char* p = *(char**)elem; free(p); @@ -423,7 +451,5 @@ static void tfileSerialCacheKey(TFileCacheKey* key, char* buf) { SERIALIZE_VAR_TO_BUF(buf, '_', char); SERIALIZE_MEM_TO_BUF(buf, key, colType); SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_MEM_TO_BUF(buf, key, version); - SERIALIZE_VAR_TO_BUF(buf, '_', char); SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); } From 28c358c33992ca1238f95a0a1a41bff2672dd2e7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Dec 2021 23:29:54 +0800 Subject: [PATCH 05/14] update tindex reader --- source/libs/index/inc/index_tfile.h | 1 + source/libs/index/src/index_tfile.c | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index cf6341ce9c..b19b887e9f 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -66,6 +66,7 @@ typedef struct TFileWriter { uint32_t offset; } TFileWriter; +// multi reader and single write typedef struct TFileReader { T_REF_DECLARE() Fst* fst; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 9acd9f6870..1cd5673faa 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -134,8 +134,15 @@ TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) { char buf[128] = {0}; tfileSerialCacheKey(key, buf); - tfileReadRef(reader); + // remove last version index reader + TFileReader** p = taosHashGet(tcache->tableCache, buf, strlen(buf)); + if (*p != NULL) { + TFileReader* oldReader = *p; + taosHashRemove(tcache->tableCache, buf, strlen(buf)); + tfileReadUnRef(oldReader); + } + tfileReadRef(reader); taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); return; } From c03e267619046c1e5c4ea7178c25cbfceb74a315 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Dec 2021 23:34:38 +0800 Subject: [PATCH 06/14] update tindex reader --- source/libs/index/src/index_tfile.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 1cd5673faa..4b38f12e1f 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -296,9 +296,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - ret = tfileReaderSearch(reader, query, result); - - return ret; + return tfileReaderSearch(reader, query, result); } int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = @@ -331,8 +329,7 @@ static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { return 0; } static int tfileWriteHeader(TFileWriter* writer) { - char buf[TFILE_HEADER_NO_FST] = {0}; - char* p = buf; + char buf[TFILE_HEADER_NO_FST] = {0}; TFileHeader* header = &writer->header; memcpy(buf, (char*)header, sizeof(buf)); @@ -360,8 +357,7 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) { } static int tfileReadLoadHeader(TFileReader* reader) { // TODO simple tfile header later - char buf[TFILE_HEADER_SIZE] = {0}; - char* p = buf; + char buf[TFILE_HEADER_SIZE] = {0}; int64_t nread = reader->ctx->read(reader->ctx, buf, sizeof(buf)); assert(nread == sizeof(buf)); From 6e7df831ce5e383c0226961bf854412ba57aba25 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Dec 2021 23:58:38 +0800 Subject: [PATCH 07/14] update tindex reader --- source/libs/index/src/index_tfile.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 4b38f12e1f..ecacdcd13c 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -386,16 +386,19 @@ static int tfileReadLoadFst(TFileReader* reader) { static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { int32_t nid; WriterCtx* ctx = reader->ctx; - int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset); + + int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset); assert(sizeof(nid) == nread); - char* buf = calloc(1, sizeof(uint64_t) * nid); + int32_t total = sizeof(uint64_t) * nid; + char* buf = calloc(1, total); if (buf == NULL) { return -1; } - nread = ctx->read(ctx, buf, sizeof(uint64_t) * nid); - uint64_t* ids = (uint64_t*)buf; + nread = ctx->read(ctx, buf, total); + assert(total == nread); + for (int32_t i = 0; i < nid; i++) { - taosArrayPush(result, ids + i); + taosArrayPush(result, (uint64_t*)buf + i); } free(buf); return 0; From 42903d3d1f7473d9b0f50fa1db26cc08587e7dec Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Dec 2021 10:33:11 +0800 Subject: [PATCH 08/14] [td-11818] support use db/create stable. --- include/common/taosmsg.h | 50 +++++-- include/common/tmsgtype.h | 8 - include/common/tname.h | 2 - include/libs/parser/parser.h | 11 +- source/client/inc/clientInt.h | 7 +- source/client/src/clientImpl.c | 27 +++- source/client/src/clientMain.c | 6 + source/client/src/clientMsgHandler.c | 29 +++- source/client/test/clientTests.cpp | 26 +++- source/common/src/tname.c | 35 +---- source/libs/parser/inc/astToMsg.h | 1 + source/libs/parser/inc/parserInt.h | 10 +- source/libs/parser/inc/parserUtil.h | 1 + source/libs/parser/src/astGenerator.c | 5 +- source/libs/parser/src/astToMsg.c | 124 ++++++++++++++++ source/libs/parser/src/astValidate.c | 187 +++++++++++++++++++++++- source/libs/parser/src/parser.c | 8 +- source/libs/parser/src/parserUtil.c | 6 +- source/libs/parser/src/sql.c | 1 + source/libs/parser/test/parserTests.cpp | 9 +- 20 files changed, 468 insertions(+), 85 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 8229cae281..8be43bd8e3 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -95,15 +95,15 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) // message from mnode to vnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-internal" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-internal" ) // message from mnode to mnode // message from mnode to qnode // message from mnode to dnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_VNODE_IN, "create-vnode-internal" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_VNODE_IN, "alter-vnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_VNODE_IN, "drop-vnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH_VNODE_IN, "auth-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_VNODE_IN, "drop-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH_VNODE_IN, "auth-vnode-internal" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_VNODE_IN, "sync-vnode-internal" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE_IN, "compact-vnode-internal" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_MNODE_IN, "create-mnode-internal" ) @@ -290,6 +290,37 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; +typedef struct { + int32_t contLen; + int32_t vgId; + int8_t tableType; + int16_t numOfColumns; + int16_t numOfTags; + int32_t tid; + int32_t sversion; + int32_t tversion; + int32_t tagDataLen; + int32_t sqlDataLen; + uint64_t uid; + uint64_t superTableUid; + uint64_t createdTime; + char tableFname[TSDB_TABLE_FNAME_LEN]; + char stbFname[TSDB_TABLE_FNAME_LEN]; + char data[]; +} SMDCreateTableMsg; + +typedef struct { + int32_t len; // one create table message + char tableName[TSDB_TABLE_FNAME_LEN]; + int16_t numOfTags; + int16_t numOfColumns; + int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string + int8_t igExists; + int8_t rspMeta; + int8_t reserved[16]; + char schema[]; +} SCreateTableMsg; + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -327,19 +358,6 @@ typedef struct { uint64_t suid; } SDropStbInternalMsg; -typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - char stbFname[TSDB_TABLE_FNAME_LEN]; - int8_t tableType; - uint64_t suid; - int32_t sversion; - int32_t numOfTags; - int32_t numOfColumns; - int32_t tagDataLen; - char data[]; -} SCreateTableMsg; - typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index 0976503115..8e7ad87a0a 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -102,14 +102,6 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MAX, "max" ) }; -// create table operation type -enum TSQL_CREATE_TABLE_TYPE { - TSQL_CREATE_TABLE = 0x1, - TSQL_CREATE_STABLE = 0x2, - TSQL_CREATE_CTABLE = 0x3, - TSQL_CREATE_STREAM = 0x4, -}; - #ifdef __cplusplus } #endif diff --git a/include/common/tname.h b/include/common/tname.h index e31bfd38a6..de9e309b55 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,8 +16,6 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H -//#include "taosmsg.h" - #define TSDB_DB_NAME_T 1 #define TSDB_TABLE_NAME_T 2 diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 8e26fa98c9..a92398602b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -29,7 +29,7 @@ extern "C" { typedef struct SField { char name[TSDB_COL_NAME_LEN]; uint8_t type; - int16_t bytes; + int32_t bytes; } SField; typedef struct SFieldInfo { @@ -38,6 +38,13 @@ typedef struct SFieldInfo { SArray *internalField; // SArray } SFieldInfo; +// TODO merged with SParseContext +typedef struct SParseCtx { + char *db; + int32_t acctId; + uint64_t requestId; +} SParseCtx; + typedef struct SCond { uint64_t uid; int32_t len; // length of tag query condition data @@ -153,7 +160,7 @@ typedef struct SParseContext { * @param msg extended error message if exists. * @return error code */ -int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); +int32_t qParseQuerySql(const char* pStr, size_t length, SParseCtx* pParseCtx, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); typedef enum { PAYLOAD_TYPE_KV = 0, diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c1fd113da7..83ff0d29fb 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -148,8 +148,11 @@ int taos_init(); void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo); void destroyTscObj(void*pObj); -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); -void destroyRequest(SRequestObj* pRequest); +void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void destroyRequest(SRequestObj* pRequest); + +char *getConnectionDB(STscObj* pObj); +void setConnectionDB(STscObj* pTscObj, const char* db); void taos_init_imp(void); int taos_options_imp(TSDB_OPTION option, const char *str); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0acb74d4a9..0deaf5ad8e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -151,8 +151,12 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { int32_t type = 0; void* output = NULL; int32_t outputLen = 0; - code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); - if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_CREATE_ACCT) { + + SParseCtx c = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj)}; + code = qParseQuerySql(pRequest->sqlstr, sqlLen, &c, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); + if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || + type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_CREATE_ACCT || + type == TSDB_SQL_CREATE_TABLE || type == TSDB_SQL_USE_DB) { pRequest->type = type; pRequest->body.requestMsg = (SReqMsgInfo){.pMsg = output, .len = outputLen}; @@ -169,6 +173,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { } else { assert(0); } + + tfree(c.db); } if (code != TSDB_CODE_SUCCESS) { @@ -436,8 +442,19 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t } } -const char *taos_get_client_info() { return version; } +char* getConnectionDB(STscObj* pObj) { + char *p = NULL; + pthread_mutex_lock(&pObj->mutex); + p = strndup(pObj->db, tListLen(pObj->db)); + pthread_mutex_unlock(&pObj->mutex); -int taos_affected_rows(TAOS_RES *res) { return 1; } + return p; +} + +void setConnectionDB(STscObj* pTscObj, const char* db) { + assert(db != NULL && pTscObj != NULL); + pthread_mutex_lock(&pTscObj->mutex); + tstrncpy(pTscObj->db, db, tListLen(pTscObj->db)); + pthread_mutex_unlock(&pTscObj->mutex); +} -int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index dce04a869b..f50765d37a 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -255,3 +255,9 @@ const char *taos_data_type(int type) { default: return "UNKNOWN"; } } + +const char *taos_get_client_info() { return version; } + +int taos_affected_rows(TAOS_RES *res) { return 1; } + +int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e70dcc63b3..074841c175 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -2891,7 +2891,14 @@ int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { } tNameGetFullDbName(&name, pCreateMsg->db); - + break; + } + case TSDB_SQL_USE_DB: { + pMsgBody->msgType = TSDB_MSG_TYPE_USE_DB; + break; + } + case TSDB_SQL_CREATE_TABLE: { + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_STB; break; } case TSDB_SQL_SHOW: @@ -2973,6 +2980,20 @@ int32_t processCreateDbRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgL // todo rsp with the vnode id list } +int32_t processUseDbRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + SUseDbRsp* pUseDbRsp = (SUseDbRsp*) pMsg; + SName name = {0}; + tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT|T_NAME_DB); + + char db[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(&name, db); + setConnectionDB(pRequest->pTscObj, db); +} + +int32_t processCreateTableRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + assert(pMsg != NULL); +} + void initMsgHandleFp() { #if 0 tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; @@ -3066,4 +3087,10 @@ void initMsgHandleFp() { buildRequestMsgFp[TSDB_SQL_CREATE_DB] = doBuildMsgSupp; handleRequestRspFp[TSDB_SQL_CREATE_DB] = processCreateDbRsp; + + buildRequestMsgFp[TSDB_SQL_USE_DB] = doBuildMsgSupp; + handleRequestRspFp[TSDB_SQL_USE_DB] = processUseDbRsp; + + buildRequestMsgFp[TSDB_SQL_CREATE_TABLE] = doBuildMsgSupp; + handleRequestRspFp[TSDB_SQL_CREATE_TABLE] = processCreateTableRsp; } \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 2e89bb45fa..0ef305d3b6 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -16,15 +16,15 @@ #include #include #include -#include "tglobal.h" #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "../inc/clientInt.h" #include "taos.h" +#include "tglobal.h" +#include "../inc/clientInt.h" namespace { } // namespace @@ -149,11 +149,29 @@ TEST(testCase, create_db_Test) { taos_close(pConn); } -TEST(testCase, create_stable_Test) { +TEST(testCase, use_db_test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} + +TEST(testCase, create_stable_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); TAOS_FIELD* pFields = taos_fetch_fields(pRes); ASSERT_TRUE(pFields == NULL); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index fa303fe4e9..9661016483 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -6,21 +6,6 @@ #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) -char* extractDBName(const char* tableId, char* name) { - size_t offset1 = strcspn(tableId, &TS_PATH_DELIMITER[0]); - size_t len = strcspn(&tableId[offset1 + 1], &TS_PATH_DELIMITER[0]); - - return strncpy(name, &tableId[offset1 + 1], len); -} - -// todo remove it -size_t tableIdPrefix(const char* name, char* prefix, int32_t len) { - tstrncpy(prefix, name, len); - strcat(prefix, TS_PATH_DELIMITER); - - return strlen(prefix); -} - bool tscValidateTableNameLength(size_t len) { return len < TSDB_TABLE_NAME_LEN; } @@ -125,7 +110,7 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { return -1; } - int32_t len = snprintf(dst, TSDB_FULL_DB_NAME_LEN, "%s.%s", name->acctId, name->dbname); + int32_t len = snprintf(dst, TSDB_FULL_DB_NAME_LEN, "%d.%s", name->acctId, name->dbname); size_t tnameLen = strlen(name->tname); if (tnameLen > 0) { @@ -141,7 +126,9 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { int32_t tNameLen(const SName* name) { assert(name != NULL); - int32_t len = (int32_t) strlen(name->acctId); + + char tmp[12] = {0}; + int32_t len = sprintf(tmp, "%d", name->acctId); int32_t len1 = (int32_t) strlen(name->dbname); int32_t len2 = (int32_t) strlen(name->tname); @@ -161,10 +148,6 @@ bool tNameIsValid(const SName* name) { return false; } - if (strlen(name->acctId) <= 0) { - return false; - } - if (name->type == TSDB_DB_NAME_T) { return strlen(name->dbname) > 0; } else { @@ -237,13 +220,6 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return -1; } - int32_t len = (int32_t)(p - str); - - // too long account id or too long db name -// if ((len >= tListLen(dst->acctId)) || (len <= 0)) { -// return -1; -// } -// memcpy (dst->acctId, str, len); dst->acctId = strtoll(str, NULL, 10); } @@ -272,9 +248,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { dst->type = TSDB_TABLE_NAME_T; char* start = (char*) ((p == NULL)? str: (p+1)); - int32_t len = (int32_t) strlen(start); - // too long account id or too long db name + int32_t len = (int32_t) strlen(start); if ((len >= tListLen(dst->tname)) || (len <= 0)) { return -1; } diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 1771bdc0ed..63328cb263 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -9,5 +9,6 @@ SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen); SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen); +SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseCtx* pParseCtx, SMsgBuf* pMsgBuf); #endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 4f506e631f..bf9e53e8d4 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -38,6 +38,14 @@ typedef struct SMsgBuf { char *buf; } SMsgBuf; +// create table operation type +enum TSQL_CREATE_TABLE_TYPE { + TSQL_CREATE_TABLE = 0x1, + TSQL_CREATE_STABLE = 0x2, + TSQL_CREATE_CTABLE = 0x3, + TSQL_CREATE_STREAM = 0x4, +}; + void clearTableMetaInfo(STableMetaInfo* pTableMetaInfo); void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t id); @@ -60,7 +68,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQ * @param type * @return */ -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen); +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen); /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index ad43b9876a..3d8729e72d 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -68,6 +68,7 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta); bool isDclSqlStatement(SSqlInfo* pSqlInfo); bool isDdlSqlStatement(SSqlInfo* pSqlInfo); +bool isDqlSqlStatement(SSqlInfo* pSqlInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index dda30c56fc..b6a6b73ccc 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -13,9 +13,10 @@ * along with this program. If not, see . */ -#include "taos.h" -#include "os.h" #include "astGenerator.h" +#include +#include "os.h" +#include "taos.h" #include "tmsgtype.h" SArray *tListItemAppend(SArray *pList, SVariant *pVar, uint8_t sortOrder) { diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index c41ba97e39..780ff8b0e8 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -1,3 +1,4 @@ +#include #include "parserInt.h" #include "parserUtil.h" @@ -219,3 +220,126 @@ SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32 return pCreateMsg; } + +int32_t createSName(SName* pName, SToken* pTableName, SParseCtx* pParseCtx, SMsgBuf* pMsgBuf) { + const char* msg1 = "name too long"; + const char* msg2 = "acctId too long"; + + int32_t code = TSDB_CODE_SUCCESS; + char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, false); + + if (p != NULL) { // db has been specified in sql string so we ignore current db path + code = tNameSetAcctId(pName, pParseCtx->acctId); + if (code != 0) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + + code = tNameFromString(pName, name, T_NAME_DB|T_NAME_TABLE); + if (code != 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + } else { // get current DB name first, and then set it into path + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db)); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + + code = tNameFromString(pName, name, T_NAME_TABLE); + if (code != 0) { + code = buildInvalidOperationMsg(pMsgBuf, msg1); + } + } + + return code; +} + +SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseCtx* pParseCtx, SMsgBuf* pMsgBuf) { + SSchema* pSchema; + + int32_t numOfCols = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pColumns); + int32_t numOfTags = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pTagColumns); + + SCreateStbMsg* pCreateTableMsg = (SCreateStbMsg*)calloc(1, sizeof(SCreateStbMsg) + (numOfCols + numOfTags) * sizeof(SSchema)); + + char* pMsg = NULL; + int8_t type = pCreateTableSql->type; + if (type == TSQL_CREATE_TABLE) { // create by using super table, tags value +#if 0 + SArray* list = pInfo->pCreateTableInfo->childTableInfo; + + int32_t numOfTables = (int32_t)taosArrayGetSize(list); + pCreateTableMsg->numOfTables = htonl(numOfTables); + + pMsg = (char*)pCreateMsg; + for (int32_t i = 0; i < numOfTables; ++i) { + SCreateTableMsg* pCreate = (SCreateTableMsg*)pMsg; + + pCreate->numOfColumns = htons(pCmd->numOfCols); + pCreate->numOfTags = htons(pCmd->count); + pMsg += sizeof(SCreateTableMsg); + + SCreatedTableInfo* p = taosArrayGet(list, i); + strcpy(pCreate->tableName, p->fullname); + pCreate->igExists = (p->igExist) ? 1 : 0; + + // use dbinfo from table id without modifying current db info + pMsg = serializeTagData(&p->tagdata, pMsg); + + int32_t len = (int32_t)(pMsg - (char*)pCreate); + pCreate->len = htonl(len); + } +#endif + } else { // create (super) table + SName n = {0}; + int32_t code = createSName(&n, &pCreateTableSql->name, pParseCtx, pMsgBuf); + if (code != 0) { + return NULL; + } + + code = tNameExtractFullName(&n, pCreateTableMsg->name); + if (code != 0) { + buildInvalidOperationMsg(pMsgBuf, "invalid table name or database not specified"); + return NULL; + } + + pCreateTableMsg->igExists = pCreateTableSql->existCheck ? 1 : 0; + + pCreateTableMsg->numOfColumns = htonl(numOfCols); + pCreateTableMsg->numOfTags = htonl(numOfTags); + + pSchema = (SSchema*) pCreateTableMsg->pSchema; + for (int i = 0; i < numOfCols; ++i) { + TAOS_FIELD* pField = taosArrayGet(pCreateTableSql->colInfo.pColumns, i); + + pSchema->type = pField->type; + pSchema->bytes = htonl(pField->bytes); + strcpy(pSchema->name, pField->name); + + pSchema++; + } + + for(int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* pField = taosArrayGet(pCreateTableSql->colInfo.pTagColumns, i); + + pSchema->type = pField->type; + pSchema->bytes = htonl(pField->bytes); + strcpy(pSchema->name, pField->name); + + pSchema++; + } + + pMsg = (char*)pSchema; + } + + int32_t msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); + *len = msgLen; + + return pCreateTableMsg; +} diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index accc786bf6..3b9a33af02 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4171,7 +4171,144 @@ static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) { return TSDB_CODE_SUCCESS; } -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { +/* is contained in pFieldList or not */ +static bool has(SArray* pFieldList, int32_t startIndex, const char* name) { + size_t numOfCols = taosArrayGetSize(pFieldList); + for (int32_t j = startIndex; j < numOfCols; ++j) { + TAOS_FIELD* field = taosArrayGet(pFieldList, j); + if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true; + } + + return false; +} + +static int32_t validateTableColumns(SArray* pFieldList, int32_t maxRowLength, int32_t maxColumns, SMsgBuf* pMsgBuf) { + const char* msg2 = "row length exceeds max length"; + const char* msg3 = "duplicated column names"; + const char* msg4 = "invalid data type"; + const char* msg5 = "invalid binary/nchar column length"; + const char* msg6 = "invalid column name"; + const char* msg7 = "too many columns"; + const char* msg8 = "illegal number of columns"; + + size_t numOfCols = taosArrayGetSize(pFieldList); + if (numOfCols > maxColumns) { + return buildInvalidOperationMsg(pMsgBuf, msg7); + } + + int32_t rowLen = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + TAOS_FIELD* pField = taosArrayGet(pFieldList, i); + if (!isValidDataType(pField->type)) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + + if (pField->bytes == 0) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + if ((pField->type == TSDB_DATA_TYPE_BINARY && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_BINARY_LEN)) || + (pField->type == TSDB_DATA_TYPE_NCHAR && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_NCHAR_LEN))) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + SToken nameToken = {.z = pField->name, .n = strlen(pField->name), .type = TK_ID}; + if (parserValidateNameToken(&nameToken) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg6); + } + + // field name must be unique + if (has(pFieldList, i + 1, pField->name) == true) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + rowLen += pField->bytes; + } + + // max row length must be less than TSDB_MAX_BYTES_PER_ROW + if (rowLen > maxRowLength) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t validateTableColumnInfo(SArray* pFieldList, SMsgBuf* pMsgBuf) { + assert(pFieldList != NULL); + + const char* msg1 = "first column must be timestamp"; + const char* msg2 = "row length exceeds max length"; + const char* msg3 = "duplicated column names"; + const char* msg4 = "invalid data type"; + const char* msg5 = "invalid binary/nchar column length"; + const char* msg6 = "invalid column name"; + const char* msg7 = "too many columns"; + const char* msg8 = "illegal number of columns"; + + // first column must be timestamp + TAOS_FIELD* pField = taosArrayGet(pFieldList, 0); + if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + // number of fields no less than 2 + size_t numOfCols = taosArrayGetSize(pFieldList); + if (numOfCols <= 1) { + return buildInvalidOperationMsg(pMsgBuf, msg8); + } + + return validateTableColumns(pFieldList, TSDB_MAX_BYTES_PER_ROW, TSDB_MAX_COLUMNS, pMsgBuf); +} + +static int32_t validateTagParams(SArray* pTagsList, SArray* pFieldList, SMsgBuf* pMsgBuf) { + assert(pTagsList != NULL); + + const char* msg1 = "invalid number of tag columns"; + const char* msg3 = "duplicated column names"; + + // number of fields at least 1 + size_t numOfTags = taosArrayGetSize(pTagsList); + if (numOfTags < 1) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + // field name must be unique + for (int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* p = taosArrayGet(pTagsList, i); + if (has(pFieldList, 0, p->name) == true) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + } + + return validateTableColumns(pFieldList, TSDB_MAX_TAGS_LEN, TSDB_MAX_TAGS, pMsgBuf); +} + +int32_t doCheckForCreateTable(SSqlInfo* pInfo, SMsgBuf* pMsgBuf) { + const char* msg1 = "invalid table name"; + + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; + + SArray* pFieldList = pCreateTable->colInfo.pColumns; + SArray* pTagList = pCreateTable->colInfo.pTagColumns; + assert(pFieldList != NULL); + + // if sql specifies db, use it, otherwise use default db + SToken* pzTableName = &(pCreateTable->name); + + bool dbIncluded = false; + if (parserValidateNameToken(pzTableName) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + if (validateTableColumnInfo(pFieldList, pMsgBuf) != TSDB_CODE_SUCCESS || + (pTagList != NULL && validateTagParams(pTagList, pFieldList, pMsgBuf) != TSDB_CODE_SUCCESS)) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; @@ -4224,7 +4361,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in } } - *output = buildUserManipulationMsg(pInfo, outputLen, id, msgBuf, msgBufLen); + *output = buildUserManipulationMsg(pInfo, outputLen, pCtx->requestId, msgBuf, msgBufLen); break; } @@ -4260,13 +4397,13 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in } } - *output = buildAcctManipulationMsg(pInfo, outputLen, id, msgBuf, msgBufLen); + *output = buildAcctManipulationMsg(pInfo, outputLen, pCtx->requestId, msgBuf, msgBufLen); break; } case TSDB_SQL_DROP_ACCT: case TSDB_SQL_DROP_USER: { - *output = buildDropUserMsg(pInfo, outputLen, id, msgBuf, msgBufLen); + *output = buildDropUserMsg(pInfo, outputLen, pCtx->requestId, msgBuf, msgBufLen); break; } @@ -4275,6 +4412,28 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in break; } + case TSDB_SQL_USE_DB: { + const char* msg = "invalid db name"; + + SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); + if (parserValidateNameToken(pToken) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + SName n = {0}; + int32_t ret = tNameSetDbName(&n, pCtx->acctId, pToken->z, pToken->n); + if (ret != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + SUseDbMsg *pUseDbMsg = (SUseDbMsg *) calloc(1, sizeof(SUseDbMsg)); + tNameExtractFullName(&n, pUseDbMsg->db); + + *output = pUseDbMsg; + *outputLen = sizeof(SUseDbMsg); + break; + } + case TSDB_SQL_ALTER_DB: case TSDB_SQL_CREATE_DB: { const char* msg1 = "invalid db name"; @@ -4304,6 +4463,26 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in break; } + case TSDB_SQL_CREATE_TABLE: { + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; + + if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) { + if ((code = doCheckForCreateTable(pInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) { + return code; + } + *output = buildCreateTableMsg(pCreateTable, outputLen, pCtx, pMsgBuf); + } else if (pCreateTable->type == TSQL_CREATE_CTABLE) { + // if ((code = doCheckForCreateFromStable(pSql, pInfo)) != TSDB_CODE_SUCCESS) { + // return code; + // } + + } else if (pCreateTable->type == TSQL_CREATE_STREAM) { + // if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) { + // return code; + } + + break; + } default: break; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index e0ac7c295b..1e61be15dd 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -31,7 +31,7 @@ bool qIsInsertSql(const char* pStr, size_t length) { } while (1); } -int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { +int32_t qParseQuerySql(const char* pStr, size_t length, SParseCtx* pParseCtx, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { SSqlInfo info = doGenerateAST(pStr); if (!info.valid) { strncpy(msg, info.msg, msgLen); @@ -39,8 +39,8 @@ int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *typ return terrno; } - if (isDclSqlStatement(&info)) { - int32_t code = qParserValidateDclSqlNode(&info, id, pOutput, outputLen, type, msg, msgLen); + if (!isDqlSqlStatement(&info)) { + int32_t code = qParserValidateDclSqlNode(&info, pParseCtx, pOutput, outputLen, type, msg, msgLen); if (code == TSDB_CODE_SUCCESS) { // do nothing } @@ -53,7 +53,7 @@ int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *typ struct SCatalog* pCatalog = NULL; int32_t code = catalogGetHandle(NULL, &pCatalog); - code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen); + code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, pParseCtx->requestId, msg, msgLen); if (code == TSDB_CODE_SUCCESS) { *pOutput = pQueryInfo; } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 13434da057..3abdaa1036 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1620,7 +1620,11 @@ bool isDclSqlStatement(SSqlInfo* pSqlInfo) { bool isDdlSqlStatement(SSqlInfo* pSqlInfo) { int32_t type = pSqlInfo->type; - return (type == TSDB_SQL_CREATE_TABLE || type == TSDB_SQL_CREATE_DB); + return (type == TSDB_SQL_CREATE_TABLE || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_DROP_DB); +} + +bool isDqlSqlStatement(SSqlInfo* pSqlInfo) { + return pSqlInfo->type == TSDB_SQL_SELECT; } #if 0 diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index d674462fc0..9922ff1be0 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -31,6 +31,7 @@ #include #include #include "astGenerator.h" +#include "parserInt.h" #include "tmsgtype.h" #include "ttoken.h" #include "ttokendef.h" diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 396e8a12fe..7e289d5158 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -717,7 +717,9 @@ TEST(testCase, show_user_Test) { void* output = NULL; int32_t type = 0; int32_t len = 0; - int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + + SParseCtx ct= {.db= "abc", .acctId = 1, .requestId = 1}; + int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, &len, &type, msg, buf.len); ASSERT_EQ(code, 0); // convert the show command to be the select query @@ -734,13 +736,14 @@ TEST(testCase, create_user_Test) { SSqlInfo info1 = doGenerateAST(sql); ASSERT_EQ(info1.valid, true); - ASSERT_EQ(isDclSqlStatement(&info1), true); void* output = NULL; int32_t type = 0; int32_t len = 0; - int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + + SParseCtx ct= {.db= "abc", .acctId = 1, .requestId = 1}; + int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, &len, &type, msg, buf.len); ASSERT_EQ(code, 0); destroySqlInfo(&info1); From 148991ebb968f23922805ebe6c3ca5ecb5c70221 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Dec 2021 10:40:53 +0800 Subject: [PATCH 09/14] [td-11818] remove unused code. --- source/client/src/clientMsgHandler.c | 2768 -------------------------- 1 file changed, 2768 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 074841c175..00acefb397 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -14,7 +14,6 @@ */ #include "os.h" -#include "catalog.h" #include "tname.h" #include "clientInt.h" #include "clientLog.h" @@ -24,2773 +23,6 @@ int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); -#if 0 -void tscProcessActivityTimer(void *handle, void *tmrId); - -static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); - -static int32_t vgIdCompare(const void *lhs, const void *rhs) { - int32_t left = *(int32_t *)lhs; - int32_t right = *(int32_t *)rhs; - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} -static int32_t removeDupVgid(int32_t *src, int32_t sz) { - if (src == NULL || sz <= 0) { - return 0; - } - qsort(src, sz, sizeof(src[0]), vgIdCompare); - - int32_t ret = 1; - for (int i = 1; i < sz; i++) { - if (src[i] != src[i - 1]) { - src[ret++] = src[i]; - } - } - return ret; -} - -static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupMsg* pVgroupInfo) { - assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); - - // Issue the query to one of the vnode among a vgroup randomly. - // change the inUse property would not affect the isUse attribute of STableMeta - pEpSet->inUse = rand() % pVgroupInfo->numOfEps; - - // apply the FQDN string length check here - bool existed = false; - - pEpSet->numOfEps = pVgroupInfo->numOfEps; - for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - pEpSet->port[i] = pVgroupInfo->epAddr[i].port; - - int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); - if (len > 0) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); - existed = true; - } - } - - assert(existed); -} - -static void tscDumpMgmtEpSet(SSqlObj *pSql) { - SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginRead(&pCorEpSet->version); - pSql->epSet = pCorEpSet->epSet; - taosCorEndRead(&pCorEpSet->version); -} -static void tscEpSetHtons(SRpcEpSet *s) { - for (int32_t i = 0; i < s->numOfEps; i++) { - s->port[i] = htons(s->port[i]); - } -} - -bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { - if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { - return false; - } - - for (int32_t i = 0; i < s1->numOfEps; i++) { - if (s1->port[i] != s2->port[i] - || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) - return false; - } - return true; -} - -void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { - // no need to update if equal - SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginWrite(&pCorEpSet->version); - pCorEpSet->epSet = *pEpSet; - taosCorEndWrite(&pCorEpSet->version); -} - -static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { - if (pVgroupInfo == NULL) { return;} - int8_t inUse = pVgroupInfo->inUse; - pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; - pEpSet->numOfEps = pVgroupInfo->numOfEps; - for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); - pEpSet->port[i] = pVgroupInfo->ep[i].port; - } -} - -static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { - SSqlCmd *pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { - return; - } - - int32_t vgId = -1; - if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { - vgId = extractSTableQueryVgroupId(pTableMetaInfo); - } else { - vgId = pTableMetaInfo->pTableMeta->vgId; - } - - assert(vgId > 0); - - SNewVgroupInfo vgroupInfo = {.vgId = -1}; - taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); - assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); - - tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); - vgroupInfo.inUse = pEpSet->inUse; - vgroupInfo.numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { - tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); - vgroupInfo.ep[i].port = pEpSet->port[i]; - } - - tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); - taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); - - // Update the local cached epSet info cached by SqlObj - int32_t inUse = pSql->epSet.inUse; - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - tscDebug("0x%"PRIx64" update the epSet in SqlObj, in use before:%d, after:%d", pSql->self, inUse, pSql->epSet.inUse); - -} - -int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { - assert(pTableMetaInfo != NULL); - - int32_t vgIndex = pTableMetaInfo->vgroupIndex; - int32_t vgId = -1; - - if (pTableMetaInfo->pVgroupTables == NULL) { - SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - vgId = pVgroupInfo->vgroups[vgIndex].vgId; - } else { - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(vgIndex >= 0 && vgIndex < numOfVgroups); - - SVgroupTableInfo *pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - vgId = pTableIdList->vgInfo.vgId; - } - - return vgId; -} - -void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { - STscObj *pObj = (STscObj *)param; - if (pObj == NULL) return; - - if (pObj != pObj->signature) { - tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); - return; - } - - SSqlObj *pSql = tres; - SSqlRes *pRes = &pSql->res; - - if (code == TSDB_CODE_SUCCESS) { - SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; - SRpcEpSet *epSet = &pRsp->epSet; - if (epSet->numOfEps > 0) { - tscEpSetHtons(epSet); - - //SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { - // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); - // for (int8_t i = 0; i < epSet->numOfEps; i++) { - // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); - // } - //} - //concurrency problem, update mgmt epset anyway - tscUpdateMgmtEpSet(pSql, epSet); - } - - pSql->pTscObj->connId = htonl(pRsp->connId); - - if (pRsp->killConnection) { - tscKillConnection(pObj); - return; - } else { - if (pRsp->queryId) { - tscKillQuery(pObj, htonl(pRsp->queryId)); - } - - if (pRsp->streamId) { - tscKillStream(pObj, htonl(pRsp->streamId)); - } - } - - int32_t total = htonl(pRsp->totalDnodes); - int32_t online = htonl(pRsp->onlineDnodes); - assert(online <= total); - - if (online < total) { - tscError("0x%"PRIx64", HB, total dnode:%d, online dnode:%d", pSql->self, total, online); - pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - } - - if (pRes->length == NULL) { - pRes->length = calloc(2, sizeof(int32_t)); - } - - pRes->length[0] = total; - pRes->length[1] = online; - } else { - tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); - if (pRes->length == NULL) { - pRes->length = calloc(2, sizeof(int32_t)); - } - - pRes->length[1] = 0; - if (pRes->length[0] == 0) { - pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node - } - } - - if (pObj->hbrid != 0) { - int32_t waitingDuring = tsShellActivityTimer * 500; - tscDebug("0x%"PRIx64" send heartbeat in %dms", pSql->self, waitingDuring); - - taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); - } else { - tscDebug("0x%"PRIx64" start to close tscObj:%p, not send heartbeat again", pSql->self, pObj); - } -} - -void tscProcessActivityTimer(void *handle, void *tmrId) { - int64_t rid = (int64_t) handle; - STscObj *pObj = taosAcquireRef(tscRefId, rid); - if (pObj == NULL) { - return; - } - - SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHB == NULL) { - taosReleaseRef(tscRefId, rid); - return; - } - - assert(pHB->self == pObj->hbrid); - - pHB->retry = 0; - int32_t code = tscBuildAndSendRequest(pHB, NULL); - taosReleaseRef(tscObjRef, pObj->hbrid); - - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to sent HB to server, reason:%s", pHB->self, tstrerror(code)); - } - - taosReleaseRef(tscRefId, rid); -} - -// handle three situation -// 1. epset retry, only return last failure ep -// 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn -// 3. other situation, no expected -void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - char* msgBuf = tscGetErrorMsgPayload(pCmd); - - if (pEpSet) { - sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); - } else if (pCmd->command >= TSDB_SQL_MGMT) { - SRpcEpSet tEpset; - - SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginRead(&pCorEpSet->version); - tEpset = pCorEpSet->epSet; - taosCorEndRead(&pCorEpSet->version); - - sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),tEpset.fqdn[(tEpset.inUse)%(tEpset.numOfEps)]); - } else { - sprintf(msgBuf, "%s", tstrerror(pRes->code)); - } -} - -int doBuildAndSendMsg(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE_MNODE || - pCmd->command == TSDB_SQL_INSERT || - pCmd->command == TSDB_SQL_CONNECT || - pCmd->command == TSDB_SQL_HB || - pCmd->command == TSDB_SQL_RETRIEVE_FUNC || - pCmd->command == TSDB_SQL_STABLEVGROUP) { - pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); - } - - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return TSDB_CODE_SUCCESS; - } - - int32_t code = tscSendMsgToServer(pSql); - - // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. - if (code != TSDB_CODE_SUCCESS) { - pRes->code = code; - tscAsyncResultOnError(pSql); - return TSDB_CODE_SUCCESS; - } - - return TSDB_CODE_SUCCESS; -} - -int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - - SSqlCmd *pCmd = &pSql->cmd; - uint32_t type = 0; - - if (pQueryInfo == NULL) { - pQueryInfo = tscGetQueryInfo(pCmd); - } - - STableMetaInfo *pTableMetaInfo = NULL; - - if (pQueryInfo != NULL) { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo != NULL) { - tNameExtractFullName(&pTableMetaInfo->name, name); - } - - type = pQueryInfo->type; - - // while numOfTables equals to 0, it must be Heartbeat - assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0); - } - - tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type); - if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL - if (pTableMetaInfo == NULL) { - pSql->res.code = TSDB_CODE_TSC_APP_ERROR; - return pSql->res.code; - } - } else if (pCmd->command >= TSDB_SQL_LOCAL) { - return (*tscProcessMsgRsp[pCmd->command])(pSql); - } - - return doBuildAndSendMsg(pSql); -} - -int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - pRetrieveMsg->free = htons(pQueryInfo->type); - pRetrieveMsg->qId = htobe64(pSql->res.qId); - - // todo valid the vgroupId at the client side - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int32_t vgIndex = pTableMetaInfo->vgroupIndex; - int32_t vgId = -1; - - if (pTableMetaInfo->pVgroupTables == NULL) { - SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - vgId = pVgroupInfo->vgroups[vgIndex].vgId; - } else { - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(vgIndex >= 0 && vgIndex < numOfVgroups); - - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - vgId = pTableIdList->vgInfo.vgId; - } - - pRetrieveMsg->header.vgId = htonl(vgId); - tscDebug("0x%"PRIx64" build fetch msg from vgId:%d, vgIndex:%d, qId:0x%" PRIx64, pSql->self, vgId, vgIndex, pSql->res.qId); - } else { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); - tscDebug("0x%"PRIx64" build fetch msg from only one vgroup, vgId:%d, qId:0x%" PRIx64, pSql->self, pTableMeta->vgId, - pSql->res.qId); - } - - pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); - pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH; - - pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - - // pSql->cmd.payloadLen is set during copying data into payload - pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - - SNewVgroupInfo vgroupInfo = {0}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - - tscDebug("0x%"PRIx64" submit msg built, numberOfEP:%d", pSql->self, pSql->epSet.numOfEps); - - return TSDB_CODE_SUCCESS; -} - -/* - * for table query, simply return the size <= 1k - */ -static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { - const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - int32_t srcColFilterSize = 0; - int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo); - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2); - - int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; - int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; - - int32_t tableSerialize = 0; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMetaInfo->pVgroupTables != NULL) { - size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); - - int32_t totalTables = 0; - for (int32_t i = 0; i < numOfGroups; ++i) { - SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); - totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); - } - - tableSerialize = totalTables * sizeof(STableIdInfo); - } - - if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) { - STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); - if (pCond != NULL && pCond->cond != NULL) { - srcColFilterSize = pCond->len; - } - } - - SCond* pCond = &pQueryInfo->tagCond.tbnameCond; - if (pCond->len > 0) { - srcColListSize += pCond->len; - } - - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + - exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; -} - -static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg, - int32_t *succeed) { - TSKEY dfltKey = htobe64(pQueryMsg->window.skey); - - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { - - int32_t vgId = -1; - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int32_t index = pTableMetaInfo->vgroupIndex; - assert(index >= 0); - - SVgroupMsg* pVgroupInfo = NULL; - if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { - assert(index < pTableMetaInfo->vgroupList->numOfVgroups); - pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; - } else { - tscError("0x%"PRIx64" No vgroup info found", pSql->self); - - *succeed = 0; - return pMsg; - } - - vgId = pVgroupInfo->vgId; - tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); - tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, index, pTableMetaInfo->vgroupList->numOfVgroups); - } else { - vgId = pTableMeta->vgId; - - SNewVgroupInfo vgroupInfo = {0}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - } - - if (pSql->epSet.numOfEps > 0){ - pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; - } - pQueryMsg->head.vgId = htonl(vgId); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pTableMeta->id.tid); - pTableIdInfo->uid = htobe64(pTableMeta->id.uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey)); - - pQueryMsg->numOfTables = htonl(1); // set the number of tables - pMsg += sizeof(STableIdInfo); - } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo - int32_t index = pTableMetaInfo->vgroupIndex; - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(index >= 0 && index < numOfVgroups); - - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); - - // set the vgroup info - tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); - pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); - - int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); - pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables - - tscDebug("0x%"PRIx64" query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql->self, - pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); - - // serialize each table id info - for(int32_t i = 0; i < numOfTables; ++i) { - STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pItem->tid); - pTableIdInfo->uid = htobe64(pItem->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey)); - pMsg += sizeof(STableIdInfo); - } - } - - char n[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, n); - - tscDebug("0x%"PRIx64" vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql->self, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); - return pMsg; -} - -// TODO refactor -static int32_t serializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { - // append the filter information after the basic column information - for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo *pColFilter = &pColFilters[f]; - - SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); - pFilterMsg->filterstr = htons(pColFilter->filterstr); - - (*pMsg) += sizeof(SColumnFilterInfo); - - if (pColFilter->filterstr) { - pFilterMsg->len = htobe64(pColFilter->len); - memcpy(*pMsg, (void *)pColFilter->pz, (size_t)(pColFilter->len + 1)); - (*pMsg) += (pColFilter->len + 1); // append the additional filter binary info - } else { - pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); - pFilterMsg->upperBndi = htobe64(pColFilter->upperBndi); - } - - pFilterMsg->lowerRelOptr = htons(pColFilter->lowerRelOptr); - pFilterMsg->upperRelOptr = htons(pColFilter->upperRelOptr); - - if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { - tscError("invalid filter info"); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, char** pMsg, int64_t id, bool validateColumn) { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - // the queried table has been removed and a new table with the same name has already been created already - // return error msg - if (pExpr->uid != pTableMeta->id.uid) { - tscError("0x%"PRIx64" table has already been destroyed", id); - return TSDB_CODE_TSC_INVALID_TABLE_NAME; - } - - if (validateColumn && !tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { - tscError("0x%"PRIx64" table schema is not matched with parsed sql", id); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - assert(pExpr->resColId < 0); - SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); - - SColIndex* pIndex = &pSqlExpr->colInfo; - - pIndex->colId = htons(pExpr->colInfo.colId); - pIndex->colIndex = htons(pExpr->colInfo.colIndex); - pIndex->flag = htons(pExpr->colInfo.flag); - pSqlExpr->uid = htobe64(pExpr->uid); - pSqlExpr->colType = htons(pExpr->colType); - pSqlExpr->colBytes = htons(pExpr->colBytes); - pSqlExpr->resType = htons(pExpr->resType); - pSqlExpr->resBytes = htons(pExpr->resBytes); - pSqlExpr->interBytes = htonl(pExpr->interBytes); - pSqlExpr->functionId = htons(pExpr->functionId); - pSqlExpr->numOfParams = htons(pExpr->numOfParams); - pSqlExpr->resColId = htons(pExpr->resColId); - pSqlExpr->flist.numOfFilters = htons(pExpr->flist.numOfFilters); - - (*pMsg) += sizeof(SSqlExpr); - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log - pSqlExpr->param[j].nType = htonl(pExpr->param[j].nType); - pSqlExpr->param[j].nLen = htonl(pExpr->param[j].nLen); - - if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { - memcpy((*pMsg), pExpr->param[j].pz, pExpr->param[j].nLen); - (*pMsg) += pExpr->param[j].nLen; - } else { - pSqlExpr->param[j].i64 = htobe64(pExpr->param[j].i64); - } - } - - serializeColFilterInfo(pExpr->flist.filterInfo, pExpr->flist.numOfFilters, pMsg); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = NULL; - STableMeta *pTableMeta = NULL; - STableMetaInfo *pTableMetaInfo = NULL; - - int32_t code = TSDB_CODE_SUCCESS; - int32_t size = tscEstimateQueryMsgSize(pSql); - assert(size > 0); - - if (TSDB_CODE_SUCCESS != tscAllocPayloadFast(pCmd, size)) { - tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_TSC_INVALID_OPERATION; // todo add test for this - } - - pQueryInfo = tscGetQueryInfo(pCmd); - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - pTableMeta = pTableMetaInfo->pTableMeta; - - SQueryAttr query = {{0}}; - tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); - query.vgId = pTableMeta->vgId; - - SArray* tableScanOperator = createTableScanPlan(&query); - SArray* queryOperator = createExecOperatorPlan(&query); - - SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; - tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); - - int32_t numOfTags = query.numOfTags; - int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); - - if (taosArrayGetSize(tableScanOperator) == 0) { - pQueryMsg->tableScanOperator = htonl(-1); - } else { - int32_t* tablescanOp = taosArrayGet(tableScanOperator, 0); - pQueryMsg->tableScanOperator = htonl(*tablescanOp); - } - - pQueryMsg->window.skey = htobe64(query.window.skey); - pQueryMsg->window.ekey = htobe64(query.window.ekey); - - pQueryMsg->order = htons(query.order.order); - pQueryMsg->orderColId = htons(query.order.orderColId); - pQueryMsg->fillType = htons(query.fillType); - pQueryMsg->limit = htobe64(query.limit.limit); - pQueryMsg->offset = htobe64(query.limit.offset); - pQueryMsg->numOfCols = htons(query.numOfCols); - - pQueryMsg->interval.interval = htobe64(query.interval.interval); - pQueryMsg->interval.sliding = htobe64(query.interval.sliding); - pQueryMsg->interval.offset = htobe64(query.interval.offset); - pQueryMsg->interval.intervalUnit = query.interval.intervalUnit; - pQueryMsg->interval.slidingUnit = query.interval.slidingUnit; - pQueryMsg->interval.offsetUnit = query.interval.offsetUnit; - - pQueryMsg->stableQuery = query.stableQuery; - pQueryMsg->topBotQuery = query.topBotQuery; - pQueryMsg->groupbyColumn = query.groupbyColumn; - pQueryMsg->hasTagResults = query.hasTagResults; - pQueryMsg->timeWindowInterpo = query.timeWindowInterpo; - pQueryMsg->queryBlockDist = query.queryBlockDist; - pQueryMsg->stabledev = query.stabledev; - pQueryMsg->tsCompQuery = query.tsCompQuery; - pQueryMsg->simpleAgg = query.simpleAgg; - pQueryMsg->pointInterpQuery = query.pointInterpQuery; - pQueryMsg->needReverseScan = query.needReverseScan; - pQueryMsg->stateWindow = query.stateWindow; - pQueryMsg->numOfTags = htonl(numOfTags); - pQueryMsg->sqlstrLen = htonl(sqlLen); - pQueryMsg->sw.gap = htobe64(query.sw.gap); - pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); - - pQueryMsg->secondStageOutput = htonl(query.numOfExpr2); - pQueryMsg->numOfOutput = htons((int16_t)query.numOfOutput); // this is the stage one output column number - - pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); - pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); - pQueryMsg->queryType = htonl(pQueryInfo->type); - pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); - - // set column list ids - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - char *pMsg = (char *)(pQueryMsg->tableCols) + numOfCols * sizeof(SColumnInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfo *pCol = &query.tableCols[i]; - - pQueryMsg->tableCols[i].colId = htons(pCol->colId); - pQueryMsg->tableCols[i].bytes = htons(pCol->bytes); - pQueryMsg->tableCols[i].type = htons(pCol->type); - //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); - pQueryMsg->tableCols[i].flist.numOfFilters = 0; - pQueryMsg->tableCols[i].flist.filterInfo = 0; - // append the filter information after the basic column information - //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); - } - - if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) { - STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); - if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->colCondLen = htons(pCond->len); - memcpy(pMsg, pCond->cond, pCond->len); - - pMsg += pCond->len; - } - } else { - pQueryMsg->colCondLen = 0; - } - - for (int32_t i = 0; i < query.numOfOutput; ++i) { - code = serializeSqlExpr(&query.pExpr1[i].base, pTableMetaInfo, &pMsg, pSql->self, true); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - for (int32_t i = 0; i < query.numOfExpr2; ++i) { - code = serializeSqlExpr(&query.pExpr2[i].base, pTableMetaInfo, &pMsg, pSql->self, false); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - int32_t succeed = 1; - - // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pQueryMsg, pSql, pTableMetaInfo, pMsg, &succeed); - if (succeed == 0) { - code = TSDB_CODE_TSC_APP_ERROR; - goto _end; - } - - SGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; - if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupCols > 0) { - pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); - pQueryMsg->orderType = htons(pGroupbyExpr->orderType); - - for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); - - *((int16_t *)pMsg) = htons(pCol->colId); - pMsg += sizeof(pCol->colId); - - *((int16_t *)pMsg) += htons(pCol->colIndex); - pMsg += sizeof(pCol->colIndex); - - *((int16_t *)pMsg) += htons(pCol->flag); - pMsg += sizeof(pCol->flag); - - memcpy(pMsg, pCol->name, tListLen(pCol->name)); - pMsg += tListLen(pCol->name); - } - } - - if (query.fillType != TSDB_FILL_NONE) { - for (int32_t i = 0; i < query.numOfOutput; ++i) { - *((int64_t *)pMsg) = htobe64(query.fillVal[i]); - pMsg += sizeof(query.fillVal[0]); - } - } - - if (query.numOfTags > 0 && query.tagColList != NULL) { - for (int32_t i = 0; i < query.numOfTags; ++i) { - SColumnInfo* pTag = &query.tagColList[i]; - - SColumnInfo* pTagCol = (SColumnInfo*) pMsg; - pTagCol->colId = htons(pTag->colId); - pTagCol->bytes = htons(pTag->bytes); - pTagCol->type = htons(pTag->type); - pTagCol->flist.numOfFilters = 0; - - pMsg += sizeof(SColumnInfo); - } - } - - // serialize tag column query condition - if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { - STagCond* pTagCond = &pQueryInfo->tagCond; - - SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); - if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->tagCondLen = htons(pCond->len); - memcpy(pMsg, pCond->cond, pCond->len); - - pMsg += pCond->len; - } - } else { - pQueryMsg->tagCondLen = 0; - } - - if (pQueryInfo->bufLen > 0) { - memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); - pMsg += pQueryInfo->bufLen; - } - - SCond* pCond = &pQueryInfo->tagCond.tbnameCond; - if (pCond->len > 0) { - strncpy(pMsg, pCond->cond, pCond->len); - pMsg += pCond->len; - } - - // compressed ts block - pQueryMsg->tsBuf.tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); - - if (pQueryInfo->tsBuf != NULL) { - // note: here used the index instead of actual vnode id. - int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; - code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - - pMsg += pQueryMsg->tsBuf.tsLen; - - pQueryMsg->tsBuf.tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); - pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); - pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); - } else { - pQueryMsg->tsBuf.tsLen = 0; - pQueryMsg->tsBuf.tsNumOfBlocks = 0; - } - - int32_t numOfOperator = (int32_t) taosArrayGetSize(queryOperator); - pQueryMsg->numOfOperator = htonl(numOfOperator); - for(int32_t i = 0; i < numOfOperator; ++i) { - int32_t *operator = taosArrayGet(queryOperator, i); - *(int32_t*)pMsg = htonl(*operator); - - pMsg += sizeof(int32_t); - } - - // support only one udf - if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { - pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); - for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); - *(int8_t*) pMsg = pUdfInfo->resType; - pMsg += sizeof(pUdfInfo->resType); - - *(int16_t*) pMsg = htons(pUdfInfo->resBytes); - pMsg += sizeof(pUdfInfo->resBytes); - - STR_TO_VARSTR(pMsg, pUdfInfo->name); - - pMsg += varDataTLen(pMsg); - - *(int32_t*) pMsg = htonl(pUdfInfo->funcType); - pMsg += sizeof(pUdfInfo->funcType); - - *(int32_t*) pMsg = htonl(pUdfInfo->bufSize); - pMsg += sizeof(pUdfInfo->bufSize); - - pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen); - memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen); - - pMsg += pUdfInfo->contLen; - } - } else { - pQueryMsg->udfContentOffset = 0; - pQueryMsg->udfContentLen = 0; - } - - memcpy(pMsg, pSql->sqlstr, sqlLen); - pMsg += sqlLen; - - int32_t msgLen = (int32_t)(pMsg - pCmd->payload); - - tscDebug("0x%"PRIx64" msg built success, len:%d bytes", pSql->self, msgLen); - pCmd->payloadLen = msgLen; - pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; - - pQueryMsg->head.contLen = htonl(msgLen); - assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); - - _end: - freeQueryAttr(&query); - taosArrayDestroy(tableScanOperator); - taosArrayDestroy(queryOperator); - return code; -} - -int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateDbMsg); - - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; - - SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; - -// assert(pCmd->numOfClause == 1); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); - assert(code == TSDB_CODE_SUCCESS); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload; - - pCmd->msgType = TSDB_MSG_TYPE_CREATE_FUNCTION; - - pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateDnodeMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; - - SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); - strncpy(pCreate->ep, t0->z, t0->n); - - pCmd->msgType = TSDB_MSG_TYPE_CREATE_DNODE; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateAcctMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; - - SStrToken *pName = &pInfo->pMiscInfo->user.user; - SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; - - strncpy(pAlterMsg->user, pName->z, pName->n); - strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); - - SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; - - pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); - pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); - pAlterMsg->cfg.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); - pAlterMsg->cfg.maxStreams = htonl(pAcctOpt->maxStreams); - pAlterMsg->cfg.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); - pAlterMsg->cfg.maxStorage = htobe64(pAcctOpt->maxStorage); - pAlterMsg->cfg.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); - pAlterMsg->cfg.maxConnections = htonl(pAcctOpt->maxConnections); - - if (pAcctOpt->stat.n == 0) { - pAlterMsg->cfg.accessState = -1; - } else { - if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { - pAlterMsg->cfg.accessState = TSDB_VN_READ_ACCCESS; - } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { - pAlterMsg->cfg.accessState = TSDB_VN_WRITE_ACCCESS; - } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { - pAlterMsg->cfg.accessState = TSDB_VN_ALL_ACCCESS; - } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { - pAlterMsg->cfg.accessState = 0; - } - } - - pCmd->msgType = TSDB_MSG_TYPE_CREATE_ACCT; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateUserMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; - - SUserInfo *pUser = &pInfo->pMiscInfo->user; - strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); - pAlterMsg->flag = (int8_t)pUser->type; - - if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - pAlterMsg->privilege = (char)pCmd->count; - } else if (pUser->type == TSDB_ALTER_USER_PASSWD) { - strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); - } else { // create user password info - strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); - } - - if (pUser->type == TSDB_ALTER_USER_PASSWD || pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - pCmd->msgType = TSDB_MSG_TYPE_ALTER_USER; - } else { - pCmd->msgType = TSDB_MSG_TYPE_CREATE_USER; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCfgDnodeMsg); - pCmd->msgType = TSDB_MSG_TYPE_CONFIG_DNODE; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SDropDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); - assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); - - pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - - pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_DROP_DB : TSDB_MSG_TYPE_DROP_TP; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->msgType = TSDB_MSG_TYPE_DROP_FUNCTION; - - pCmd->payloadLen = sizeof(SDropFuncMsg); - - return TSDB_CODE_SUCCESS; -} - - -int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropTableMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); - - pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; - pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - pCmd->msgType = TSDB_MSG_TYPE_DROP_TABLE; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char dnodeEp[TSDB_EP_LEN] = {0}; - tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); - - pCmd->payloadLen = sizeof(SDropDnodeMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; - tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); - pCmd->msgType = TSDB_MSG_TYPE_DROP_DNODE; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char user[TSDB_USER_LEN] = {0}; - tstrncpy(user, pCmd->payload, TSDB_USER_LEN); - - pCmd->payloadLen = sizeof(SDropUserMsg); - pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_DROP_USER:TSDB_MSG_TYPE_DROP_ACCT; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropUserMsg *pDropMsg = (SDropUserMsg *)pCmd->payload; - tstrncpy(pDropMsg->user, user, tListLen(user)); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SUseDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); - pCmd->msgType = TSDB_MSG_TYPE_USE_DB; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SSyncDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); - pCmd->msgType = TSDB_MSG_TYPE_SYNC_DB; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_SHOW; - pCmd->payloadLen = sizeof(SShowMsg) + 100; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; - SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { - pShowMsg->type = pShowInfo->showType; - pShowMsg->payloadLen = 0; - pCmd->payloadLen = sizeof(SShowMsg); - - return TSDB_CODE_SUCCESS; - } - - if (tNameIsEmpty(&pTableMetaInfo->name)) { - char *p = cloneCurrentDBName(pSql); - tstrncpy(pShowMsg->db, p, sizeof(pShowMsg->db)); - tfree(p); - } else { - tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); - } - - pShowMsg->type = pShowInfo->showType; - - if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SStrToken *pPattern = &pShowInfo->pattern; - if (pPattern->type > 0) { // only show tables support wildcard query - strncpy(pShowMsg->payload, pPattern->z, pPattern->n); - pShowMsg->payloadLen = htons(pPattern->n); - } - } else { - SStrToken *pEpAddr = &pShowInfo->prefix; - assert(pEpAddr->n > 0 && pEpAddr->type > 0); - - strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); - pShowMsg->payloadLen = htons(pEpAddr->n); - } - - pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SKillQueryMsg); - - switch (pCmd->command) { - case TSDB_SQL_KILL_QUERY: - pCmd->msgType = TSDB_MSG_TYPE_KILL_QUERY; - break; - case TSDB_SQL_KILL_CONNECTION: - pCmd->msgType = TSDB_MSG_TYPE_KILL_CONN; - break; - case TSDB_SQL_KILL_STREAM: - pCmd->msgType = TSDB_MSG_TYPE_KILL_STREAM; - break; - } - return TSDB_CODE_SUCCESS; -} - -int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &(pSql->cmd); - int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); - - SCreateTableSql *pCreateTableInfo = pInfo->pCreateTableInfo; - if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { - int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); - size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); - } else { - size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); - } - - if (pCreateTableInfo->pSelect != NULL) { - size += (pCreateTableInfo->pSelect->sqlstr.n + 1); - } - - return size + TSDB_EXTRA_PAYLOAD_SIZE; -} - -int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - int msgLen = 0; - int size = 0; - SSchema *pSchema; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - // Reallocate the payload size - size = tscEstimateCreateTableMsgLength(pSql, pInfo); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for create table msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; - - SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); - char* pMsg = NULL; - - int8_t type = pInfo->pCreateTableInfo->type; - if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value - SArray* list = pInfo->pCreateTableInfo->childTableInfo; - - int32_t numOfTables = (int32_t) taosArrayGetSize(list); - pCreateTableMsg->numOfTables = htonl(numOfTables); - - pMsg = (char*) pCreateMsg; - for(int32_t i = 0; i < numOfTables; ++i) { - SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; - - pCreate->numOfColumns = htons(pCmd->numOfCols); - pCreate->numOfTags = htons(pCmd->count); - pMsg += sizeof(SCreateTableMsg); - - SCreatedTableInfo* p = taosArrayGet(list, i); - strcpy(pCreate->tableName, p->fullname); - pCreate->igExists = (p->igExist)? 1 : 0; - - // use dbinfo from table id without modifying current db info - pMsg = serializeTagData(&p->tagdata, pMsg); - - int32_t len = (int32_t)(pMsg - (char*) pCreate); - pCreate->len = htonl(len); - } - } else { // create (super) table - pCreateTableMsg->numOfTables = htonl(1); // only one table will be created - - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); - assert(code == 0); - - SCreateTableSql *pCreateTable = pInfo->pCreateTableInfo; - - pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; - pCreateMsg->numOfColumns = htons(pCmd->numOfCols); - pCreateMsg->numOfTags = htons(pCmd->count); - - pCreateMsg->sqlLen = 0; - pMsg = (char *)pCreateMsg->schema; - - pSchema = (SSchema *)pCreateMsg->schema; - - for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - pSchema->type = pField->type; - strcpy(pSchema->name, pField->name); - pSchema->bytes = htons(pField->bytes); - - pSchema++; - } - - pMsg = (char *)pSchema; - if (type == TSQL_CREATE_STREAM) { // check if it is a stream sql - SSqlNode *pQuerySql = pInfo->pCreateTableInfo->pSelect; - - strncpy(pMsg, pQuerySql->sqlstr.z, pQuerySql->sqlstr.n + 1); - pCreateMsg->sqlLen = htons(pQuerySql->sqlstr.n + 1); - pMsg += pQuerySql->sqlstr.n + 1; - } - } - - tscFieldInfoClear(&pQueryInfo->fieldsInfo); - - msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); - pCreateTableMsg->contLen = htonl(msgLen); - pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_CREATE_TABLE; - - assert(msgLen + minMsgSize() <= size); - return TSDB_CODE_SUCCESS; -} - -int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; -} - -int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - char *pMsg; - int msgLen = 0; - - SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; - int size = tscEstimateAlterTableMsgLength(pCmd); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; - - tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); - pAlterTableMsg->type = htons(pAlterInfo->type); - - pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); - SSchema *pSchema = pAlterTableMsg->schema; - for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - pSchema->type = pField->type; - strcpy(pSchema->name, pField->name); - pSchema->bytes = htons(pField->bytes); - pSchema++; - } - - pMsg = (char *)pSchema; - pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen); - if (pAlterInfo->tagData.dataLen > 0) { - memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); - } - pMsg += pAlterInfo->tagData.dataLen; - - msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); - - pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_ALTER_TABLE; - - assert(msgLen + minMsgSize() <= size); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_UPDATE_TAG_VAL; - - SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; - pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - - SNewVgroupInfo vgroupInfo = {.vgId = -1}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - assert(vgroupInfo.vgId > 0); - - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - - return TSDB_CODE_SUCCESS; -} - -int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SAlterDbMsg); - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; - - SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; - pAlterDbMsg->dbType = -1; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); - - return TSDB_CODE_SUCCESS; -} -int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - if (pInfo->list == NULL || taosArrayGetSize(pInfo->list) <= 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - STscObj *pObj = pSql->pTscObj; - SSqlCmd *pCmd = &pSql->cmd; - SArray *pList = pInfo->list; - int32_t size = (int32_t)taosArrayGetSize(pList); - - int32_t *result = malloc(sizeof(int32_t) * size); - if (result == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < size; i++) { - tSqlExprItem* pSub = taosArrayGet(pList, i); - tVariant* pVar = &pSub->pNode->value; - if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { - result[i] = (int32_t)(pVar->i64); - } else { - free(result); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - int count = removeDupVgid(result, size); - pCmd->payloadLen = sizeof(SCompactMsg) + count * sizeof(int32_t); - pCmd->msgType = TSDB_MSG_TYPE_COMPACT_VNODE; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - free(result); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - SCompactMsg *pCompactMsg = (SCompactMsg *)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - if (tNameIsEmpty(&pTableMetaInfo->name)) { - pthread_mutex_lock(&pObj->mutex); - tstrncpy(pCompactMsg->db, pObj->db, sizeof(pCompactMsg->db)); - pthread_mutex_unlock(&pObj->mutex); - } else { - tNameGetFullDbName(&pTableMetaInfo->name, pCompactMsg->db); - } - - pCompactMsg->numOfVgroup = htons(count); - for (int32_t i = 0; i < count; i++) { - pCompactMsg->vgid[i] = htons(result[i]); - } - free(result); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - pCmd->payloadLen = sizeof(SRetrieveTableMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg*)pCmd->payload; - pRetrieveMsg->qId = htobe64(pSql->res.qId); - pRetrieveMsg->free = htons(pQueryInfo->type); - - return TSDB_CODE_SUCCESS; -} - -/* - * this function can only be called once. - * by using pRes->rspType to denote its status - * - * if pRes->rspType is 1, no more result - */ -static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - pRes->code = TSDB_CODE_SUCCESS; - if (pRes->rspType == 0) { - pRes->numOfRows = numOfRes; - pRes->row = 0; - pRes->rspType = 1; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { - return pRes->code; - } - - tscSetResRawPtr(pRes, pQueryInfo); - } else { - tscResetForNextRetrieve(pRes); - } - - uint8_t code = pSql->res.code; - if (pSql->fp) { - if (code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); - } else { - tscAsyncResultOnError(pSql); - } - } - - return code; -} - -int tscProcessDescribeTableRsp(SSqlObj *pSql) { - SSqlCmd * pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - int32_t numOfRes = tinfo.numOfColumns + tinfo.numOfTags; - return tscLocalResultCommonBuilder(pSql, numOfRes); -} - -int tscProcessLocalRetrieveRsp(SSqlObj *pSql) { - int32_t numOfRes = 1; - pSql->res.completed = true; - return tscLocalResultCommonBuilder(pSql, numOfRes); -} - -int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd* pCmd = &pSql->cmd; - - int32_t code = pRes->code; - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return code; - } - - if (pRes->pMerger == NULL) { // no result from subquery, so abort here directly. - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - return code; - } - - // global aggregation may be the upstream for parent query - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo->pQInfo == NULL) { - STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; - tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; - - SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(group, &tableKeyInfo); - taosArrayPush(tableGroupInfo.pGroupList, &group); - - tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute query processing", pSql->self, pSql->self); - pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, &tableGroupInfo, NULL, NULL, pRes->pMerger, MERGE_STAGE, pSql->self); - } - - uint64_t localQueryId = pSql->self; - qTableQuery(pQueryInfo->pQInfo, &localQueryId); - convertQueryResult(pRes, pQueryInfo, pSql->self, true); - - code = pRes->code; - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - } else { - tscAsyncResultOnError(pSql); - } - - return code; -} - -int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 0); } - -int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - STscObj *pObj = pSql->pTscObj; - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_CONNECT; - pCmd->payloadLen = sizeof(SConnectMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload; - - // TODO refactor full_name - char *db; // ugly code to move the space - - pthread_mutex_lock(&pObj->mutex); - db = strstr(pObj->db, TS_PATH_DELIMITER); - - db = (db == NULL) ? pObj->db : db + 1; - tstrncpy(pConnect->db, db, sizeof(pConnect->db)); - pthread_mutex_unlock(&pObj->mutex); - - tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); - tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); - - pConnect->pid = htonl(taosGetPId()); - taosGetCurrentAPPName(pConnect->appName, NULL); - - return TSDB_CODE_SUCCESS; -} - -/** - * multi table meta req pkg format: - * |SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... - * 4B - **/ -int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->msgType = TSDB_MSG_TYPE_TABLES_META; - assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); - - tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, - pCmd->payloadLen); - - return pCmd->payloadLen; -} - -int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char* pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; - pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); - pMsg += sizeof(SSTableVgroupMsg); - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, i); - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); - assert(code == TSDB_CODE_SUCCESS); - - pMsg += TSDB_TABLE_FNAME_LEN; - } - - pCmd->msgType = TSDB_MSG_TYPE_STABLE_VGROUP; - pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char *pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); - - SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg; - pRetrieveFuncMsg->num = htonl(numOfFuncs); - - pMsg += sizeof(SRetrieveFuncMsg); - for(int32_t i = 0; i < numOfFuncs; ++i) { - SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i); - STR_TO_NET_VARSTR(pMsg, pUdf->name); - pMsg += varDataNetTLen(pMsg); - } - - pCmd->msgType = TSDB_MSG_TYPE_RETRIEVE_FUNC; - pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; - - pthread_mutex_lock(&pObj->mutex); - - int32_t numOfQueries = 2; - SSqlObj *tpSql = pObj->sqlList; - while (tpSql) { - tpSql = tpSql->next; - numOfQueries++; - } - - int32_t numOfStreams = 2; - SSqlStream *pStream = pObj->streamList; - while (pStream) { - pStream = pStream->next; - numOfStreams++; - } - - int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - pthread_mutex_unlock(&pObj->mutex); - tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // TODO the expired hb and client can not be identified by server till now. - SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload; - tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); - - pHeartbeat->numOfQueries = numOfQueries; - pHeartbeat->numOfStreams = numOfStreams; - - pHeartbeat->pid = htonl(taosGetPId()); - taosGetCurrentAPPName(pHeartbeat->appName, NULL); - - int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj); - - pthread_mutex_unlock(&pObj->mutex); - - pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); - pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - - if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && - (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { - tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, - pMetaMsg->tid, pMetaMsg->tableFname); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) { - tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { - tscError("invalid numOfColumns:%d", pMetaMsg->numOfColumns); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { - pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); - } - - SSchema* pSchema = pMetaMsg->schema; - - int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; - for (int i = 0; i < numOfTotalCols; ++i) { - pSchema->bytes = htons(pSchema->bytes); - pSchema->colId = htons(pSchema->colId); - - if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - assert(i == 0); - } - - pSchema++; - } - - return TSDB_CODE_SUCCESS; -} - -// update the vgroupInfo if needed -static void doUpdateVgroupInfo(int32_t vgId, SVgroupMsg *pVgroupMsg) { - assert(vgId > 0); - - SNewVgroupInfo vgroupInfo = {.inUse = -1}; - taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); - - // vgroup info exists, compare with it - if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, pVgroupMsg)) || (vgroupInfo.inUse < 0)) { - vgroupInfo = createNewVgroupInfo(pVgroupMsg); - taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); - tscDebug("add/update new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); - } -} - -static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMetaMsg, bool updateSTable) { - if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - // add or update the corresponding super table meta data info - int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - - // The super tableMeta already exists, create it according to tableMeta and add it to hash map - if (updateSTable) { - STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); - uint32_t size = tscGetTableMetaSize(pSupTableMeta); - int32_t code = taosHashPut(tscTableMetaMap, pTableMeta->sTableName, len, pSupTableMeta, size); - assert(code == TSDB_CODE_SUCCESS); - - tfree(pSupTableMeta); - } - - CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); - taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); - tfree(cMeta); - } else { - uint32_t s = tscGetTableMetaSize(pTableMeta); - taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), pTableMeta, s); - } -} - -int tscProcessTableMetaRsp(SSqlObj *pSql) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - int32_t code = tableMetaMsgConvert(pMetaMsg); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - assert(pTableMetaInfo->pTableMeta == NULL); - - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (pTableMeta == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); - tfree(pTableMeta); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - assert(strncmp(pMetaMsg->tableFname, name, tListLen(pMetaMsg->tableFname)) == 0); - - doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, true); - if (pTableMeta->tableType != TSDB_SUPER_TABLE) { - doUpdateVgroupInfo(pTableMeta->vgId, &pMetaMsg->vgroup); - } - - tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self, - pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns, - pTableMeta->tableInfo.numOfTags); - - free(pTableMeta); - return TSDB_CODE_SUCCESS; -} - -static SArray* createVgroupIdListFromMsg(char* pMsg, SHashObj* pSet, char* name, int32_t* size, uint64_t id) { - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; - - pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); - - SArray* vgroupIdList = taosArrayInit(pVgroupMsg->numOfVgroups, sizeof(int32_t)); - - if (pVgroupMsg->numOfVgroups <= 0) { - tscDebug("0x%" PRIx64 " empty vgroup id list, no corresponding tables for stable:%s", id, name); - } else { - // just init, no need to lock - for (int32_t j = 0; j < pVgroupMsg->numOfVgroups; ++j) { - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - vmsg->vgId = htonl(vmsg->vgId); - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); - } - - taosArrayPush(vgroupIdList, &vmsg->vgId); - - if (taosHashGet(pSet, &vmsg->vgId, sizeof(vmsg->vgId)) == NULL) { - taosHashPut(pSet, &vmsg->vgId, sizeof(vmsg->vgId), "", 0); - doUpdateVgroupInfo(vmsg->vgId, vmsg); - } - } - } - - return vgroupIdList; -} - -static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t id) { - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; - pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - - *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); - - size_t vgroupsz = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); - SVgroupsInfo *pVgroupInfo = calloc(1, vgroupsz); - assert(pVgroupInfo != NULL); - - pVgroupInfo->numOfVgroups = pVgroupMsg->numOfVgroups; - if (pVgroupInfo->numOfVgroups <= 0) { - tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", id); - } else { - for (int32_t j = 0; j < pVgroupInfo->numOfVgroups; ++j) { - // just init, no need to lock - SVgroupMsg *pVgroup = &pVgroupInfo->vgroups[j]; - - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - vmsg->vgId = htonl(vmsg->vgId); - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); - } - - pVgroup->numOfEps = vmsg->numOfEps; - pVgroup->vgId = vmsg->vgId; - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - pVgroup->epAddr[k].port = vmsg->epAddr[k].port; - tstrncpy(pVgroup->epAddr[k].fqdn, vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); -// pVgroup->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); - } - - doUpdateVgroupInfo(pVgroup->vgId, vmsg); - } - } - - return pVgroupInfo; -} - -int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - pFuncMsg->num = htonl(pFuncMsg->num); - assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo)); - - char* pMsg = pFuncMsg->content; - for(int32_t i = 0; i < pFuncMsg->num; ++i) { - SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; - - for(int32_t j = 0; j < pFuncMsg->num; ++j) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); - if (strcmp(pUdfInfo->name, pFunc->name) != 0) { - continue; - } - - if (pUdfInfo->content) { - continue; - } - - pUdfInfo->resBytes = htons(pFunc->resBytes); - pUdfInfo->resType = pFunc->resType; - pUdfInfo->funcType = htonl(pFunc->funcType); - pUdfInfo->contLen = htonl(pFunc->len); - pUdfInfo->bufSize = htonl(pFunc->bufSize); - - pUdfInfo->content = malloc(pUdfInfo->contLen); - memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); - - pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; - } - } - - // master sqlObj locates in param - SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(parent == NULL) { - return pSql->res.code; - } - - SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd); - - assert(parent->signature == parent && (int64_t)pSql->param == parent->self); - taosArrayDestroy(parQueryInfo->pUdfInfo); - - parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj. - pQueryInfo->pUdfInfo = NULL; - return TSDB_CODE_SUCCESS; -} - -int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { - char *rsp = pSql->res.pRsp; - - SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; - pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); - pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup); - pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf); - - rsp += sizeof(SMultiTableMeta); - - SSqlObj* pParentSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(pParentSql == NULL) { - return pSql->res.code; - } - - SSqlCmd *pParentCmd = &pParentSql->cmd; - SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - - char* buf = NULL; - char* pMsg = pMultiMeta->meta; - - // decompresss the message payload - if (pMultiMeta->compressed) { - buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); - int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, - buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0); - assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta)); - - pMsg = buf; - } - - if (pParentCmd->pTableMetaMap == NULL) { - pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - } - - for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; - int32_t code = tableMetaMsgConvert(pMetaMsg); - if (code != TSDB_CODE_SUCCESS) { - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return code; - } - - bool freeMeta = false; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); - tfree(pTableMeta); - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) { - STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; - size_t keyLen = strnlen(pMetaMsg->tableFname, TSDB_TABLE_FNAME_LEN); - void* t = taosHashGet(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen); - assert(t == NULL); - - taosHashPut(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen, &p, sizeof(STableMetaVgroupInfo)); - } else { - freeMeta = true; - } - - // for each super table, only update meta information once - bool updateStableMeta = false; - if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) { - updateStableMeta = true; - taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0); - } - - // create the tableMeta and add it into the TableMeta map - doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta); - - // for each vgroup, only update the information once. - int64_t vgId = pMetaMsg->vgroup.vgId; - if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { - doUpdateVgroupInfo((int32_t) vgId, &pMetaMsg->vgroup); - taosHashPut(pSet, &vgId, sizeof(vgId), "", 0); - } - - pMsg += pMetaMsg->contLen; - if (freeMeta) { - tfree(pTableMeta); - } - } - - for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { - char fname[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(fname, pMsg, TSDB_TABLE_FNAME_LEN); - size_t len = strnlen(fname, TSDB_TABLE_FNAME_LEN); - - pMsg += TSDB_TABLE_FNAME_LEN; - - STableMetaVgroupInfo* p = taosHashGet(pParentCmd->pTableMetaMap, fname, len); - assert(p != NULL); - - int32_t size = 0; - if (p->vgroupIdList!= NULL) { - taosArrayDestroy(p->vgroupIdList); - } - - p->vgroupIdList = createVgroupIdListFromMsg(pMsg, pSet, fname, &size, pSql->self); - - int32_t numOfVgId = (int32_t) taosArrayGetSize(p->vgroupIdList); - int32_t s = sizeof(tFilePage) + numOfVgId * sizeof(int32_t); - - tFilePage* idList = calloc(1, s); - idList->num = numOfVgId; - memcpy(idList->data, TARRAY_GET_START(p->vgroupIdList), numOfVgId * sizeof(int32_t)); - - void* idListInst = taosCachePut(tscVgroupListBuf, fname, len, idList, s, 5000); - taosCacheRelease(tscVgroupListBuf, (void*) &idListInst, false); - - tfree(idList); - pMsg += size; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd); - if (pMultiMeta->numOfUdf > 0) { - assert(pQueryInfo->pUdfInfo != NULL); - } - - for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) { - SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; - - for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); - if (strcmp(pUdfInfo->name, pFunc->name) != 0) { - continue; - } - - if (pUdfInfo->content) { - continue; - } - - pUdfInfo->resBytes = htons(pFunc->resBytes); - pUdfInfo->resType = pFunc->resType; - pUdfInfo->funcType = htonl(pFunc->funcType); - pUdfInfo->contLen = htonl(pFunc->len); - pUdfInfo->bufSize = htonl(pFunc->bufSize); - - pUdfInfo->content = malloc(pUdfInfo->contLen); - memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); - - pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; - } - } - - pSql->res.code = TSDB_CODE_SUCCESS; - pSql->res.numOfTotal = pMultiMeta->numOfTables; - tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables); - - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return TSDB_CODE_SUCCESS; -} - -int tscProcessSTableVgroupRsp(SSqlObj *pSql) { - // master sqlObj locates in param - SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(parent == NULL) { - return pSql->res.code; - } - - assert(parent->signature == parent && (int64_t)pSql->param == parent->self); - - SSqlRes* pRes = &pSql->res; - - // NOTE: the order of several table must be preserved. - SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; - pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); - char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); - - SSqlCmd* pCmd = &parent->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - char fName[TSDB_TABLE_FNAME_LEN] = {0}; - for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { - char* name = pMsg; - pMsg += TSDB_TABLE_FNAME_LEN; - - STableMetaInfo *pInfo = NULL; - for(int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - STableMetaInfo *pInfo1 = tscGetTableMetaInfoFromCmd(pCmd, j); - memset(fName, 0, tListLen(fName)); - - tNameExtractFullName(&pInfo1->name, fName); - if (strcmp(name, fName) != 0) { - continue; - } - - pInfo = pInfo1; - break; - } - - if (!pInfo){ - continue; - } - int32_t size = 0; - pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); - pMsg += size; - } - - taosReleaseRef(tscObjRef, parent->self); - return pSql->res.code; -} - -int tscProcessShowRsp(SSqlObj *pSql) { - STableMetaMsg *pMetaMsg; - SShowRsp * pShow; - SSchema * pSchema; - - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - pShow = (SShowRsp *)pRes->pRsp; - pShow->qhandle = htobe64(pShow->qhandle); - pRes->qId = pShow->qhandle; - - tscResetForNextRetrieve(pRes); - pMetaMsg = &(pShow->tableMeta); - - pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); - - pSchema = pMetaMsg->schema; - pMetaMsg->tid = ntohs(pMetaMsg->tid); - for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { - pSchema->bytes = htons(pSchema->bytes); - pSchema++; - } - - tfree(pTableMetaInfo->pTableMeta); - pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - - SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - - SColumnIndex index = {0}; - pSchema = pMetaMsg->schema; - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { - index.columnIndex = i; - tscColumnListInsert(pQueryInfo->colList, i, uid, pSchema); - - TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); - SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); - - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, - pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); - } - - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - tscFieldInfoUpdateOffset(pQueryInfo); - return 0; -} - -static void createHbObj(STscObj* pObj) { - if (pObj->hbrid != 0) { - return; - } - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (NULL == pSql) return; - - pSql->fp = tscProcessHeartBeatRsp; - - SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd); - if (pQueryInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSql); - return; - } - - pQueryInfo->command = TSDB_SQL_HB; - - pSql->cmd.command = pQueryInfo->command; - if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSql); - return; - } - - pSql->param = pObj; - pSql->pTscObj = pObj; - pSql->signature = pSql; - - registerSqlObj(pSql); - tscDebug("0x%"PRIx64" HB is allocated, pObj:%p", pSql->self, pObj); - - pObj->hbrid = pSql->self; -} - -int tscProcessUseDbRsp(SSqlObj *pSql) { - STscObj * pObj = pSql->pTscObj; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - pthread_mutex_lock(&pObj->mutex); - int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); - pthread_mutex_unlock(&pObj->mutex); - - return ret; -} - -//todo only invalid the buffered data that belongs to dropped databases -int tscProcessDropDbRsp(SSqlObj *pSql) { - //TODO LOCK DB WHEN MODIFY IT - //pSql->pTscObj->db[0] = 0; - - taosHashClear(tscTableMetaMap); - taosHashClear(tscVgroupMap); - taosCacheEmpty(tscVgroupListBuf); - return 0; -} - -int tscProcessDropTableRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - tscRemoveCachedTableMeta(pTableMetaInfo, pSql->self); - tfree(pTableMetaInfo->pTableMeta); - return 0; -} - -int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - tscDebug("0x%"PRIx64" remove tableMeta in hashMap after alter-table: %s", pSql->self, name); - - bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - tfree(pTableMetaInfo->pTableMeta); - - if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta - if (pSql->res.pRsp == NULL) { - tscDebug("0x%"PRIx64" unexpected resp from mnode, super table: %s failed to update super table meta ", pSql->self, name); - return 0; - } - return tscProcessTableMetaRsp(pSql); - } - - return 0; -} - -int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { - UNUSED(pSql); - return 0; -} -int tscProcessCompactRsp(SSqlObj *pSql) { - UNUSED(pSql); - return TSDB_CODE_SUCCESS; -} - -int tscProcessShowCreateRsp(SSqlObj *pSql) { - return tscLocalResultCommonBuilder(pSql, 1); -} - -int tscProcessQueryRsp(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - - SQueryTableRsp *pQueryAttr = (SQueryTableRsp *)pRes->pRsp; - pQueryAttr->qId = htobe64(pQueryAttr->qId); - - pRes->qId = pQueryAttr->qId; - pRes->data = NULL; - - tscResetForNextRetrieve(pRes); - tscDebug("0x%"PRIx64" query rsp received, qId:0x%"PRIx64, pSql->self, pRes->qId); - return 0; -} - -static void decompressQueryColData(SSqlObj *pSql, SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int32_t compLen) { - int32_t decompLen = 0; - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - int32_t *compSizes; - char *pData = *data; - compSizes = (int32_t *)(pData + compLen); - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); - char *outputBuf = tcalloc(pRes->numOfRows, (pField->bytes + offset)); - - char *p = outputBuf; - int32_t bufOffset; - for (int32_t i = 0; i < numOfCols; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - bufOffset = pInfo->field.bytes * pRes->numOfRows; - - int32_t flen = (*(tDataTypes[pInfo->field.type].decompFunc))(pData, htonl(compSizes[i]), pRes->numOfRows, p, bufOffset, - compressed, NULL, 0); - - p += flen; - decompLen +=flen; - pData += htonl(compSizes[i]); - } - - /* Resize rsp as decompressed data will occupy more space */ - pRes->rspLen = pRes->rspLen - (compLen + numOfCols * sizeof(int32_t)) + decompLen; - char *new_rsp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (new_rsp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; - } else { - pRes->pRsp = new_rsp; - *data = ((SRetrieveTableRsp *)pRes->pRsp)->data; - pData = *data + compLen + numOfCols * sizeof(int32_t); - } - - tscDebug("0x%"PRIx64" decompress col data, compressed size:%d, decompressed size:%d", - pSql->self, (int32_t)(compLen + numOfCols * sizeof(int32_t)), decompLen); - - int32_t tailLen = pRes->rspLen - sizeof(SRetrieveTableRsp) - decompLen; - memmove(*data + decompLen, pData, tailLen); - memmove(*data, outputBuf, decompLen); - - tfree(outputBuf); -} - -int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); - - SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; - if (pRetrieve == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; - } - - pRes->numOfRows = htonl(pRetrieve->numOfRows); - pRes->precision = htons(pRetrieve->precision); - pRes->offset = htobe64(pRetrieve->offset); - pRes->useconds = htobe64(pRetrieve->useconds); - pRes->completed = (pRetrieve->completed == 1); - pRes->data = pRetrieve->data; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { - return pRes->code; - } - - //Decompress col data if compressed from server - if (pRetrieve->compressed) { - int32_t compLen = htonl(pRetrieve->compLen); - decompressQueryColData(pSql, pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); - } - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || - ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || - (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE))) { - tscSetResRawPtr(pRes, pQueryInfo); - } - - if (pSql->pSubscription != NULL) { - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); - - char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; - - int32_t numOfTables = htonl(*(int32_t*)p); - p += sizeof(int32_t); - for (int i = 0; i < numOfTables; i++) { - int64_t uid = htobe64(*(int64_t*)p); - p += sizeof(int64_t); - p += sizeof(int32_t); // skip tid - TSKEY key = htobe64(*(TSKEY*)p); - p += sizeof(TSKEY); - tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key); - } - } - - pRes->row = 0; - tscDebug("0x%"PRIx64" numOfRows:%d, offset:%" PRId64 ", complete:%d, qId:0x%"PRIx64, pSql->self, pRes->numOfRows, pRes->offset, - pRes->completed, pRes->qId); - - return 0; -} - -void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); - -static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_META; - - tscAddQueryInfo(&pNew->cmd); - - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { - tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); - - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - STableMetaInfo *pNewTableMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); - assert(pNewQueryInfo->numOfTables == 1); - - tNameAssign(&pNewTableMetaInfo->name, &pTableMetaInfo->name); - - registerSqlObj(pNew); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - - tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d, metaRid from %"PRId64" to %"PRId64, - pSql->self, pNew->self, autocreate, pSql->metaRid, pNew->self); - pSql->metaRid = pNew->self; - - { - STableInfoMsg *pInfoMsg = (STableInfoMsg *)pNew->cmd.payload; - int32_t code = tNameExtractFullName(&pNewTableMetaInfo->name, pInfoMsg->tableFname); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pInfoMsg->createFlag = htons(autocreate? 1 : 0); - char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); - - // tag data exists - if (autocreate && pSql->cmd.insertParam.tagData.dataLen != 0) { - pMsg = serializeTagData(&pSql->cmd.insertParam.tagData, pMsg); - } - - pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); - pNew->cmd.msgType = TSDB_MSG_TYPE_TABLE_META; - } - - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_MULTI_META; - - int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList); - int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList); - int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0; - - int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) { - tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; - pInfo->metaClone = metaClone? 1:0; - pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); - pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); - pInfo->numOfUdfs = htonl(numOfUdf); - - char* start = pInfo->tableNames; - int32_t len = 0; - for(int32_t i = 0; i < numOfTable; ++i) { - char* name = taosArrayGetP(pNameList, i); - if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) { - len = sprintf(start, "%s,", name); - } else { - len = sprintf(start, "%s", name); - } - - start += len; - } - - for(int32_t i = 0; i < numOfVgroupList; ++i) { - char* name = taosArrayGetP(pVgroupNameList, i); - if (i < numOfVgroupList - 1 || numOfUdf > 0) { - len = sprintf(start, "%s,", name); - } else { - len = sprintf(start, "%s", name); - } - - start += len; - } - - for(int32_t i = 0; i < numOfUdf; ++i) { - SUdfInfo * u = taosArrayGet(pUdfList, i); - if (i < numOfUdf - 1) { - len = sprintf(start, "%s,", u->name); - } else { - len = sprintf(start, "%s", u->name); - } - - start += len; - } - - pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); - pNew->cmd.msgType = TSDB_MSG_TYPE_TABLES_META; - - registerSqlObj(pNew); - tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self, - pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen); - - pNew->fp = fp; - pNew->param = (void *)pSql->self; - - tscDebug("0x%"PRIx64" metaRid from 0x%" PRIx64 " to 0x%" PRIx64 , pSql->self, pSql->metaRid, pNew->self); - - pSql->metaRid = pNew->self; - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { - assert(tNameIsValid(&pTableMetaInfo->name)); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - size_t len = strlen(name); - // just make runtime happy - if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) { - memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); - } - - if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity)) { - tfree(pTableMetaInfo->pTableMeta); - } - - STableMeta* pMeta = pTableMetaInfo->pTableMeta; - STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); - - if (pMeta && pMeta->id.uid > 0) { - // in case of child table, here only get the - if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta)); - pSql->pBuf = (void *)(pSTMeta); - if (code != TSDB_CODE_SUCCESS) { - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); - } - } - - tscDebug("0x%"PRIx64 " %s retrieve tableMeta from cache, numOfCols:%d, numOfTags:%d", pSql->self, name, pMeta->tableInfo.numOfColumns, pMeta->tableInfo.numOfTags); - return TSDB_CODE_SUCCESS; - } - - if (onlyLocal) { - return TSDB_CODE_TSC_NO_META_CACHED; - } - - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); -} - -int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { - return tscGetTableMetaImpl(pSql, pTableMetaInfo, false, false); -} - -int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal) { - return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists, onlyLocal); -} - -int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC; - - if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) { - tscError("%p malloc failed for new queryinfo", pSql); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - - pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo)); - for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { - SUdfInfo info = {0}; - SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i); - info = *p1; - info.name = strdup(p1->name); - taosArrayPush(pNewQueryInfo->pUdfInfo, &info); - } - - pNew->cmd.active = pNewQueryInfo; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { - tscError("%p malloc failed for payload to get table meta", pSql); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew); - registerSqlObj(pNew); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - - tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self); - - pSql->metaRid = pNew->self; - - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (pTableMetaInfo->vgroupList == NULL) { - return false; - } - } - - // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore - return true; -} - -#endif - int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; pMsgBody->msgInfo.len = sizeof(SConnectMsg); From 7246eabb70316376864c6e2290cae93f9d157230 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Dec 2021 10:42:26 +0800 Subject: [PATCH 10/14] [td-11818]Remove unused code. --- source/client/src/clientMsgHandler.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 00acefb397..2f996c6ced 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -14,6 +14,7 @@ */ #include "os.h" +#include "tdef.h" #include "tname.h" #include "clientInt.h" #include "clientLog.h" From bc48abe82c194e7b528ff2cc7b51ace0aaa86598 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Dec 2021 11:34:12 +0800 Subject: [PATCH 11/14] [td-11818] Add files. --- source/client/src/clientEnv.c | 466 +++++++++++++++++++++++++++ source/libs/catalog/src/catalog.c | 2 +- source/libs/qcom/test/CMakeLists.txt | 19 ++ source/libs/qcom/test/queryTest.cpp | 83 +++++ 4 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 source/client/src/clientEnv.c create mode 100644 source/libs/qcom/test/CMakeLists.txt create mode 100644 source/libs/qcom/test/queryTest.cpp diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c new file mode 100644 index 0000000000..b7a622c2e8 --- /dev/null +++ b/source/client/src/clientEnv.c @@ -0,0 +1,466 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosmsg.h" +#include "query.h" +#include "clientInt.h" +#include "clientLog.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "trpc.h" +#include "ttime.h" +#include "ttimezone.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +SAppInfo appInfo; +int32_t tscReqRef = -1; +int32_t tscConnRef = -1; + +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; +volatile int32_t tscInitRes = 0; + +static void registerRequest(SRequestObj* pRequest) { + STscObj *pTscObj = (STscObj *)taosAcquireRef(tscConnRef, pRequest->pTscObj->id); + assert(pTscObj != NULL); + + // connection has been released already, abort creating request. + pRequest->self = taosAddRef(tscReqRef, pRequest); + + int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1); + + if (pTscObj->pAppInfo) { + SInstanceActivity *pActivity = &pTscObj->pAppInfo->summary; + + int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1); + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d", pRequest->self, + pRequest->pTscObj->id, num, currentInst, total); + } +} + +static void deregisterRequest(SRequestObj* pRequest) { + assert(pRequest != NULL); + + STscObj* pTscObj = pRequest->pTscObj; + SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; + + int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + + tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); + taosReleaseRef(tscConnRef, pTscObj->id); +} + +static void tscInitLogFile() { + taosReadGlobalLogCfg(); + if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { + printf("failed to create log dir:%s\n", tsLogDir); + } + + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + +void closeTransporter(STscObj* pTscObj) { + if (pTscObj == NULL || pTscObj->pTransporter == NULL) { + return; + } + + tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pTransporter, pTscObj->id); + rpcClose(pTscObj->pTransporter); + pTscObj->pTransporter = NULL; +} + +// TODO refactor +void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "TSC"; + rpcInit.numOfThreads = numOfThread; + rpcInit.cfp = processMsgFromServer; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char *)user; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; +// rpcInit.spi = 1; + rpcInit.secret = (char *)auth; + + void* pDnodeConn = rpcOpen(&rpcInit); + if (pDnodeConn == NULL) { + tscError("failed to init connection to server"); + return NULL; + } + + return pDnodeConn; +} + +void destroyTscObj(void *pObj) { + STscObj *pTscObj = pObj; + + atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); + pthread_mutex_destroy(&pTscObj->mutex); + tfree(pTscObj); +} + +void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo) { + STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); + if (NULL == pObj) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + pObj->pAppInfo = pAppInfo; + if (pAppInfo != NULL) { + pObj->pTransporter = pAppInfo->pTransporter; + } + + tstrncpy(pObj->user, user, sizeof(pObj->user)); + memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); + + pthread_mutex_init(&pObj->mutex, NULL); + pObj->id = taosAddRef(tscConnRef, pObj); + + tscDebug("connObj created, 0x%"PRIx64, pObj->id); + return pObj; +} + +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { + assert(pObj != NULL); + + SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); + if (NULL == pRequest) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + // TODO generated request uuid + pRequest->requestId = 0; + + pRequest->metric.start = taosGetTimestampMs(); + + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; +// pRequest->body.requestMsg. = param; + pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + tsem_init(&pRequest->body.rspSem, 0, 0); + + registerRequest(pRequest); + return pRequest; +} + +static void doDestroyRequest(void* p) { + assert(p != NULL); + SRequestObj* pRequest = (SRequestObj*)p; + + assert(RID_VALID(pRequest->self)); + + tfree(pRequest->msgBuf); + tfree(pRequest->sqlstr); + tfree(pRequest->pInfo); + + tfree(pRequest->body.resInfo.pRspMsg); + + deregisterRequest(pRequest); + tfree(pRequest); +} + +void destroyRequest(SRequestObj* pRequest) { + if (pRequest == NULL) { + return; + } + + taosReleaseRef(tscReqRef, pRequest->self); +} + +void taos_init_imp(void) { + // In the APIs of other program language, taos_cleanup is not available yet. + // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. + atexit(taos_cleanup); + + errno = TSDB_CODE_SUCCESS; + srand(taosGetTimestampSec()); + + deltaToUtcInitOnce(); + taosInitGlobalCfg(); + taosReadCfgFromFile(); + + tscInitLogFile(); + if (taosCheckAndPrintCfg()) { + tscInitRes = -1; + return; + } + + taosInitNotes(); + initMsgHandleFp(); + + rpcInit(); + + tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); + + taosSetCoreDump(true); + + initTaskQueue(); + + tscConnRef = taosOpenRef(200, destroyTscObj); + tscReqRef = taosOpenRef(40960, doDestroyRequest); + + taosGetAppName(appInfo.appName, NULL); + appInfo.pid = taosGetPId(); + appInfo.startTime = taosGetTimestampMs(); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + tscDebug("client is initialized successfully"); +} + +int taos_init() { + pthread_once(&tscinit, taos_init_imp); + return tscInitRes; +} + +int taos_options_imp(TSDB_OPTION option, const char *str) { + SGlobalCfg *cfg = NULL; + + switch (option) { + case TSDB_OPTION_CONFIGDIR: + cfg = taosGetConfigOption("configDir"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(configDir, str, TSDB_FILENAME_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set config file directory:%s", str); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + case TSDB_OPTION_SHELL_ACTIVITY_TIMER: + cfg = taosGetConfigOption("shellActivityTimer"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tsShellActivityTimer = atoi(str); + if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; + if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + } + break; + + case TSDB_OPTION_LOCALE: { // set locale + cfg = taosGetConfigOption("locale"); + assert(cfg != NULL); + + size_t len = strlen(str); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("Invalid locale:%s, use default", str); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + char sep = '.'; + + if (strlen(tsLocale) == 0) { // locale does not set yet + char* defaultLocale = setlocale(LC_CTYPE, ""); + + // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. + // The launch of current system will abort soon. + if (defaultLocale == NULL) { + tscError("failed to get default locale, please set the correct locale in current OS"); + return -1; + } + + tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); + } + + // set the user specified locale + char *locale = setlocale(LC_CTYPE, str); + + if (locale != NULL) { // failed to set the user specified locale + tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { // set the user specified locale failed, use default LC_CTYPE as current locale + locale = setlocale(LC_CTYPE, tsLocale); + tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); + } + + tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + + char *charset = strrchr(tsLocale, sep); + if (charset != NULL) { + charset += 1; + + charset = taosCharsetReplace(charset); + + if (taosValidateEncodec(charset)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset set:%s", charset); + } else { + tscInfo("charset changed from %s to %s", tsCharset, charset); + } + + tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + + } else { + tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); + } + + free(charset); + } else { // it may be windows system + tscInfo("charset remains:%s", tsCharset); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + } + + case TSDB_OPTION_CHARSET: { + /* set charset will override the value of charset, assigned during system locale changed */ + cfg = taosGetConfigOption("charset"); + assert(cfg != NULL); + + size_t len = strlen(str); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("failed to set charset:%s", str); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + if (taosValidateEncodec(str)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset is set:%s", str); + } else { + tscInfo("charset changed from %s to %s", tsCharset, str); + } + + tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { + tscInfo("charset:%s not valid", str); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + + break; + } + + case TSDB_OPTION_TIMEZONE: + cfg = taosGetConfigOption("timezone"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); + tsSetTimeZone(); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + default: + // TODO return the correct error code to client in the format for taos_errstr() + tscError("Invalid option %d", option); + return -1; + } + + return 0; +} + +#if 0 +#include "cJSON.h" +static setConfRet taos_set_config_imp(const char *config){ + setConfRet ret = {SET_CONF_RET_SUCC, {0}}; + static bool setConfFlag = false; + if (setConfFlag) { + ret.retCode = SET_CONF_RET_ERR_ONLY_ONCE; + strcpy(ret.retMsg, "configuration can only set once"); + return ret; + } + taosInitGlobalCfg(); + cJSON *root = cJSON_Parse(config); + if (root == NULL){ + ret.retCode = SET_CONF_RET_ERR_JSON_PARSE; + strcpy(ret.retMsg, "parse json error"); + return ret; + } + + int size = cJSON_GetArraySize(root); + if(!cJSON_IsObject(root) || size == 0) { + ret.retCode = SET_CONF_RET_ERR_JSON_INVALID; + strcpy(ret.retMsg, "json content is invalid, must be not empty object"); + return ret; + } + + if(size >= 1000) { + ret.retCode = SET_CONF_RET_ERR_TOO_LONG; + strcpy(ret.retMsg, "json object size is too long"); + return ret; + } + + for(int i = 0; i < size; i++){ + cJSON *item = cJSON_GetArrayItem(root, i); + if(!item) { + ret.retCode = SET_CONF_RET_ERR_INNER; + strcpy(ret.retMsg, "inner error"); + return ret; + } + if(!taosReadConfigOption(item->string, item->valuestring, NULL, NULL, TAOS_CFG_CSTATUS_OPTION, TSDB_CFG_CTYPE_B_CLIENT)){ + ret.retCode = SET_CONF_RET_ERR_PART; + if (strlen(ret.retMsg) == 0){ + snprintf(ret.retMsg, RET_MSG_LENGTH, "part error|%s", item->string); + }else{ + int tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + size_t leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, "|", leftSize); + tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, item->string, leftSize); + } + } + } + cJSON_Delete(root); + setConfFlag = true; + return ret; +} + +setConfRet taos_set_config(const char *config){ + pthread_mutex_lock(&setConfMutex); + setConfRet ret = taos_set_config_imp(config); + pthread_mutex_unlock(&setConfMutex); + return ret; +} +#endif \ No newline at end of file diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1134c7763a..e3e8485760 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -605,7 +605,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* p SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - snprintf(dbName, sizeof(dbName), "%s.%s", name->acctId, name->dbname); + snprintf(dbName, sizeof(dbName), "%d.%s", name->acctId, name->dbname); CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, dbName, name->tname, &pTableMeta)); diff --git a/source/libs/qcom/test/CMakeLists.txt b/source/libs/qcom/test/CMakeLists.txt new file mode 100644 index 0000000000..7adec3752a --- /dev/null +++ b/source/libs/qcom/test/CMakeLists.txt @@ -0,0 +1,19 @@ + +MESSAGE(STATUS "build qcom unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(queryUtilTest ${SOURCE_LIST}) + +TARGET_INCLUDE_DIRECTORIES( + queryUtilTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qcom/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/qcom/inc" +) + +TARGET_LINK_LIBRARIES( + queryUtilTest + PUBLIC os util gtest qcom common +) diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp new file mode 100644 index 0000000000..d2aedff8fd --- /dev/null +++ b/source/libs/qcom/test/queryTest.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taosmsg.h" +#include "query.h" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +namespace { +typedef struct SParam { + int32_t v; +} SParam; +int32_t testPrint(void* p) { + SParam* param = (SParam*)p; + printf("hello world, %d\n", param->v); + tfree(p); + return 0; +} + +int32_t testPrintError(void* p) { + SParam* param = (SParam*) p; + tfree(p); + + return -1; +} +} // namespace + +class QueryTestEnv : public testing::Environment { + public: + virtual void SetUp() { initTaskQueue(); } + + virtual void TearDown() { cleanupTaskQueue(); } + + QueryTestEnv() {} + virtual ~QueryTestEnv() {} +}; + +int main(int argc, char** argv) { + testing::AddGlobalTestEnvironment(new QueryTestEnv()); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, async_task_test) { + SParam* p = (SParam*)calloc(1, sizeof(SParam)); + taosAsyncExec(testPrint, p, NULL); + usleep(5000); +} + +TEST(testCase, many_async_task_test) { + for(int32_t i = 0; i < 50; ++i) { + SParam* p = (SParam*) calloc(1, sizeof(SParam)); + p->v = i; + taosAsyncExec(testPrint, p, NULL); + } + + usleep(10000); +} + +TEST(testCase, error_in_async_test) { + int32_t code = 0; + SParam* p = (SParam*) calloc(1, sizeof(SParam)); + taosAsyncExec(testPrintError, p, &code); + usleep(1000); + printf("Error code:%d after asynchronously exec function\n", code); +} \ No newline at end of file From 4b25449576ac0d0c182e5f1b57ba2238f9233b5e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 23 Dec 2021 13:25:55 +0800 Subject: [PATCH 12/14] tq consume refactor --- include/dnode/vnode/tq/tq.h | 65 +++++++++++++++++----------- source/dnode/vnode/tq/src/tq.c | 78 ++++++++++++++++++++-------------- 2 files changed, 87 insertions(+), 56 deletions(-) diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 5eeaaa1011..f5d5cc9a16 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -23,6 +23,7 @@ #include "taosmsg.h" #include "tlist.h" #include "trpc.h" +#include "ttimer.h" #include "tutil.h" #ifdef __cplusplus @@ -103,7 +104,7 @@ typedef struct STqTopicVhandle { typedef struct STqExec { void* runtimeEnv; SSDataBlock* (*exec)(void* runtimeEnv); - void* (*assign)(void* runtimeEnv, SSubmitBlk* inputData); + void* (*assign)(void* runtimeEnv, void* inputData); void (*clear)(void* runtimeEnv); char* (*serialize)(struct STqExec*); struct STqExec* (*deserialize)(char*); @@ -114,33 +115,33 @@ typedef struct STqRspHandle { void* ahandle; } STqRspHandle; -typedef enum { - TQ_ITEM_READY, - TQ_ITEM_PROCESS, - TQ_ITEM_EMPTY -} STqItemStatus; +typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; + +typedef struct STqTopic STqTopic; typedef struct STqBufferItem { int64_t offset; // executors are identical but not concurrent // so there must be a copy in each item - STqExec* executor; - int32_t status; - int64_t size; - void* content; + STqExec* executor; + int32_t status; + int64_t size; + void* content; + STqTopic* pTopic; } STqMsgItem; -typedef struct STqTopic { +struct STqTopic { // char* topic; //c style, end with '\0' // int64_t cgId; // void* ahandle; + // int32_t head; + // int32_t tail; int64_t nextConsumeOffset; int64_t floatingCursor; int64_t topicId; - int32_t head; - int32_t tail; + void* logReader; STqMsgItem buffer[TQ_BUFFER_SIZE]; -} STqTopic; +}; typedef struct STqListHandle { STqTopic topic; @@ -148,11 +149,11 @@ typedef struct STqListHandle { } STqList; typedef struct STqGroup { - int64_t clientId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - //STqList* head; + int64_t clientId; + int64_t cgId; + void* ahandle; + int32_t topicNum; + STqList* head; SList* topicList; // SList STqRspHandle rspHandle; } STqGroup; @@ -162,20 +163,23 @@ typedef struct STqQueryMsg { struct STqQueryMsg* next; } STqQueryMsg; -typedef struct STqLogReader { +typedef struct STqLogHandle { void* logHandle; - int32_t (*logRead)(void* logHandle, void** data, int64_t ver); + void* (*openLogReader)(void* logHandle); + void (*closeLogReader)(void* logReader); + int32_t (*logRead)(void* logReader, void** data, int64_t ver); + int64_t (*logGetFirstVer)(void* logHandle); int64_t (*logGetSnapshotVer)(void* logHandle); int64_t (*logGetLastVer)(void* logHandle); -} STqLogReader; +} STqLogHandle; typedef struct STqCfg { // TODO } STqCfg; typedef struct STqMemRef { - SMemAllocatorFactory* pAlloctorFactory; + SMemAllocatorFactory* pAllocatorFactory; SMemAllocator* pAllocator; } STqMemRef; @@ -265,13 +269,24 @@ typedef struct STQ { // the handle of meta kvstore char* path; STqCfg* tqConfig; - STqLogReader* tqLogReader; + STqLogHandle* tqLogHandle; STqMemRef tqMemRef; STqMetaStore* tqMeta; } STQ; +typedef struct STqMgmt { + int8_t inited; + tmr_h timer; +} STqMgmt; + +static STqMgmt tqMgmt; + +// init once +int tqInit(); +void tqCleanUp(); + // open in each vnode -STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemAllocatorFactory* allocFac); void tqClose(STQ*); // void* will be replace by a msg type diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 972740183d..9e5aaf61fc 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -37,7 +37,22 @@ void* tqSerializeItem(STqMsgItem* pItem, void* ptr); const void* tqDeserializeTopic(const void* pBytes, STqTopic* pTopic); const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem); -STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) { +int tqInit() { + int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); + if(old == 1) return 0; + + tqMgmt.timer = taosTmrInit(0, 0, 0, "TQ"); + return 0; +} + +void tqCleanUp() { + int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 0); + if(old == 0) return; + taosTmrStop(tqMgmt.timer); + taosTmrCleanUp(tqMgmt.timer); +} + +STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -45,8 +60,8 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemA } pTq->path = strdup(path); pTq->tqConfig = tqConfig; - pTq->tqLogReader = tqLogReader; - pTq->tqMemRef.pAlloctorFactory = allocFac; + pTq->tqLogHandle = tqLogHandle; + pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if (pTq->tqMemRef.pAllocator == NULL) { // TODO: error code of buffer pool @@ -360,9 +375,9 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { (*pRsp)->pCont = ptr; break; } - *((int64_t*)buffer) = htonll(pTopic->topicId); + *((int64_t*)buffer) = htobe64(pTopic->topicId); buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = htonll(pTopic->buffer[idx].size); + *((int64_t*)buffer) = htobe64(pTopic->buffer[idx].size); buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); memcpy(buffer, pTopic->buffer[idx].content, pTopic->buffer[idx].size); buffer = POINTER_SHIFT(buffer, pTopic->buffer[idx].size); @@ -384,41 +399,42 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { } else { ASSERT(0); } - } } + if (numOfMsgs > 0) { + // set code and other msg + rpcSendResponse(*pRsp); + } else { + // most recent data has been fetched + + // enable timer for blocking wait + // once new data written when waiting, launch query and rsp + } + + // fetched a num of msgs, rpc response for(int i = 0; i < pArray->size; i++) { STqMsgItem* pItem = taosArrayGet(pArray, i); - void* raw; //read from wal + void* raw = NULL; + /*int code = pTq->tqLogReader->logRead(, &raw, pItem->offset);*/ + int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset); + if(code < 0) { + //TODO: error + } //get msgType //if submitblk pItem->executor->assign(pItem->executor->runtimeEnv, raw); SSDataBlock* content = pItem->executor->exec(pItem->executor->runtimeEnv); pItem->content = content; //if other type, send just put into buffer - pItem->content = raw; + /*pItem->content = raw;*/ int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_PROCESS, TQ_ITEM_READY); ASSERT(old == TQ_ITEM_PROCESS); - } - - if (numOfMsgs < 0) { - return -1; - } - - if (numOfMsgs == 0) { - // most recent data has been fetched - - // enable timer for blocking wait - // once new data written when waiting, launch query and rsp - return -1; - } - - // fetched a num of msgs, rpc response + taosArrayDestroy(pArray); return 0; } @@ -500,10 +516,10 @@ void* tqSerializeTopic(STqTopic* pTopic, void* ptr) { ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); *(int64_t*)ptr = pTopic->topicId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = pTopic->head; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - *(int32_t*)ptr = pTopic->tail; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); + /**(int32_t*)ptr = pTopic->head;*/ + /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ + /**(int32_t*)ptr = pTopic->tail;*/ + /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ for (int i = 0; i < TQ_BUFFER_SIZE; i++) { ptr = tqSerializeItem(&pTopic->buffer[i], ptr); } @@ -557,10 +573,10 @@ const void* tqDeserializeTopic(const void* pBytes, STqTopic* topic) { ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); topic->topicId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - topic->head = *(int32_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - topic->tail = *(int32_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); + /*topic->head = *(int32_t*)ptr;*/ + /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ + /*topic->tail = *(int32_t*)ptr;*/ + /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ for (int i = 0; i < TQ_BUFFER_SIZE; i++) { ptr = tqDeserializeItem(ptr, &topic->buffer[i]); } From b2574fa2f4831950d3f19719a86b933566116f13 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 23 Dec 2021 13:35:22 +0800 Subject: [PATCH 13/14] fix compile error --- source/dnode/vnode/tq/src/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 4529e868e1..5ceb062bf2 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -71,7 +71,7 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemA free(pTq); #if 0 allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); -#endi +#endif return NULL; } From fa861e8d838ce82ca47b906f57512c9a60127dba Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 23 Dec 2021 00:50:20 -0500 Subject: [PATCH 14/14] TD-12451 insert physical plan serialization --- include/libs/planner/planner.h | 6 +- source/libs/planner/inc/plannerInt.h | 5 +- source/libs/planner/src/physicalPlan.c | 24 ++++++- source/libs/planner/src/physicalPlanJson.c | 82 +++++++++++++++++++++- source/libs/planner/src/planner.c | 4 +- source/libs/planner/test/phyPlanTests.cpp | 3 +- source/libs/scheduler/src/scheduler.c | 4 +- 7 files changed, 117 insertions(+), 11 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 153cf0bb3e..d4469be5e3 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -119,9 +119,9 @@ typedef struct SSubplanId { typedef struct SSubplan { SSubplanId id; // unique id of the subplan - int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN + int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN|QUERY_TYPE_MODIFY int32_t level; // the execution level of current subplan, starting from 0. - SEpSet execEpSet; // for the scan sub plan, the optional execution node + SEpSet execEpSet; // for the scan/modify subplan, the optional execution node SArray *pChildern; // the datasource subplan,from which to fetch the result SArray *pParents; // the data destination subplan, get data from current subplan SPhyNode *pNode; // physical plan of current subplan @@ -152,7 +152,7 @@ int32_t qExplainQuery(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode /** * Convert to subplan to string for the scheduler to send to the executor */ -int32_t qSubPlanToString(const SSubplan* subplan, char** str); +int32_t qSubPlanToString(const SSubplan* subplan, char** str, int32_t* len); int32_t qStringToSubplan(const char* str, SSubplan** subplan); diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 1bee95b8e5..ed29839905 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -102,7 +102,7 @@ int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag); int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep); -int32_t subPlanToString(const SSubplan *pPhyNode, char** str); +int32_t subPlanToString(const SSubplan *pPhyNode, char** str, int32_t* len); int32_t stringToSubplan(const char* str, SSubplan** subplan); /** @@ -121,6 +121,9 @@ void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode); const char* opTypeToOpName(int32_t type); int32_t opNameToOpType(const char* name); +const char* dsinkTypeToDsinkName(int32_t type); +int32_t dsinkNameToDsinkType(const char* name); + #ifdef __cplusplus } #endif diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 9db9e059b3..8388458b4c 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -47,16 +47,38 @@ const char* opTypeToOpName(int32_t type) { int32_t opNameToOpType(const char* name) { for (int32_t i = 1; i < sizeof(gOpName) / sizeof(gOpName[0]); ++i) { - if (strcmp(name, gOpName[i])) { + if (0 == strcmp(name, gOpName[i])) { return i; } } return OP_Unknown; } +const char* dsinkTypeToDsinkName(int32_t type) { + switch (type) { + case DSINK_Dispatch: + return "Dispatch"; + case DSINK_Insert: + return "Insert"; + default: + break; + } + return "Unknown"; +} + +int32_t dsinkNameToDsinkType(const char* name) { + if (0 == strcmp(name, "Dispatch")) { + return DSINK_Dispatch; + } else if (0 == strcmp(name, "Insert")) { + return DSINK_Insert; + } + return DSINK_Unknown; +} + static SDataSink* initDataSink(int32_t type, int32_t size) { SDataSink* sink = (SDataSink*)vailidPointer(calloc(1, size)); sink->info.type = type; + sink->info.name = dsinkTypeToDsinkName(type); return sink; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 15c0e632a7..bf052f34b4 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -695,6 +695,70 @@ static bool phyNodeFromJson(const cJSON* json, void* obj) { return res; } +static const char* jkInserterNumOfTables = "NumOfTables"; +static const char* jkInserterDataSize = "DataSize"; + +static bool inserterToJson(const void* obj, cJSON* json) { + const SDataInserter* inserter = (const SDataInserter*)obj; + bool res = cJSON_AddNumberToObject(json, jkInserterNumOfTables, inserter->numOfTables); + if (res) { + res = cJSON_AddNumberToObject(json, jkInserterDataSize, inserter->size); + } + // todo pData + return res; +} + +static bool inserterFromJson(const cJSON* json, void* obj) { + SDataInserter* inserter = (SDataInserter*)obj; + inserter->numOfTables = getNumber(json, jkInserterNumOfTables); + inserter->size = getNumber(json, jkInserterDataSize); + // todo pData +} + +static bool specificDataSinkToJson(const void* obj, cJSON* json) { + const SDataSink* dsink = (const SDataSink*)obj; + switch (dsink->info.type) { + case DSINK_Dispatch: + return true; + case DSINK_Insert: + return inserterToJson(obj, json); + default: + break; + } + return false; +} + +static bool specificDataSinkFromJson(const cJSON* json, void* obj) { + SDataSink* dsink = (SDataSink*)obj; + switch (dsink->info.type) { + case DSINK_Dispatch: + return true; + case DSINK_Insert: + return inserterFromJson(json, obj); + default: + break; + } + return false; +} + +static const char* jkDataSinkName = "Name"; + +static bool dataSinkToJson(const void* obj, cJSON* json) { + const SDataSink* dsink = (const SDataSink*)obj; + bool res = cJSON_AddStringToObject(json, jkDataSinkName, dsink->info.name); + if (res) { + res = addObject(json, dsink->info.name, specificDataSinkToJson, dsink); + } + return res; +} + +static bool dataSinkFromJson(const cJSON* json, void* obj) { + SDataSink* dsink = (SDataSink*)obj; + dsink->info.name = getString(json, jkDataSinkName); + dsink->info.type = dsinkNameToDsinkType(dsink->info.name); + return fromObject(json, dsink->info.name, specificDataSinkFromJson, dsink, true); +} + static const char* jkIdQueryId = "QueryId"; static const char* jkIdTemplateId = "TemplateId"; static const char* jkIdSubplanId = "SubplanId"; @@ -721,6 +785,7 @@ static bool subplanIdFromJson(const cJSON* json, void* obj) { static const char* jkSubplanId = "Id"; static const char* jkSubplanNode = "Node"; +static const char* jkSubplanDataSink = "DataSink"; static cJSON* subplanToJson(const SSubplan* subplan) { cJSON* jSubplan = cJSON_CreateObject(); @@ -734,6 +799,9 @@ static cJSON* subplanToJson(const SSubplan* subplan) { if (res) { res = addObject(jSubplan, jkSubplanNode, phyNodeToJson, subplan->pNode); } + if (res) { + res = addObject(jSubplan, jkSubplanDataSink, dataSinkToJson, subplan->pDataSink); + } if (!res) { cJSON_Delete(jSubplan); @@ -751,6 +819,9 @@ static SSubplan* subplanFromJson(const cJSON* json) { if (res) { res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, sizeof(SPhyNode), false); } + if (res) { + res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false); + } if (!res) { qDestroySubplan(subplan); @@ -759,13 +830,22 @@ static SSubplan* subplanFromJson(const cJSON* json) { return subplan; } -int32_t subPlanToString(const SSubplan* subplan, char** str) { +int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { + if (QUERY_TYPE_MODIFY == subplan->type) { + SDataInserter* insert = (SDataInserter*)(subplan->pDataSink); + *len = insert->size; + *str = insert->pData; + insert->pData == NULL; + return TSDB_CODE_SUCCESS; + } + cJSON* json = subplanToJson(subplan); if (NULL == json) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } *str = cJSON_Print(json); + *len = strlen(*str); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 7722a7d363..e8523249e4 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -50,8 +50,8 @@ int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr return setSubplanExecutionNode(subplan, templateId, ep); } -int32_t qSubPlanToString(const SSubplan *subplan, char** str) { - return subPlanToString(subplan, str); +int32_t qSubPlanToString(const SSubplan *subplan, char** str, int32_t* len) { + return subPlanToString(subplan, str, len); } int32_t qStringToSubplan(const char* str, SSubplan** subplan) { diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 3be3337304..ddd1151742 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -62,8 +62,9 @@ protected: size_t num = taosArrayGetSize(subplans); for (size_t j = 0; j < num; ++j) { std::cout << "no " << j << ":" << std::endl; + int32_t len = 0; char* str = nullptr; - ASSERT_EQ (TSDB_CODE_SUCCESS, qSubPlanToString((const SSubplan*)taosArrayGetP(subplans, j), &str)); + ASSERT_EQ (TSDB_CODE_SUCCESS, qSubPlanToString((const SSubplan*)taosArrayGetP(subplans, j), &str, &len)); std::cout << str << std::endl; free(str); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 99a9b06fe4..5671a0d747 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -521,8 +521,8 @@ _return: int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { SSubplan *plan = task->plan; - - SCH_ERR_RET(qSubPlanToString(plan, &task->msg)); + int32_t len = 0; + SCH_ERR_RET(qSubPlanToString(plan, &task->msg, &len)); if (plan->execEpSet.numOfEps <= 0) { SCH_ERR_RET(schSetTaskExecEpSet(job, &plan->execEpSet)); }