diff --git a/example/src/tmq.c b/example/src/tmq.c index 5b8f66f666..abd4f78610 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -14,7 +14,9 @@ */ #include +#include #include +#include #include #include #include "taos.h" @@ -24,7 +26,7 @@ static void msg_process(TAOS_RES* msg) { char buf[1024]; memset(buf, 0, 1024); printf("topic: %s\n", tmq_get_topic_name(msg)); - printf("vg:%d\n", tmq_get_vgroup_id(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); while (1) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; @@ -140,8 +142,8 @@ int32_t create_topic() { return 0; } -void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) { - printf("commit %d\n", resp); +void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { + printf("commit %d tmq %p offsets %p param %p\n", resp, tmq, offsets, param); } tmq_t* build_consumer() { @@ -161,7 +163,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); /*tmq_conf_set(conf, "td.connect.db", "abc1");*/ - tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); return tmq; @@ -215,12 +217,24 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { return; } + tmq_list_t* subList = NULL; + tmq_subscription(tmq, &subList); + char** subTopics = tmq_list_to_c_array(subList); + int32_t sz = tmq_list_get_size(subList); + printf("subscribed topics: "); + for (int32_t i = 0; i < sz; i++) { + printf("%s, ", subTopics[i]); + } + printf("\n"); + tmq_list_destroy(subList); + while (running) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); if (tmqmessage) { msg_process(tmqmessage); taos_free_result(tmqmessage); + tmq_commit(tmq, NULL, 1); /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } } diff --git a/include/client/taos.h b/include/client/taos.h index 19c008bb09..6e20900668 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -213,7 +213,7 @@ typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; typedef struct tmq_conf_t tmq_conf_t; typedef struct tmq_list_t tmq_list_t; -typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *)); +typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); @@ -253,12 +253,12 @@ typedef enum tmq_conf_res_t tmq_conf_res_t; DLL_EXPORT tmq_conf_t *tmq_conf_new(); DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); -DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); +DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param); /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ -DLL_EXPORT char *tmq_get_topic_name(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); +DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); // TODO #if 0 DLL_EXPORT char *tmq_get_table_name(TAOS_RES *res); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5de490d6c6..3d6bb06015 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1524,6 +1524,7 @@ typedef struct { int32_t tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam); int32_t tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam); +// TDMT_VND_CREATE_STB ============== typedef struct SVCreateStbReq { const char* name; tb_uid_t suid; @@ -1536,17 +1537,14 @@ typedef struct SVCreateStbReq { int tEncodeSVCreateStbReq(SCoder* pCoder, const SVCreateStbReq* pReq); int tDecodeSVCreateStbReq(SCoder* pCoder, SVCreateStbReq* pReq); +// TDMT_VND_DROP_STB ============== typedef struct SVDropStbReq { - // data -#ifdef WINDOWS - size_t avoidCompilationErrors; -#endif - + const char* name; + tb_uid_t suid; } SVDropStbReq; -typedef struct SVCreateStbRsp { - int code; -} SVCreateStbRsp; +int32_t tEncodeSVDropStbReq(SCoder* pCoder, const SVDropStbReq* pReq); +int32_t tDecodeSVDropStbReq(SCoder* pCoder, SVDropStbReq* pReq); typedef struct SVCreateTbReq { tb_uid_t uid; @@ -1603,19 +1601,37 @@ int tDecodeSVCreateTbBatchRsp(SCoder* pCoder, SVCreateTbBatchRsp* pRsp); int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +// TDMT_VND_DROP_TABLE ================= typedef struct { - int64_t ver; - char* name; - uint8_t type; - tb_uid_t suid; + const char* name; + int8_t igNotExists; } SVDropTbReq; typedef struct { - int tmp; // TODO: to avoid compile error + int32_t code; } SVDropTbRsp; -int32_t tSerializeSVDropTbReq(void** buf, SVDropTbReq* pReq); -void* tDeserializeSVDropTbReq(void* buf, SVDropTbReq* pReq); +typedef struct { + int32_t nReqs; + union { + SVDropTbReq* pReqs; + SArray* pArray; + }; +} SVDropTbBatchReq; + +int32_t tEncodeSVDropTbBatchReq(SCoder* pCoder, const SVDropTbBatchReq* pReq); +int32_t tDecodeSVDropTbBatchReq(SCoder* pCoder, SVDropTbBatchReq* pReq); + +typedef struct { + int32_t nRsps; + union { + SVDropTbRsp* pRsps; + SArray* pArray; + }; +} SVDropTbBatchRsp; + +int32_t tEncodeSVDropTbBatchRsp(SCoder* pCoder, const SVDropTbBatchRsp* pRsp); +int32_t tDecodeSVDropTbBatchRsp(SCoder* pCoder, SVDropTbBatchRsp* pRsp); typedef struct { SMsgHead head; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index e1ee946ce4..b0439c722c 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -170,9 +170,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_UPDATE_TAG_VAL, "vnode-update-tag-val", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLE_META, "vnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLES_META, "vnode-tables-meta", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateTbReq, SVCreateTbRsp) + TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_STB, "vnode-alter-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONSUME, "vnode-mq-consume", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 6f77e8edb6..6513a44e88 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -22,19 +22,20 @@ extern "C" { #include "tdef.h" -#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) +#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) #define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) #define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) -#define FOREACH(node, list) \ - for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) +#define FOREACH(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ + (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) #define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode) #define INSERT_LIST(target, src) nodesListInsertList((target), cell, src) -#define WHERE_EACH(node, list) \ +#define WHERE_EACH(node, list) \ SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ while (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)) @@ -43,16 +44,26 @@ extern "C" { // only be use in WHERE_EACH #define ERASE_NODE(list) cell = nodesListErase((list), cell) -#define FORBOTH(node1, list1, node2, list2) \ - for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ - (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ - cell1 = cell1->pNext, cell2 = cell2->pNext) +#define FORBOTH(node1, list1, node2, list2) \ + for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), \ + *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ + (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), \ + (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), \ + (node1 != NULL && node2 != NULL); \ + cell1 = cell1->pNext, cell2 = cell2->pNext) #define REPLACE_LIST1_NODE(newNode) cell1->pNode = (SNode*)(newNode) #define REPLACE_LIST2_NODE(newNode) cell2->pNode = (SNode*)(newNode) -#define FOREACH_FOR_REWRITE(node, list) \ - for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) +#define FOREACH_FOR_REWRITE(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ + (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) + +#define DESTORY_LIST(list) \ + do { \ + nodesDestroyList(list); \ + list = NULL; \ + } while (0) typedef enum ENodeType { // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN, @@ -202,7 +213,7 @@ typedef enum ENodeType { /** * The first field of a node of any type is guaranteed to be the ENodeType. - * Hence the type of any node can be gotten by casting it to SNode. + * Hence the type of any node can be gotten by casting it to SNode. */ typedef struct SNode { ENodeType type; @@ -211,41 +222,36 @@ typedef struct SNode { typedef struct SListCell { struct SListCell* pPrev; struct SListCell* pNext; - SNode* pNode; + SNode* pNode; } SListCell; typedef struct SNodeList { - int32_t length; + int32_t length; SListCell* pHead; SListCell* pTail; } SNodeList; -#define SNodeptr void* +#define SNodeptr void* SNodeptr nodesMakeNode(ENodeType type); -void nodesDestroyNode(SNodeptr pNode); +void nodesDestroyNode(SNodeptr pNode); SNodeList* nodesMakeList(); -int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); -int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); -int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode); -int32_t nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode); -int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); -int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); -int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode); +int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); +int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); +int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode); +int32_t nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode); +int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); +int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); +int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); -void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc); -SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); -void nodesDestroyList(SNodeList* pList); +void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc); +SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); +void nodesDestroyList(SNodeList* pList); // Only clear the linked list structure, without releasing the elements inside void nodesClearList(SNodeList* pList); -typedef enum EDealRes { - DEAL_RES_CONTINUE = 1, - DEAL_RES_IGNORE_CHILD, - DEAL_RES_ERROR, - DEAL_RES_END -} EDealRes; +typedef enum EDealRes { DEAL_RES_CONTINUE = 1, DEAL_RES_IGNORE_CHILD, DEAL_RES_ERROR, DEAL_RES_END } EDealRes; typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext); @@ -261,18 +267,18 @@ void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* bool nodesEqualNode(const SNodeptr a, const SNodeptr b); -SNodeptr nodesCloneNode(const SNodeptr pNode); +SNodeptr nodesCloneNode(const SNodeptr pNode); SNodeList* nodesCloneList(const SNodeList* pList); const char* nodesNodeName(ENodeType type); -int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen); -int32_t nodesStringToNode(const char* pStr, SNode** pNode); +int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen); +int32_t nodesStringToNode(const char* pStr, SNode** pNode); int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToList(const char* pStr, SNodeList** pList); -int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len); -char *nodesGetNameFromColumnNode(SNode *pNode); +int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len); +char* nodesGetNameFromColumnNode(SNode* pNode); int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots); #ifdef __cplusplus diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 238c4c8538..0332b8ce69 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -88,6 +88,7 @@ typedef struct SValueNode { double d; char* p; } datum; + int64_t typeData; char unit; } SValueNode; @@ -294,13 +295,14 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext); typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType; - int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, SNodeList** pCols); typedef bool (*FFuncClassifier)(int32_t funcId); int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs); +int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes); + bool nodesIsExprNode(const SNode* pNode); bool nodesIsUnaryOp(const SOperatorNode* pOp); @@ -312,6 +314,7 @@ bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); void* nodesGetValueFromNode(SValueNode* pNode); +int32_t nodesSetValueNodeValue(SValueNode* pNode, void *value); char* nodesGetStrValueFromNode(SValueNode* pNode); char* getFillModeString(EFillMode mode); void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 576ac8a364..0b2301739a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -318,6 +318,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) #define TSDB_CODE_VND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0516) #define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0517) +#define TSDB_CODE_VND_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0518) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -616,6 +617,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_OFFSET_LESS_ZERO TAOS_DEF_ERROR_CODE(0, 0x2637) #define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638) #define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639) +#define TSDB_CODE_PAR_INVALID_DROP_STABLE TAOS_DEF_ERROR_CODE(0, 0x263A) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tencode.h b/include/util/tencode.h index c5066996f3..b081d1a157 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -79,17 +79,6 @@ typedef struct { #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -// #define TCODER_MALLOC(PCODER, SIZE) \ -// ({ \ -// void* ptr = NULL; \ -// SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \ -// if (pMem) { \ -// pMem->next = (PCODER)->mList; \ -// (PCODER)->mList = pMem; \ -// ptr = (void*)&pMem[1]; \ -// } \ -// ptr; \ -// }) static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) { void* ptr = NULL; SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(SCoderMem*) + size); @@ -102,8 +91,9 @@ static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) { } #define tEncodeSize(E, S, SIZE, RET) \ - do{ \ + do { \ SCoder coder = {0}; \ + RET = 0; \ tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \ if ((E)(&coder, S) == 0) { \ SIZE = coder.pos; \ @@ -111,7 +101,7 @@ static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) { RET = -1; \ } \ tCoderClear(&coder); \ - }while(0) + } while (0) // #define tEncodeSize(E, S, SIZE) \ // ({ \ // SCoder coder = {0}; \ diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index a2382ad541..eeae1b47da 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -52,72 +52,21 @@ typedef struct SSkipListNode { #define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] #define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] -/* - * @version 0.3 - * @date 2017/11/12 - * the simple version of skip list. - * - * for multi-thread safe purpose, we employ TdThreadRwlock to guarantee to generate - * deterministic result. Later, we will remove the lock in SkipList to further enhance the performance. - * In this case, one should use the concurrent skip list (by using michael-scott algorithm) instead of - * this simple version in a multi-thread environment, to achieve higher performance of read/write operations. - * - * Note: Duplicated primary key situation. - * In case of duplicated primary key, two ways can be employed to handle this situation: - * 1. add as normal insertion without special process. - * 2. add an overflow pointer at each list node, all nodes with the same key will be added in the overflow pointer. - * In this case, the total steps of each search will be reduced significantly. - * Currently, we implement the skip list in a line with the first means, maybe refactor it soon. - * - * Memory consumption: the memory alignment causes many memory wasted. So, employ a memory - * pool will significantly reduce the total memory consumption, as well as the calloc/malloc operation costs. - * - */ - -// state struct, record following information: -// number of links in each level. -// avg search steps, for latest 1000 queries -// avg search rsp time, for latest 1000 queries -// total memory size -typedef struct tSkipListState { - // in bytes, sizeof(SSkipList)+sizeof(SSkipListNode)*SSkipList->nSize - uint64_t nTotalMemSize; - uint64_t nLevelNodeCnt[MAX_SKIP_LIST_LEVEL]; - uint64_t queryCount; // total query count - - /* - * only record latest 1000 queries - * when the value==1000, = 0, - * nTotalStepsForQueries = 0, - * nTotalElapsedTimeForQueries = 0 - */ - uint64_t nRecQueries; - uint16_t nTotalStepsForQueries; - uint64_t nTotalElapsedTimeForQueries; - - uint16_t nInsertObjs; - uint16_t nTotalStepsForInsert; - uint64_t nTotalElapsedTimeForInsert; -} tSkipListState; - typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus; typedef struct SSkipList { - uint32_t seed; - __compar_fn_t comparFn; - __sl_key_fn_t keyFn; - TdThreadRwlock *lock; - uint16_t len; - uint8_t maxLevel; - uint8_t flags; - uint8_t type; // static info above - uint8_t level; - uint32_t size; - SSkipListNode *pHead; // point to the first element - SSkipListNode *pTail; // point to the last element -#if SKIP_LIST_RECORD_PERFORMANCE - tSkipListState state; // skiplist state -#endif + uint32_t seed; + __compar_fn_t comparFn; + __sl_key_fn_t keyFn; + TdThreadRwlock *lock; + uint16_t len; + uint8_t maxLevel; + uint8_t flags; + uint8_t type; // static info above + uint8_t level; + uint32_t size; + SSkipListNode *pHead; // point to the first element + SSkipListNode *pTail; // point to the last element tGenericSavedFunc *insertHandleFn; } SSkipList; diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index ab27d7f00c..43e886faf5 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -46,11 +46,12 @@ typedef struct SStmtTableCache { void* boundTags; } SStmtTableCache; -typedef struct SQueryFields { +typedef struct SStmtQueryResInfo { TAOS_FIELD* fields; TAOS_FIELD* userFields; uint32_t numOfCols; -} SQueryFields; + int32_t precision; +} SStmtQueryResInfo; typedef struct SStmtBindInfo { bool needParse; @@ -72,17 +73,17 @@ typedef struct SStmtExecInfo { } SStmtExecInfo; typedef struct SStmtSQLInfo { - STMT_TYPE type; - STMT_STATUS status; - bool autoCreate; - uint64_t runTimes; - SHashObj* pTableCache; //SHash - SQuery* pQuery; - char* sqlStr; - int32_t sqlLen; - SArray* nodeList; - SQueryPlan* pQueryPlan; - SQueryFields fields; + STMT_TYPE type; + STMT_STATUS status; + bool autoCreate; + uint64_t runTimes; + SHashObj* pTableCache; //SHash + SQuery* pQuery; + char* sqlStr; + int32_t sqlLen; + SArray* nodeList; + SQueryPlan* pQueryPlan; + SStmtQueryResInfo queryRes; } SStmtSQLInfo; typedef struct STscStmt { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index ca6a11a668..5ddffa0cbd 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -74,17 +74,44 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { } int32_t stmtBackupQueryFields(STscStmt* pStmt) { - SQueryFields *pFields = &pStmt->sql.fields; - int32_t size = pFields->numOfCols * sizeof(TAOS_FIELD); + SStmtQueryResInfo *pRes = &pStmt->sql.queryRes; + pRes->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols; + pRes->precision = pStmt->exec.pRequest->body.resInfo.precision; - pFields->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols; - pFields->fields = taosMemoryMalloc(size); - pFields->userFields = taosMemoryMalloc(size); - if (NULL == pFields->fields || NULL == pFields->userFields) { + int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); + pRes->fields = taosMemoryMalloc(size); + pRes->userFields = taosMemoryMalloc(size); + if (NULL == pRes->fields || NULL == pRes->userFields) { STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); } - memcpy(pFields->fields, pStmt->exec.pRequest->body.resInfo.fields, size); - memcpy(pFields->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); + memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size); + memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtRestoreQueryFields(STscStmt* pStmt) { + SStmtQueryResInfo *pRes = &pStmt->sql.queryRes; + int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); + + pStmt->exec.pRequest->body.resInfo.numOfCols = pRes->numOfCols; + pStmt->exec.pRequest->body.resInfo.precision = pRes->precision; + + if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { + pStmt->exec.pRequest->body.resInfo.fields = taosMemoryMalloc(size); + if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { + STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + } + memcpy(pStmt->exec.pRequest->body.resInfo.fields, pRes->fields, size); + } + + if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { + pStmt->exec.pRequest->body.resInfo.userFields = taosMemoryMalloc(size); + if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { + STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + } + memcpy(pStmt->exec.pRequest->body.resInfo.userFields, pRes->userFields, size); + } return TSDB_CODE_SUCCESS; } @@ -235,6 +262,8 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { } int32_t stmtCleanSQLInfo(STscStmt* pStmt) { + taosMemoryFree(pStmt->sql.queryRes.fields); + taosMemoryFree(pStmt->sql.queryRes.userFields); taosMemoryFree(pStmt->sql.sqlStr); qDestroyQuery(pStmt->sql.pQuery); qDestroyQueryPlan(pStmt->sql.pQueryPlan); @@ -497,6 +526,8 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; pStmt->exec.pRequest->body.pDag = NULL; STMT_ERR_RET(stmtBackupQueryFields(pStmt)); + } else { + STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); } STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId)); @@ -509,7 +540,11 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { } if (colIdx < 0) { - qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + int32_t code = qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + if (code) { + tscError("qBindStmtColsValue failed, error:%s", tstrerror(code)); + STMT_ERR_RET(code); + } } else { if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) { tscError("bind column index not in sequence"); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 93a20c3d45..698c0cc4e7 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -68,6 +68,7 @@ struct tmq_conf_t { char* pass; char* db; tmq_commit_cb* commitCb; + void* commitCbUserParam; }; struct tmq_t { @@ -78,7 +79,8 @@ struct tmq_t { int32_t autoCommitInterval; int32_t resetOffsetCfg; int64_t consumerId; - tmq_commit_cb* commit_cb; + tmq_commit_cb* commitCb; + void* commitCbUserParam; // status int8_t status; @@ -372,10 +374,18 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; - if (pParam->tmq->commit_cb) { - pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL); + if (pParam->tmq->commitCb) { + pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, NULL, pParam->tmq->commitCbUserParam); + } + if (!pParam->async) + tsem_post(&pParam->rspSem); + else { + tsem_destroy(&pParam->rspSem); + /*if (pParam->pArray) {*/ + /*taosArrayDestroy(pParam->pArray);*/ + /*}*/ + taosMemoryFree(pParam); } - if (!pParam->async) tsem_post(&pParam->rspSem); return 0; } @@ -384,7 +394,7 @@ tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { *topics = tmq_list_new(); } for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* topic = taosArrayGetP(tmq->clientTopics, i); + SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); tmq_list_append(*topics, topic->topicName); } return TMQ_RESP_ERR__SUCCESS; @@ -477,7 +487,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { strcpy(pTmq->groupId, conf->groupId); pTmq->autoCommit = conf->autoCommit; pTmq->autoCommitInterval = conf->autoCommitInterval; - pTmq->commit_cb = conf->commitCb; + pTmq->commitCb = conf->commitCb; + pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; // assign consumerId @@ -557,7 +568,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tscError("failed to malloc request"); } - SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); if (pParam == NULL) { return -1; } @@ -572,6 +583,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in }; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqCommitCb; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -582,13 +594,12 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in if (!async) { tsem_wait(&pParam->rspSem); resp = pParam->rspErr; - } + tsem_destroy(&pParam->rspSem); + taosMemoryFree(pParam); - tsem_destroy(&pParam->rspSem); - taosMemoryFree(pParam); - - if (pArray) { - taosArrayDestroy(pArray); + if (pArray) { + taosArrayDestroy(pArray); + } } return resp; @@ -688,9 +699,10 @@ FAIL: return code; } -void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { +void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) { // conf->commitCb = cb; + conf->commitCbUserParam = param; } #if 0 @@ -1306,10 +1318,10 @@ const char* tmq_err2str(tmq_resp_err_t err) { return "fail"; } -char* tmq_get_topic_name(TAOS_RES* res) { +const char* tmq_get_topic_name(TAOS_RES* res) { if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; - return pRspObj->topic; + return strchr(pRspObj->topic, '.') + 1; } else { return NULL; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 044021e0df..874e09b403 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -490,21 +490,6 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR return 0; } -int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->ver); - tlen += taosEncodeString(buf, pReq->name); - tlen += taosEncodeFixedU8(buf, pReq->type); - return tlen; -} - -void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { - buf = taosDecodeFixedI64(buf, &pReq->ver); - buf = taosDecodeString(buf, &pReq->name); - buf = taosDecodeFixedU8(buf, &pReq->type); - return buf; -} - int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3811,3 +3796,119 @@ int tDecodeSVCreateTbRsp(SCoder *pCoder, SVCreateTbRsp *pRsp) { tEndDecode(pCoder); return 0; } + +// TDMT_VND_DROP_TABLE ================= +static int32_t tEncodeSVDropTbReq(SCoder *pCoder, const SVDropTbReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeCStr(pCoder, pReq->name) < 0) return -1; + if (tEncodeI8(pCoder, pReq->igNotExists) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSVDropTbReq(SCoder *pCoder, SVDropTbReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->igNotExists) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} + +static int32_t tEncodeSVDropTbRsp(SCoder *pCoder, const SVDropTbRsp *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32(pCoder, pReq->code) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSVDropTbRsp(SCoder *pCoder, SVDropTbRsp *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32(pCoder, &pReq->code) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDropTbBatchReq(SCoder *pCoder, const SVDropTbBatchReq *pReq) { + int32_t nReqs = taosArrayGetSize(pReq->pArray); + SVDropTbReq *pDropTbReq; + + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, nReqs) < 0) return -1; + for (int iReq = 0; iReq < nReqs; iReq++) { + pDropTbReq = (SVDropTbReq *)taosArrayGet(pReq->pArray, iReq); + if (tEncodeSVDropTbReq(pCoder, pDropTbReq) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDropTbBatchReq(SCoder *pCoder, SVDropTbBatchReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pReq->nReqs) < 0) return -1; + pReq->pReqs = (SVDropTbReq *)tCoderMalloc(pCoder, sizeof(SVDropTbReq) * pReq->nReqs); + if (pReq->pReqs == NULL) return -1; + for (int iReq = 0; iReq < pReq->nReqs; iReq++) { + if (tDecodeSVDropTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDropTbBatchRsp(SCoder *pCoder, const SVDropTbBatchRsp *pRsp) { + int32_t nRsps = taosArrayGetSize(pRsp->pArray); + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, nRsps) < 0) return -1; + for (int iRsp = 0; iRsp < nRsps; iRsp++) { + if (tEncodeSVDropTbRsp(pCoder, (SVDropTbRsp *)taosArrayGet(pRsp->pArray, iRsp)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDropTbBatchRsp(SCoder *pCoder, SVDropTbBatchRsp *pRsp) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pRsp->nRsps) < 0) return -1; + pRsp->pRsps = (SVDropTbRsp *)tCoderMalloc(pCoder, sizeof(SVDropTbRsp) * pRsp->nRsps); + if (pRsp->pRsps == NULL) return -1; + for (int iRsp = 0; iRsp < pRsp->nRsps; iRsp++) { + if (tDecodeSVDropTbRsp(pCoder, pRsp->pRsps + iRsp) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDropStbReq(SCoder *pCoder, const SVDropStbReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeCStr(pCoder, pReq->name) < 0) return -1; + if (tEncodeI64(pCoder, pReq->suid) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDropStbReq(SCoder *pCoder, SVDropStbReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index fb8e5e7fb2..76cee15f01 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -254,8 +254,17 @@ static void dmSendRpcRedirectRsp(SDnode *pDnode, const SRpcMsg *pReq) { epSet.eps[i].port = htons(epSet.eps[i].port); } + SRpcMsg resp; + SMEpSet msg = {.epSet = epSet}; + int32_t len = tSerializeSMEpSet(NULL, 0, &msg); + resp.pCont = rpcMallocCont(len); + resp.contLen = len; + tSerializeSMEpSet(resp.pCont, len, &msg); - rpcSendRedirectRsp(pReq->handle, &epSet); + resp.code = TSDB_CODE_RPC_REDIRECT; + resp.handle = pReq->handle; + resp.refId = pReq->refId; + rpcSendResponse(&resp); } static inline void dmSendRpcRsp(SDnode *pDnode, const SRpcMsg *pRsp) { diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 5083104039..b96444bebc 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -47,7 +47,7 @@ typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); -typedef struct SQWorkerMgmt SQHandle; +typedef struct SQWorker SQHandle; typedef struct { const char *name; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index f717531030..ec8dfa4d47 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -428,17 +428,23 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt } static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { - SName name = {0}; + SName name = {0}; + SVDropStbReq req = {0}; + int32_t contLen = 0; + int32_t ret = 0; + SMsgHead *pHead = NULL; + SCoder coder = {0}; + tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - SVDropTbReq req = {0}; - req.ver = 0; req.name = (char *)tNameGetTableName(&name); - req.type = TD_SUPER_TABLE; req.suid = pStb->uid; - int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = taosMemoryMalloc(contLen); + tEncodeSize(tEncodeSVDropStbReq, &req, contLen, ret); + if (ret < 0) return NULL; + + contLen += sizeof(SMsgHead); + pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -448,7 +454,10 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, pHead->vgId = htonl(pVgroup->vgId); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); - tSerializeSVDropTbReq(&pBuf, &req); + + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, contLen - sizeof(SMsgHead), TD_ENCODER); + tEncodeSVDropStbReq(&coder, &req); + tCoderClear(&coder); *pContLen = contLen; return pHead; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 2b3af85066..13ee26b6cd 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -144,6 +144,10 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM int32_t vgId = pRebVg->pVgEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); + if (pVgObj == NULL) { + taosMemoryFree(buf); + return -1; + } STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgObj); @@ -509,7 +513,9 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ // TODO replace assert with error check - ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); + if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { + mError("persist rebalance output error, possibly vnode splitted or dropped"); + } if (rebInput.pTopic) { SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic; diff --git a/source/dnode/qnode/inc/qndInt.h b/source/dnode/qnode/inc/qndInt.h index 307bf3efb8..c18a43c4fb 100644 --- a/source/dnode/qnode/inc/qndInt.h +++ b/source/dnode/qnode/inc/qndInt.h @@ -29,7 +29,7 @@ extern "C" { #endif -typedef struct SQWorkerMgmt SQHandle; +typedef struct SQWorker SQHandle; typedef struct SQnode { int32_t qndId; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 724c7b312e..85d0f76268 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -8,7 +8,6 @@ target_sources( "src/vnd/vnodeBufPool.c" "src/vnd/vnodeCfg.c" "src/vnd/vnodeCommit.c" - "src/vnd/vnodeInt.c" "src/vnd/vnodeQuery.c" "src/vnd/vnodeStateMgr.c" "src/vnd/vnodeModule.c" @@ -33,7 +32,6 @@ target_sources( "src/tsdb/tsdbMemTable.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" - "src/tsdb/tsdbScan.c" "src/tsdb/tsdbSma.c" "src/tsdb/tsdbWrite.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f7e8f24402..daf8c4f339 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -32,7 +32,7 @@ #include "tmsg.h" #include "trow.h" -#include "tdbInt.h" +#include "tdb.h" #ifdef __cplusplus extern "C" { @@ -57,9 +57,6 @@ int vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); -int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg); -int32_t vnodeCompact(SVnode *pVnode); -int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int vnodeValidateTableHash(SVnode *pVnode, char *tableFName); diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 0001a43231..ca7e823d6b 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -43,7 +43,6 @@ int metaEncodeEntry(SCoder* pCoder, const SMetaEntry* pME); int metaDecodeEntry(SCoder* pCoder, SMetaEntry* pME); // metaTable ================== -int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); // metaQuery ================== int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4fc94da06a..9ac8434949 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -73,7 +73,6 @@ struct STsdb { SVnode *pVnode; bool repoLocked; TdThreadMutex mutex; - STsdbCfg config; STsdbMemTable *mem; STsdbMemTable *imem; SRtn rtn; @@ -185,7 +184,7 @@ struct STsdbFS { }; #define REPO_ID(r) TD_VID((r)->pVnode) -#define REPO_CFG(r) (&(r)->config) +#define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg) #define REPO_FS(r) ((r)->fs) #define REPO_META(r) ((r)->pVnode->pMeta) #define REPO_TFS(r) ((r)->pVnode->pTfs) @@ -534,23 +533,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { } } -// tsdbDBDef -// typedef struct SDBFile SDBFile; -// typedef DB_ENV* TDBEnv; - -// struct SDBFile { -// int32_t fid; -// DB* pDB; -// char* path; -// }; - -// int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF); -// void tsdbCloseDBF(SDBFile* pDBF); -// int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path); -// void tsdbCloseBDBEnv(DB_ENV* pEnv); -// int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize); -// void* tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize); - // tsdbFile #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 8b92e71a24..7a3c8c4bf6 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -25,7 +25,7 @@ #include "tcompare.h" #include "tcompression.h" #include "tdatablock.h" -#include "tdbInt.h" +#include "tdb.h" #include "tencode.h" #include "tfs.h" #include "tglobal.h" @@ -52,7 +52,7 @@ typedef struct STsdb STsdb; typedef struct STQ STQ; typedef struct SVState SVState; typedef struct SVBufPool SVBufPool; -typedef struct SQWorkerMgmt SQHandle; +typedef struct SQWorker SQHandle; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -75,6 +75,7 @@ int metaClose(SMeta* pMeta); int metaBegin(SMeta* pMeta); int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index e65f5229af..85a49ffabd 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -47,66 +47,66 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { // open env ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); if (ret < 0) { - metaError("vgId: %d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTbDb ret = tdbDbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); if (ret < 0) { - metaError("vgId: %d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSkmDb ret = tdbDbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); if (ret < 0) { - metaError("vgId: %d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pUidIdx ret = tdbDbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pNameIdx ret = tdbDbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pCtbIdx ret = tdbDbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTagIdx ret = tdbDbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTtlIdx ret = tdbDbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open index if (metaOpenIdx(pMeta) < 0) { - metaError("vgId: %d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } - metaDebug("vgId: %d meta is opened", TD_VID(pVnode)); + metaDebug("vgId:%d meta is opened", TD_VID(pVnode)); *ppMeta = pMeta; return 0; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 5df7b97f2a..60c844e394 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -23,7 +23,7 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { void metaReaderClear(SMetaReader *pReader) { tCoderClear(&pReader->coder); - TDB_FREE(pReader->pBuf); + tdbFree(pReader->pBuf); } int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t uid) { @@ -96,15 +96,17 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { metaReaderInit(&pTbCur->mr, pMeta, 0); - tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc); + tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); + + tdbDbcMoveToFirst(pTbCur->pDbc); return pTbCur; } void metaCloseTbCursor(SMTbCursor *pTbCur) { if (pTbCur) { - TDB_FREE(pTbCur->pKey); - TDB_FREE(pTbCur->pVal); + tdbFree(pTbCur->pKey); + tdbFree(pTbCur->pVal); metaReaderClear(&pTbCur->mr); if (pTbCur->pDbc) { tdbDbcClose(pTbCur->pDbc); @@ -119,7 +121,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) { STbCfg tbCfg; for (;;) { - ret = tdbDbNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbDbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } @@ -169,7 +171,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo pSW->pSchema = pSchema; - TDB_FREE(pVal); + tdbFree(pVal); return pSW; } @@ -185,7 +187,9 @@ struct SMCtbCursor { SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMCtbCursor *pCtbCur = NULL; + SCtbIdxKey ctbIdxKey; int ret; + int c; pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); if (pCtbCur == NULL) { @@ -193,12 +197,20 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { } pCtbCur->suid = uid; - ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur); + ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); if (ret < 0) { taosMemoryFree(pCtbCur); return NULL; } + // move to the suid + ctbIdxKey.suid = uid; + ctbIdxKey.uid = INT64_MIN; + tdbDbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); + if (c > 0) { + tdbDbcMoveToNext(pCtbCur->pCur); + } + return pCtbCur; } @@ -207,8 +219,8 @@ void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { if (pCtbCur->pCur) { tdbDbcClose(pCtbCur->pCur); - TDB_FREE(pCtbCur->pKey); - TDB_FREE(pCtbCur->pVal); + tdbFree(pCtbCur->pKey); + tdbFree(pCtbCur->pVal); } taosMemoryFree(pCtbCur); @@ -219,12 +231,15 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { int ret; SCtbIdxKey *pCtbIdxKey; - ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); + ret = tdbDbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); if (ret < 0) { return 0; } pCtbIdxKey = pCtbCur->pKey; + if (pCtbIdxKey->suid > pCtbCur->suid) { + return 0; + } return pCtbIdxKey->uid; } @@ -283,7 +298,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { while (true) { // TODO: lock during iterate? - if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { pSmaIdxKey = pCur->pKey; ASSERT(pSmaIdxKey != NULL); @@ -297,7 +312,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { ++pSW->number; STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); @@ -306,13 +321,13 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { pSW->tSma = tptr; pBuf = pSmaVal; if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); return NULL; } - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); continue; } break; @@ -354,7 +369,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { tb_uid_t uid = 0; while (true) { // TODO: lock during iterate? - if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { ASSERT(pSmaIdxKey != NULL); pSmaIdxKey = pCur->pKey; @@ -425,11 +440,11 @@ void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { if (tDecodeTSma(pBuf, pCfg) == NULL) { tdDestroyTSma(pCfg); taosMemoryFree(pCfg); - TDB_FREE(pVal); + tdbFree(pVal); return NULL; } - TDB_FREE(pVal); + tdbFree(pVal); return pCfg; #endif #endif diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index fd5f679d0b..bba707076c 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -289,7 +289,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { pVal = pBuf = buf; metaEncodeTbInfo(&pBuf, pTbCfg); vLen = POINTER_DISTANCE(pBuf, buf); - ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbPut(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -311,7 +311,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { pVal = pBuf = buf; metaEncodeSchemaEx(&pBuf, &schemaWrapper); vLen = POINTER_DISTANCE(pBuf, buf); - ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn); + ret = tdbDbPut(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn); if (ret < 0) { return -1; } @@ -325,7 +325,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = nameLen + 1 + sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbPut(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -336,7 +336,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbPut(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -347,7 +347,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(ctbIdxKey); pVal = NULL; vLen = 0; - ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbPut(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -362,7 +362,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbPut(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -407,7 +407,7 @@ static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_ pSchemaWrapper = taosMemoryMalloc(sizeof(*pSchemaWrapper)); metaDecodeSchemaEx(pBuf, pSchemaWrapper, isGetEx); - TDB_FREE(pVal); + tdbFree(pVal); return pSchemaWrapper; } @@ -438,7 +438,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { while (true) { // TODO: lock during iterate? - if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { pSmaIdxKey = pCur->pKey; ASSERT(pSmaIdxKey != NULL); @@ -450,7 +450,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { } if ((pSW == NULL) && ((pSW = taosMemoryCalloc(1, sizeof(*pSW))) == NULL)) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); return NULL; } @@ -458,7 +458,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { ++pSW->number; STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); @@ -467,13 +467,13 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { pSW->tSma = tptr; pBuf = pSmaVal; if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); return NULL; } - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); continue; } break; @@ -530,7 +530,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { int32_t kLen = sizeof(pSmaCfg->indexUid); int32_t vLen = POINTER_DISTANCE(qBuf, pBuf); - ret = tdbDbInsert(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); + ret = tdbDbPut(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); if (ret < 0) { taosMemoryFreeClear(pBuf); return -1; @@ -545,7 +545,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { val = NULL; vLen = 0; - ret = tdbDbInsert(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); + ret = tdbDbPut(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); if (ret < 0) { taosMemoryFreeClear(pBuf); return -1; @@ -613,7 +613,7 @@ int64_t metaSmaCursorNext(SMSmaCursor *pCur) { void *pBuf; SSmaIdxKey *smaIdxKey; - ret = tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, &pCur->pVal, &pCur->vLen); + ret = tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, &pCur->pVal, &pCur->vLen); if (ret < 0) { return 0; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 3e70bbb479..cb0b3dc81c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -61,19 +61,58 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { if (metaHandleEntry(pMeta, &me) < 0) goto _err; - metaDebug("vgId: %d super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + metaDebug("vgId:%d super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; _err: - metaError("vgId: %d failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + metaError("vgId:%d failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, tstrerror(terrno)); return -1; } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - // TODO + SMetaReader mr = {0}; + + // validate req + metaReaderInit(&mr, pMeta, 0); + if (metaGetTableEntryByUid(&mr, pReq->suid) < 0) { + terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; + goto _err; + } + + // do drop + // drop from pTbDb + // drop from pSkmDb + // drop from pUidIdx + // drop from pNameIdx + // { + // TDBC *pDbc1 = NULL; + // void *pKey = NULL; + // void *pVal = NULL; + // int kLen = 0; + // int vLen = 0; + // int ret = 0; + + // // drop from pCtbIdx + // ret = tdbDbcOpen(pMeta->pCtbIdx, &pDbc1); + // tdbDbcMoveTo(pDbc1, &pReq->suid, sizeof(pReq->suid), NULL /*cmpr*/, 0 /*TDB_FORWARD_SEARCH*/); + // tdbDbcGet(pDbc1, &pKey, &kLen, &pVal, vLen); + // tdbDbcDrop(pDbc1); + // // drop from pTagIdx + // // drop from pTtlIdx + // } + + // clear and return + metaReaderClear(&mr); + metaError("vgId:%d super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; + +_err: + metaReaderClear(&mr); + metaError("vgId:%d failed to drop super table %s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(terrno)); + return -1; } int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { @@ -179,7 +218,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tCoderClear(&coder); // write to table.db - if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbDbPut(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -192,11 +231,11 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbDbPut(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbDbPut(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -219,12 +258,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.uid = pME->uid; - return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbDbPut(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbDbPut(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -265,7 +304,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbDbPut(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index cf1154071d..89ec55cca1 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -91,16 +91,8 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t sversion = 0; if (pHandle->sver != sversion) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); -#if 0 - tb_uid_t quid; - STbCfg* pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->msgIter.uid); - if (pTbCfg->type == META_CHILD_TABLE) { - quid = pTbCfg->ctbCfg.suid; - } else { - quid = pHandle->msgIter.uid; - } - pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true); -#endif + + // this interface use suid instead of uid pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.suid, sversion, true); pHandle->sver = sversion; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 63c888dab3..6a8af73694 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -55,7 +55,7 @@ typedef struct { #define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) #define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) #define TSDB_COMMIT_EXBUF(ch) TSDB_READ_EXBUF(&((ch)->readh)) -#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRows) +#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->pVnode->config.tsdbCfg.maxRows) #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) static void tsdbStartCommit(STsdb *pRepo); diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index b52ccbe163..0827ba6eab 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -56,9 +56,8 @@ int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) { dir); pTsdb->pVnode = pVnode; pTsdb->repoLocked = false; - tdbMutexInit(&pTsdb->mutex, NULL); - pTsdb->config = pVnode->config.tsdbCfg; - pTsdb->fs = tsdbNewFS(&pTsdb->config); + taosThreadMutexInit(&pTsdb->mutex, NULL); + pTsdb->fs = tsdbNewFS(REPO_CFG(pTsdb)); // create dir (TODO: use tfsMkdir) taosMkDir(pTsdb->path); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d8593f01d5..cb91596cb4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -311,7 +311,7 @@ static bool emptyQueryTimewindow(STsdbReadHandle* pTsdbReadHandle) { // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. static int64_t getEarliestValidTimestamp(STsdb* pTsdb) { - STsdbCfg* pCfg = &pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdb); int64_t now = taosGetTimestamp(pCfg->precision); return now - (tsTickPerDay[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick @@ -404,7 +404,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, SQueryTableDataCond* pC pReadHandle->defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); } - pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRows); + pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->pVnode->config.tsdbCfg.maxRows); if (pReadHandle->pDataCols == NULL) { tsdbError("%p failed to malloc buf for pDataCols, %s", pReadHandle, pReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -889,7 +889,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { } static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { - STsdbCfg* pCfg = &pHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pHandle->pTsdb); size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = INT32_MIN; @@ -1169,7 +1169,7 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo) { SQueryFilePos* cur = &pTsdbReadHandle->cur; - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); TSKEY key; int32_t code = TSDB_CODE_SUCCESS; @@ -1756,7 +1756,7 @@ int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* p static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { SQueryFilePos* cur = &pTsdbReadHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); initTableMemIterator(pTsdbReadHandle, pCheckInfo); @@ -2200,7 +2200,7 @@ static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exi int32_t numOfBlocks = 0; int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); STimeWindow win = TSWINDOW_INITIALIZER; while (true) { @@ -2306,7 +2306,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* // find the start data block in file pTsdbReadHandle->locateStart = true; - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->days, pCfg->precision); tsdbRLockFS(pFileHandle); @@ -2407,7 +2407,7 @@ static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exis // find the start data block in file if (!pTsdbReadHandle->locateStart) { pTsdbReadHandle->locateStart = true; - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->days, pCfg->precision); tsdbRLockFS(pFileHandle); @@ -2498,7 +2498,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index ea96eabd22..d7c9a70c18 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -248,7 +248,8 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { ASSERT(pBlock->numOfSubBlocks > 0); - int8_t update = pReadh->pRepo->config.update; + STsdbCfg *pCfg = REPO_CFG(pReadh->pRepo); + int8_t update = pCfg->update; SBlock *iBlock = pBlock; if (pBlock->numOfSubBlocks > 1) { @@ -279,7 +280,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds, bool mergeBitmap) { ASSERT(pBlock->numOfSubBlocks > 0); - int8_t update = pReadh->pRepo->config.update; + int8_t update = pReadh->pRepo->pVnode->config.tsdbCfg.update; SBlock *iBlock = pBlock; if (pBlock->numOfSubBlocks > 1) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index d5cf6613fd..d131cef8a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -223,7 +223,7 @@ static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { } static SPoolMem *openPool() { - SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool)); + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); pPool->prev = pPool->next = pPool; pPool->size = 0; @@ -245,7 +245,7 @@ static void clearPool(SPoolMem *pPool) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - tdbOsFree(pMem); + taosMemoryFree(pMem); } while (1); assert(pPool->size == 0); @@ -254,7 +254,7 @@ static void clearPool(SPoolMem *pPool) { static void closePool(SPoolMem *pPool) { if (pPool) { clearPool(pPool); - tdbOsFree(pPool); + taosMemoryFree(pPool); } } @@ -263,7 +263,7 @@ static void *poolMalloc(void *arg, size_t size) { SPoolMem *pPool = (SPoolMem *)arg; SPoolMem *pMem; - pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size); + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); if (!pMem) { assert(0); } @@ -290,7 +290,7 @@ static void poolFree(void *arg, void *ptr) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - tdbOsFree(pMem); + taosMemoryFree(pMem); } int32_t tsdbInitSma(STsdb *pTsdb) { diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index ac3baf8c3a..74878e817f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -55,12 +55,12 @@ static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, i } static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { - int ret; - FKeyComparator compFunc; + int ret; + tdb_cmpr_fn_t compFunc; // Create a database compFunc = tsdbSmaKeyCmpr; - ret = tdbDbOpen(pFName, TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, ppDB); + ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); return 0; } @@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t ret; - ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbDbPut(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { tsdbError("Failed to create insert sma data into db, ret = %d", ret); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 82fa6b9ae5..9fb6aad472 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -60,9 +60,10 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { SSubmitBlk *pBlock = NULL; SSubmitBlkIter blkIter = {0}; STSRow *row = NULL; - TSKEY now = taosGetTimestamp(pTsdb->config.precision); - TSKEY minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep2; - TSKEY maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.days; + STsdbCfg *pCfg = REPO_CFG(pTsdb); + TSKEY now = taosGetTimestamp(pCfg->precision); + TSKEY minKey = now - tsTickPerDay[pCfg->precision] * pCfg->keep2; + TSKEY maxKey = now + tsTickPerDay[pCfg->precision] * pCfg->days; terrno = TSDB_CODE_SUCCESS; pMsg->length = htonl(pMsg->length); diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 1c0e65aa95..5412b5d1c2 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -42,13 +42,13 @@ int vnodeBegin(SVnode *pVnode) { // begin meta if (metaBegin(pVnode->pMeta) < 0) { - vError("vgId: %d failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // begin tsdb if (tsdbBegin(pVnode->pTsdb) < 0) { - vError("vgId: %d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -93,7 +93,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // free info binary taosMemoryFree(data); - vInfo("vgId: %d vnode info is saved, fname: %s", pInfo->config.vgId, fname); + vInfo("vgId:%d vnode info is saved, fname: %s", pInfo->config.vgId, fname); return 0; @@ -115,7 +115,7 @@ int vnodeCommitInfo(const char *dir, const SVnodeInfo *pInfo) { return -1; } - vInfo("vgId: %d vnode info is committed", pInfo->config.vgId); + vInfo("vgId:%d vnode info is committed", pInfo->config.vgId); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ae3b15f3a2..3737bcfe3b 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -23,13 +23,13 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { // check config if (vnodeCheckCfg(pCfg) < 0) { - vError("vgId: %d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); return -1; } // create vnode env if (tfsMkdir(pTfs, path) < 0) { - vError("vgId: %d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); return -1; } @@ -39,11 +39,11 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.applied = -1; if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { - vError("vgId: %d failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); return -1; } - vInfo("vgId: %d vnode is created", pCfg->vgId); + vInfo("vgId:%d vnode is created", pCfg->vgId); return 0; } @@ -70,7 +70,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode = (SVnode *)taosMemoryCalloc(1, sizeof(*pVnode) + strlen(path) + 1); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - vError("vgId: %d failed to open vnode since %s", info.config.vgId, tstrerror(terrno)); + vError("vgId:%d failed to open vnode since %s", info.config.vgId, tstrerror(terrno)); return NULL; } @@ -85,13 +85,13 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open buffer pool if (vnodeOpenBufPool(pVnode, pVnode->config.isHeap ? 0 : pVnode->config.szBuf / 3) < 0) { - vError("vgId: %d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open meta if (metaOpen(pVnode, &pVnode->pMeta) < 0) { - vError("vgId: %d failed to open vnode meta since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode meta since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } @@ -122,7 +122,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { - vError("vgId: %d failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } @@ -130,25 +130,25 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal); if (pVnode->pTq == NULL) { - vError("vgId: %d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open query if (vnodeQueryOpen(pVnode)) { - vError("vgId: %d failed to open vnode query since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode query since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // vnode begin if (vnodeBegin(pVnode) < 0) { - vError("vgId: %d failed to begin since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to begin since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open sync if (vnodeSyncOpen(pVnode, dir)) { - vError("vgId: %d failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index f5ab2cbdad..ed8a978226 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -17,10 +17,10 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { @@ -45,6 +45,47 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { #endif return 0; } +static void tdSRowDemo() { +#define DEMO_N_COLS 3 + + int16_t schemaVersion = 0; + int32_t numOfCols = DEMO_N_COLS; // ts + int + SRowBuilder rb = {0}; + + SSchema schema[DEMO_N_COLS] = { + {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = SCHEMA_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = SCHEMA_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = SCHEMA_SMA_ON}}; + + SSchema *pSchema = schema; + STSchema *pTSChema = tdGetSTSChemaFromSSChema(&pSchema, numOfCols); + + tdSRowInit(&rb, schemaVersion); + tdSRowSetTpInfo(&rb, numOfCols, pTSChema->flen); + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSChema); + void *row = taosMemoryCalloc(1, maxLen); // make sure the buffer is enough + + // set row buf + tdSRowResetBuf(&rb, row); + + for (int32_t idx = 0; idx < pTSChema->numOfCols; ++idx) { + STColumn *pColumn = pTSChema->columns + idx; + if (idx == 0) { + int64_t tsKey = 1651234567; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &tsKey, true, pColumn->offset, idx); + } else if (idx == 1) { + int32_t val1 = 10; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &val1, true, pColumn->offset, idx); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, true, pColumn->offset, idx); + } + } + + // print + tdSRowPrint(row, pTSChema, __func__); + + taosMemoryFree(pTSChema); +} int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { void *ptr = NULL; @@ -52,7 +93,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg int len; int ret; - vTrace("vgId: %d start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vTrace("vgId:%d start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); pVnode->state.applied = version; @@ -62,7 +103,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg len = pMsg->contLen - sizeof(SMsgHead); if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { - vError("vgId: %d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -75,7 +116,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg if (vnodeProcessAlterStbReq(pVnode, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_STB: - if (vnodeProcessDropStbReq(pVnode, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_TABLE: if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; @@ -84,7 +125,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg if (vnodeProcessAlterTbReq(pVnode, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TABLE: - if (vnodeProcessDropTbReq(pVnode, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_SMA: { // timeRangeSMA if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { @@ -119,7 +160,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg break; } - vDebug("vgId: %d process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); + vDebug("vgId:%d process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); // commit if need if (vnodeShouldCommit(pVnode)) { @@ -134,7 +175,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg return 0; _err: - vDebug("vgId: %d process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vDebug("vgId:%d process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), tstrerror(terrno), version); return -1; } @@ -409,9 +450,32 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpc return 0; } -static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { - // TODO - // ASSERT(0); +static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SVDropStbReq req = {0}; + int rcode = TSDB_CODE_SUCCESS; + SCoder coder = {0}; + + pRsp->msgType = TDMT_VND_CREATE_STB_RSP; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + // decode request + tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER); + if (tDecodeSVDropStbReq(&coder, &req) < 0) { + rcode = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + // process request + // if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { + // rcode = terrno; + // goto _exit; + // } + + // return rsp +_exit: + pRsp->code = rcode; + tCoderClear(&coder); return 0; } @@ -421,9 +485,15 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcM return 0; } -static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { - // TODO - ASSERT(0); +static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SVDropTbReq req = {0}; + SVDropTbReq rsp = {0}; + + // decode req + + // process req + + // return rsp return 0; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c7fdb61d16..ee8cdc6b1b 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -375,6 +375,7 @@ typedef struct SStreamBlockScanInfo { uint64_t numOfExec; // execution times void* readerHandle; // stream block reader handle SArray* pColMatchInfo; // + SNode* pCondition; } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { @@ -672,7 +673,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, - SArray* pTableIdList, SExecTaskInfo* pTaskInfo); + SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pConditions); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 882c3c3252..9ff02369c5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -6538,7 +6538,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfCols = 0; SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo); + createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo, pScanPhyNode->node.pConditions); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index dc87b864f1..8977e6e7c0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -514,6 +514,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamBlockScanInfo* pInfo = pOperator->info; + int32_t rows = 0; pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) { @@ -580,6 +581,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) pTaskInfo->code = terrno; return NULL; } + rows = pBlockInfo->rows; + doFilter(pInfo->pCondition, pInfo->pRes); break; } @@ -588,16 +591,16 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) pInfo->numOfExec++; pInfo->numOfRows += pBlockInfo->rows; - if (pBlockInfo->rows == 0) { + if (rows == 0) { pOperator->status = OP_EXEC_DONE; } - return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + return (rows == 0) ? NULL : pInfo->pRes; } } SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, - SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { + SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pCondition) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -635,6 +638,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pInfo->readerHandle = streamReadHandle; pInfo->pRes = pResBlock; + pInfo->pCondition = pCondition; pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0e6ec4f945..fa862df2b1 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -588,6 +588,259 @@ static int32_t jsonToLogicSortNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkPartitionLogicPlanPartitionKeys = "PartitionKeys"; + +static int32_t logicPartitionNodeToJson(const void* pObj, SJson* pJson) { + const SPartitionLogicNode* pNode = (const SPartitionLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkPartitionLogicPlanPartitionKeys, pNode->pPartitionKeys); + } + + return code; +} + +static int32_t jsonToLogicPartitionNode(const SJson* pJson, void* pObj) { + SPartitionLogicNode* pNode = (SPartitionLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkPartitionLogicPlanPartitionKeys, &pNode->pPartitionKeys); + } + + return code; +} + +static const char* jkSubplanIdQueryId = "QueryId"; +static const char* jkSubplanIdGroupId = "GroupId"; +static const char* jkSubplanIdSubplanId = "SubplanId"; + +static int32_t subplanIdToJson(const void* pObj, SJson* pJson) { + const SSubplanId* pNode = (const SSubplanId*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSubplanIdQueryId, pNode->queryId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSubplanIdGroupId, pNode->groupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSubplanIdSubplanId, pNode->subplanId); + } + + return code; +} + +static int32_t jsonToSubplanId(const SJson* pJson, void* pObj) { + SSubplanId* pNode = (SSubplanId*)pObj; + + int32_t code = tjsonGetUBigIntValue(pJson, jkSubplanIdQueryId, &pNode->queryId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkSubplanIdGroupId, &pNode->groupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkSubplanIdSubplanId, &pNode->subplanId); + } + + return code; +} + +static const char* jkEndPointFqdn = "Fqdn"; +static const char* jkEndPointPort = "Port"; + +static int32_t epToJson(const void* pObj, SJson* pJson) { + const SEp* pNode = (const SEp*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkEndPointFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkEndPointPort, pNode->port); + } + + return code; +} + +static int32_t jsonToEp(const SJson* pJson, void* pObj) { + SEp* pNode = (SEp*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkEndPointFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetSmallIntValue(pJson, jkEndPointPort, &pNode->port); + } + + return code; +} + +static const char* jkEpSetInUse = "InUse"; +static const char* jkEpSetNumOfEps = "NumOfEps"; +static const char* jkEpSetEps = "Eps"; + +static int32_t epSetToJson(const void* pObj, SJson* pJson) { + const SEpSet* pNode = (const SEpSet*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps); + } + + return code; +} + +static int32_t jsonToEpSet(const SJson* pJson, void* pObj) { + SEpSet* pNode = (SEpSet*)pObj; + + int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp)); + } + + return code; +} + +static const char* jkVgroupInfoVgId = "VgId"; +static const char* jkVgroupInfoHashBegin = "HashBegin"; +static const char* jkVgroupInfoHashEnd = "HashEnd"; +static const char* jkVgroupInfoEpSet = "EpSet"; +static const char* jkVgroupInfoNumOfTable = "NumOfTable"; + +static int32_t vgroupInfoToJson(const void* pObj, SJson* pJson) { + const SVgroupInfo* pNode = (const SVgroupInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupInfoVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashBegin, pNode->hashBegin); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashEnd, pNode->hashEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkVgroupInfoEpSet, epSetToJson, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoNumOfTable, pNode->numOfTable); + } + + return code; +} + +static int32_t jsonToVgroupInfo(const SJson* pJson, void* pObj) { + SVgroupInfo* pNode = (SVgroupInfo*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkVgroupInfoVgId, &pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUIntValue(pJson, jkVgroupInfoHashBegin, &pNode->hashBegin); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUIntValue(pJson, jkVgroupInfoHashEnd, &pNode->hashEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkVgroupInfoEpSet, jsonToEpSet, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkVgroupInfoNumOfTable, &pNode->numOfTable); + } + + return code; +} + +static const char* jkVgroupsInfoNum = "Num"; +static const char* jkVgroupsInfoVgroups = "Vgroups"; + +static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) { + const SVgroupsInfo* pNode = (const SVgroupsInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), + pNode->numOfVgroups); + } + + return code; +} + +static int32_t jsonToVgroupsInfo(const SJson* pJson, void* pObj) { + SVgroupsInfo* pNode = (SVgroupsInfo*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkVgroupsInfoNum, &pNode->numOfVgroups); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkVgroupsInfoVgroups, jsonToVgroupInfo, pNode->vgroups, sizeof(SVgroupInfo)); + } + + return code; +} + +static const char* jkLogicSubplanId = "Id"; +static const char* jkLogicSubplanChildren = "Children"; +static const char* jkLogicSubplanRootNode = "RootNode"; +static const char* jkLogicSubplanType = "SubplanType"; +static const char* jkLogicSubplanVgroupsSize = "VgroupsSize"; +static const char* jkLogicSubplanVgroups = "Vgroups"; +static const char* jkLogicSubplanLevel = "Level"; +static const char* jkLogicSubplanSplitFlag = "SplitFlag"; + +static int32_t logicSubplanToJson(const void* pObj, SJson* pJson) { + const SLogicSubplan* pNode = (const SLogicSubplan*)pObj; + + int32_t code = tjsonAddObject(pJson, jkLogicSubplanId, subplanIdToJson, &pNode->id); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkLogicSubplanChildren, pNode->pChildren); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkLogicSubplanRootNode, nodeToJson, pNode->pNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicSubplanType, pNode->subplanType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicSubplanVgroupsSize, VGROUPS_INFO_SIZE(pNode->pVgroupList)); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkLogicSubplanVgroups, vgroupsInfoToJson, pNode->pVgroupList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicSubplanLevel, pNode->level); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicSubplanSplitFlag, pNode->splitFlag); + } + + return code; +} + +static int32_t jsonToLogicSubplan(const SJson* pJson, void* pObj) { + SLogicSubplan* pNode = (SLogicSubplan*)pObj; + + int32_t code = tjsonToObject(pJson, jkLogicSubplanId, jsonToSubplanId, &pNode->id); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkLogicSubplanChildren, &pNode->pChildren); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkLogicSubplanRootNode, (SNode**)&pNode->pNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkLogicSubplanType, pNode->subplanType); + } + int32_t objSize = 0; + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkLogicSubplanVgroupsSize, &objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonMakeObject(pJson, jkLogicSubplanVgroups, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkLogicSubplanLevel, &pNode->level); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkLogicSubplanSplitFlag, &pNode->splitFlag); + } + + return code; +} + static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; @@ -837,63 +1090,6 @@ static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { retur static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiScanNode(pJson, pObj); } -static const char* jkEndPointFqdn = "Fqdn"; -static const char* jkEndPointPort = "Port"; - -static int32_t epToJson(const void* pObj, SJson* pJson) { - const SEp* pNode = (const SEp*)pObj; - - int32_t code = tjsonAddStringToObject(pJson, jkEndPointFqdn, pNode->fqdn); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkEndPointPort, pNode->port); - } - - return code; -} - -static int32_t jsonToEp(const SJson* pJson, void* pObj) { - SEp* pNode = (SEp*)pObj; - - int32_t code = tjsonGetStringValue(pJson, jkEndPointFqdn, pNode->fqdn); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetSmallIntValue(pJson, jkEndPointPort, &pNode->port); - } - - return code; -} - -static const char* jkEpSetInUse = "InUse"; -static const char* jkEpSetNumOfEps = "NumOfEps"; -static const char* jkEpSetEps = "Eps"; - -static int32_t epSetToJson(const void* pObj, SJson* pJson) { - const SEpSet* pNode = (const SEpSet*)pObj; - - int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps); - } - - return code; -} - -static int32_t jsonToEpSet(const SJson* pJson, void* pObj) { - SEpSet* pNode = (SEpSet*)pObj; - - int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp)); - } - - return code; -} - static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet"; static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite"; static const char* jkSysTableScanPhysiPlanAccountId = "AccountId"; @@ -1342,38 +1538,6 @@ static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) { return static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { return jsonToPhysicDataSinkNode(pJson, pObj); } -static const char* jkSubplanIdQueryId = "QueryId"; -static const char* jkSubplanIdGroupId = "GroupId"; -static const char* jkSubplanIdSubplanId = "SubplanId"; - -static int32_t subplanIdToJson(const void* pObj, SJson* pJson) { - const SSubplanId* pNode = (const SSubplanId*)pObj; - - int32_t code = tjsonAddIntegerToObject(pJson, jkSubplanIdQueryId, pNode->queryId); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkSubplanIdGroupId, pNode->groupId); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkSubplanIdSubplanId, pNode->subplanId); - } - - return code; -} - -static int32_t jsonToSubplanId(const SJson* pJson, void* pObj) { - SSubplanId* pNode = (SSubplanId*)pObj; - - int32_t code = tjsonGetUBigIntValue(pJson, jkSubplanIdQueryId, &pNode->queryId); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkSubplanIdGroupId, &pNode->groupId); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkSubplanIdSubplanId, &pNode->subplanId); - } - - return code; -} - static const char* jkQueryNodeAddrId = "Id"; static const char* jkQueryNodeAddrInUse = "InUse"; static const char* jkQueryNodeAddrNumOfEps = "NumOfEps"; @@ -1747,23 +1911,51 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { break; case TSDB_DATA_TYPE_BOOL: code = tjsonGetBoolValue(pJson, jkValueDatum, &pNode->datum.b); + *(bool*)&pNode->typeData = pNode->datum.b; break; case TSDB_DATA_TYPE_TINYINT: + code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int8_t*)&pNode->typeData = pNode->datum.i; + break; case TSDB_DATA_TYPE_SMALLINT: + code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int16_t*)&pNode->typeData = pNode->datum.i; + break; case TSDB_DATA_TYPE_INT: + code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int32_t*)&pNode->typeData = pNode->datum.i; + break; case TSDB_DATA_TYPE_BIGINT: + code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; case TSDB_DATA_TYPE_TIMESTAMP: code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int64_t*)&pNode->typeData = pNode->datum.i; break; case TSDB_DATA_TYPE_UTINYINT: + code = tjsonGetUBigIntValue(pJson, jkValueDatum, &pNode->datum.u); + *(uint8_t*)&pNode->typeData = pNode->datum.u; + break; case TSDB_DATA_TYPE_USMALLINT: + code = tjsonGetUBigIntValue(pJson, jkValueDatum, &pNode->datum.u); + *(uint16_t*)&pNode->typeData = pNode->datum.u; + break; case TSDB_DATA_TYPE_UINT: + code = tjsonGetUBigIntValue(pJson, jkValueDatum, &pNode->datum.u); + *(uint32_t*)&pNode->typeData = pNode->datum.u; + break; case TSDB_DATA_TYPE_UBIGINT: code = tjsonGetUBigIntValue(pJson, jkValueDatum, &pNode->datum.u); + *(uint64_t*)&pNode->typeData = pNode->datum.u; break; case TSDB_DATA_TYPE_FLOAT: + code = tjsonGetDoubleValue(pJson, jkValueDatum, &pNode->datum.d); + *(float*)&pNode->typeData = pNode->datum.d; + break; case TSDB_DATA_TYPE_DOUBLE: code = tjsonGetDoubleValue(pJson, jkValueDatum, &pNode->datum.d); + *(double*)&pNode->typeData = pNode->datum.d; break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: @@ -1964,78 +2156,6 @@ static int32_t jsonToTableNode(const SJson* pJson, void* pObj) { return code; } -static const char* jkVgroupInfoVgId = "VgId"; -static const char* jkVgroupInfoHashBegin = "HashBegin"; -static const char* jkVgroupInfoHashEnd = "HashEnd"; -static const char* jkVgroupInfoEpSet = "EpSet"; -static const char* jkVgroupInfoNumOfTable = "NumOfTable"; - -static int32_t vgroupInfoToJson(const void* pObj, SJson* pJson) { - const SVgroupInfo* pNode = (const SVgroupInfo*)pObj; - - int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupInfoVgId, pNode->vgId); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashBegin, pNode->hashBegin); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashEnd, pNode->hashEnd); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkVgroupInfoEpSet, epSetToJson, &pNode->epSet); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkVgroupInfoNumOfTable, pNode->numOfTable); - } - - return code; -} - -static int32_t jsonToVgroupInfo(const SJson* pJson, void* pObj) { - SVgroupInfo* pNode = (SVgroupInfo*)pObj; - - int32_t code = tjsonGetIntValue(pJson, jkVgroupInfoVgId, &pNode->vgId); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetUIntValue(pJson, jkVgroupInfoHashBegin, &pNode->hashBegin); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetUIntValue(pJson, jkVgroupInfoHashEnd, &pNode->hashEnd); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonToObject(pJson, jkVgroupInfoEpSet, jsonToEpSet, &pNode->epSet); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkVgroupInfoNumOfTable, &pNode->numOfTable); - } - - return code; -} - -static const char* jkVgroupsInfoNum = "Num"; -static const char* jkVgroupsInfoVgroups = "Vgroups"; - -static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) { - const SVgroupsInfo* pNode = (const SVgroupsInfo*)pObj; - - int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), - pNode->numOfVgroups); - } - - return code; -} - -static int32_t jsonToVgroupsInfo(const SJson* pJson, void* pObj) { - SVgroupsInfo* pNode = (SVgroupsInfo*)pObj; - - int32_t code = tjsonGetIntValue(pJson, jkVgroupsInfoNum, &pNode->numOfVgroups); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonToArray(pJson, jkVgroupsInfoVgroups, jsonToVgroupInfo, pNode->vgroups, sizeof(SVgroupInfo)); - } - - return code; -} - static const char* jkRealTableMetaSize = "MetaSize"; static const char* jkRealTableMeta = "Meta"; static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize"; @@ -2582,7 +2702,10 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { break; case QUERY_NODE_LOGIC_PLAN_SORT: return logicSortNodeToJson(pObj, pJson); + case QUERY_NODE_LOGIC_PLAN_PARTITION: + return logicPartitionNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_SUBPLAN: + return logicSubplanToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN: break; case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: @@ -2667,6 +2790,10 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicProjectNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SORT: return jsonToLogicSortNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_PARTITION: + return jsonToLogicPartitionNode(pJson, pObj); + case QUERY_NODE_LOGIC_SUBPLAN: + return jsonToLogicSubplan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return jsonToPhysiTagScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 8e48df284f..d7dba10718 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -871,21 +871,18 @@ void nodesClearList(SNodeList* pList) { void* nodesGetValueFromNode(SValueNode* pNode) { switch (pNode->node.resType.type) { case TSDB_DATA_TYPE_BOOL: - return (void*)&pNode->datum.b; case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - return (void*)&pNode->datum.i; case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: - return (void*)&pNode->datum.u; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: - return (void*)&pNode->datum.d; + return (void*)&pNode->typeData; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: @@ -897,6 +894,68 @@ void* nodesGetValueFromNode(SValueNode* pNode) { return NULL; } +int32_t nodesSetValueNodeValue(SValueNode* pNode, void *value) { + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: + pNode->datum.b = *(bool*)value; + *(bool*)&pNode->typeData = pNode->datum.b; + break; + case TSDB_DATA_TYPE_TINYINT: + pNode->datum.i = *(int8_t*)value; + *(int8_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_SMALLINT: + pNode->datum.i = *(int16_t*)value; + *(int16_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_INT: + pNode->datum.i = *(int32_t*)value; + *(int32_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_BIGINT: + pNode->datum.i = *(int64_t*)value; + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_TIMESTAMP: + pNode->datum.i = *(int64_t*)value; + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_UTINYINT: + pNode->datum.u = *(int8_t*)value; + *(int8_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_USMALLINT: + pNode->datum.u = *(int16_t*)value; + *(int16_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_UINT: + pNode->datum.u = *(int32_t*)value; + *(int32_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_UBIGINT: + pNode->datum.u = *(uint64_t*)value; + *(uint64_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_FLOAT: + pNode->datum.d = *(float*)value; + *(float*)&pNode->typeData = pNode->datum.d; + break; + case TSDB_DATA_TYPE_DOUBLE: + pNode->datum.d = *(double*)value; + *(double*)&pNode->typeData = pNode->datum.d; + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + pNode->datum.p = (char*)value; + break; + default: + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + char* nodesGetStrValueFromNode(SValueNode* pNode) { switch (pNode->node.resType.type) { case TSDB_DATA_TYPE_BOOL: { @@ -1090,30 +1149,31 @@ static EDealRes collectColumns(SNode* pNode, void* pContext) { int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, SNodeList** pCols) { if (NULL == pSelect || NULL == pCols) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } SCollectColumnsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pTableAlias = pTableAlias, .collectType = type, - .pCols = nodesMakeList(), + .pCols = (NULL == *pCols ? nodesMakeList() : *pCols), .pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)}; if (NULL == cxt.pCols || NULL == cxt.pColHash) { return TSDB_CODE_OUT_OF_MEMORY; } - + *pCols = NULL; nodesWalkSelectStmt(pSelect, clause, collectColumns, &cxt); taosHashCleanup(cxt.pColHash); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pCols); return cxt.errCode; } - if (0 == LIST_LENGTH(cxt.pCols)) { + if (LIST_LENGTH(cxt.pCols) > 0) { + *pCols = cxt.pCols; + } else { nodesDestroyList(cxt.pCols); - cxt.pCols = NULL; } - *pCols = cxt.pCols; + return TSDB_CODE_SUCCESS; } @@ -1134,7 +1194,7 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) { int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs) { if (NULL == pSelect || NULL == pFuncs) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } SCollectFuncsCxt cxt = { @@ -1157,6 +1217,46 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod return TSDB_CODE_SUCCESS; } +typedef struct SCollectSpecialNodesCxt { + int32_t errCode; + ENodeType type; + SNodeList* pNodes; +} SCollectSpecialNodesCxt; + +static EDealRes collectSpecialNodes(SNode* pNode, void* pContext) { + SCollectSpecialNodesCxt* pCxt = (SCollectSpecialNodesCxt*)pContext; + if (pCxt->type == nodeType(pNode)) { + pCxt->errCode = nodesListStrictAppend(pCxt->pNodes, nodesCloneNode(pNode)); + return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + return DEAL_RES_CONTINUE; +} + +int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes) { + if (NULL == pSelect || NULL == pNodes) { + return TSDB_CODE_FAILED; + } + + SCollectSpecialNodesCxt cxt = { + .errCode = TSDB_CODE_SUCCESS, .type = type, .pNodes = (NULL == *pNodes ? nodesMakeList() : *pNodes)}; + if (NULL == cxt.pNodes) { + return TSDB_CODE_OUT_OF_MEMORY; + } + *pNodes = NULL; + nodesWalkSelectStmt(pSelect, SQL_CLAUSE_GROUP_BY, collectSpecialNodes, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(cxt.pNodes); + return cxt.errCode; + } + if (LIST_LENGTH(cxt.pNodes) > 0) { + *pNodes = cxt.pNodes; + } else { + nodesDestroyList(cxt.pNodes); + } + + return TSDB_CODE_SUCCESS; +} + char* getFillModeString(EFillMode mode) { switch (mode) { case FILL_MODE_NONE: diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 39cbf5de13..8ecfab72ce 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -30,9 +30,9 @@ typedef struct SAstCreateContext { SParseContext* pQueryCxt; SMsgBuf msgBuf; bool notSupport; - bool valid; SNode* pRootNode; int16_t placeholderNo; + int32_t errCode; } SAstCreateContext; typedef enum EDatabaseOptionType { diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 9fdfe4ff7e..30f0d8ed8b 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -23,13 +23,12 @@ } %syntax_error { - if (pCxt->valid) { + if (TSDB_CODE_SUCCESS == pCxt->errCode) { if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); } - pCxt->valid = false; } } @@ -42,8 +41,8 @@ %left NK_CONCAT. /************************************************ create/alter account *****************************************/ -cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } -cmd ::= ALTER ACCOUNT NK_ID alter_account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +cmd ::= ALTER ACCOUNT NK_ID alter_account_options. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } %type account_options { int32_t } %destructor account_options { } @@ -323,7 +322,7 @@ cmd ::= SHOW QNODES. cmd ::= SHOW FUNCTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); } cmd ::= SHOW INDEXES FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, A, B); } cmd ::= SHOW STREAMS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } -cmd ::= SHOW ACCOUNTS. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +cmd ::= SHOW ACCOUNTS. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } cmd ::= SHOW APPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT, NULL, NULL); } cmd ::= SHOW CONNECTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT, NULL, NULL); } cmd ::= SHOW LICENCE. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); } @@ -491,7 +490,7 @@ signed_literal(A) ::= NK_STRING(B). signed_literal(A) ::= NK_BOOL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); } signed_literal(A) ::= TIMESTAMP NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); } signed_literal(A) ::= duration_literal(B). { A = releaseRawExprNode(pCxt, B); } -signed_literal(A) ::= NULL. { A = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } +signed_literal(A) ::= NULL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &B); } signed_literal(A) ::= literal_func(B). { A = releaseRawExprNode(pCxt, B); } %type literal_list { SNodeList* } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index c7438b0c3d..20e1bba77f 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -21,7 +21,7 @@ #define CHECK_OUT_OF_MEM(p) \ do { \ if (NULL == (p)) { \ - pCxt->valid = false; \ + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); \ return NULL; \ } \ @@ -30,7 +30,7 @@ #define CHECK_RAW_EXPR_NODE(node) \ do { \ if (NULL == (node) || QUERY_NODE_RAW_EXPR != nodeType(node)) { \ - pCxt->valid = false; \ + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; \ return NULL; \ } \ } while (0) @@ -42,9 +42,9 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { pCxt->msgBuf.buf = pParseCxt->pMsg; pCxt->msgBuf.len = pParseCxt->msgLen; pCxt->notSupport = false; - pCxt->valid = true; pCxt->pRootNode = NULL; pCxt->placeholderNo = 0; + pCxt->errCode = TSDB_CODE_SUCCESS; } static void copyStringFormStringToken(SToken* pToken, char* pBuf, int32_t len) { @@ -63,42 +63,38 @@ static void trimEscape(SToken* pName) { static bool checkUserName(SAstCreateContext* pCxt, SToken* pUserName) { if (NULL == pUserName) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { if (pUserName->n >= TSDB_USER_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } } - if (pCxt->valid) { + if (TSDB_CODE_SUCCESS == pCxt->errCode) { trimEscape(pUserName); } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, char* pPassword) { if (NULL == pPasswordToken) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else if (pPasswordToken->n >= (TSDB_USET_PASSWORD_LEN - 2)) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } else { strncpy(pPassword, pPasswordToken->z, pPasswordToken->n); strdequote(pPassword); if (strtrim(pPassword) <= 0) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkAndSplitEndpoint(SAstCreateContext* pCxt, const SToken* pEp, char* pFqdn, int32_t* pPort) { if (NULL == pEp) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else if (pEp->n >= TSDB_FQDN_LEN + 2 + 6) { // format 'fqdn:port' - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } else { char ep[TSDB_FQDN_LEN + 2 + 6]; strncpy(ep, pEp->z, pEp->n); @@ -106,66 +102,59 @@ static bool checkAndSplitEndpoint(SAstCreateContext* pCxt, const SToken* pEp, ch strtrim(ep); char* pColon = strchr(ep, ':'); if (NULL == pColon) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); } else { strncpy(pFqdn, ep, pColon - ep); *pPort = strtol(pColon + 1, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkFqdn(SAstCreateContext* pCxt, const SToken* pFqdn) { if (NULL == pFqdn) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { if (pFqdn->n >= TSDB_FQDN_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t* pPort) { if (NULL == pPortToken) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { *pPort = strtol(pPortToken->z, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool query) { if (NULL == pDbName) { if (query && NULL == pCxt->pQueryCxt->db) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED); } } else { trimEscape(pDbName); if (pDbName->n >= TSDB_DB_NAME_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pDbName->z); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pDbName->z); } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) { trimEscape(pTableName); if (NULL != pTableName && pTableName->n >= TSDB_TABLE_NAME_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z); return false; } return true; @@ -174,8 +163,7 @@ static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) { static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) { trimEscape(pColumnName); if (NULL != pColumnName && pColumnName->n >= TSDB_COL_NAME_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z); return false; } return true; @@ -184,8 +172,7 @@ static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) { static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { trimEscape(pIndexName); if (NULL != pIndexName && pIndexName->n >= TSDB_INDEX_NAME_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pIndexName->z); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pIndexName->z); return false; } return true; @@ -224,7 +211,7 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { if (NULL == pNode || QUERY_NODE_RAW_EXPR != nodeType(pNode)) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; return nil_token; } SRawExprNode* target = (SRawExprNode*)pNode; @@ -235,16 +222,12 @@ SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { SNodeList* list = nodesMakeList(); CHECK_OUT_OF_MEM(list); - if (TSDB_CODE_SUCCESS != nodesListAppend(list, pNode)) { - pCxt->valid = false; - } + pCxt->errCode = nodesListAppend(list, pNode); return list; } SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { - if (TSDB_CODE_SUCCESS != nodesListAppend(pList, pNode)) { - pCxt->valid = false; - } + pCxt->errCode = nodesListAppend(pList, pNode); return pList; } @@ -381,10 +364,10 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d strcpy(func->functionName, "cast"); func->node.resType = dt; if (TSDB_DATA_TYPE_BINARY == dt.type) { - func->node.resType.bytes += 2; + func->node.resType.bytes += 2; } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + 2; - } + } nodesListMakeAppend(&func->pParameterList, pExpr); return (SNode*)func; } @@ -531,7 +514,7 @@ SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) { } SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { - if (NULL == pNode || !pCxt->valid) { + if (NULL == pNode || TSDB_CODE_SUCCESS != pCxt->errCode) { return pNode; } int32_t len = TMIN(sizeof(((SExprNode*)pNode)->aliasName) - 1, pAlias->n); @@ -995,7 +978,7 @@ static bool needDbShowStmt(ENodeType type) { SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbNamePattern) { if (needDbShowStmt(type) && NULL == pDbName && NULL == pCxt->pQueryCxt->db) { snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "db not specified"); - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; return NULL; } SShowStmt* pStmt = nodesMakeNode(type); @@ -1256,7 +1239,7 @@ SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups) { SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) { if (pLibPath->n <= 2) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; return NULL; } SCreateFunctionStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 270cd30231..ee1a92d8b3 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -53,20 +53,20 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { } case TK_NK_ILLEGAL: { snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unrecognized token: \"%s\"", t0.z); - cxt.valid = false; + cxt.errCode = TSDB_CODE_PAR_SYNTAX_ERROR; goto abort_parse; } case TK_NK_HEX: case TK_NK_OCT: case TK_NK_BIN: { snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unsupported token: \"%s\"", t0.z); - cxt.valid = false; + cxt.errCode = TSDB_CODE_PAR_SYNTAX_ERROR; goto abort_parse; } default: Parse(pParser, t0.type, t0, &cxt); // ParseTrace(stdout, ""); - if (!cxt.valid) { + if (TSDB_CODE_SUCCESS != cxt.errCode) { goto abort_parse; } } @@ -74,7 +74,7 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { abort_parse: ParseFree(pParser, (FFree)taosMemoryFree); - if (cxt.valid) { + if (TSDB_CODE_SUCCESS == cxt.errCode) { *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; @@ -82,5 +82,5 @@ abort_parse: (*pQuery)->pRoot = cxt.pRootNode; (*pQuery)->placeholderNum = cxt.placeholderNo; } - return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; + return cxt.errCode; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index d0e670df6e..0aa4a3e6cb 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -208,6 +208,23 @@ static int32_t calcConstProjections(SCalcConstContext* pCxt, SNodeList* pProject return TSDB_CODE_SUCCESS; } +static int32_t calcConstGroupBy(SCalcConstContext* pCxt, SSelectStmt* pSelect) { + int32_t code = calcConstList(pSelect->pGroupByList); + if (TSDB_CODE_SUCCESS == code) { + SNode* pNode = NULL; + FOREACH(pNode, pSelect->pGroupByList) { + SNode* pGroupPara = NULL; + FOREACH(pGroupPara, ((SGroupingSetNode*)pNode)->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pGroupPara)) { + return code; + } + } + } + DESTORY_LIST(pSelect->pGroupByList); + } + return code; +} + static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { int32_t code = calcConstProjections(pCxt, pSelect->pProjectionList, subquery); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ec859329f5..63ad7c76ef 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -20,12 +20,12 @@ #include "functionMgt.h" #include "parUtil.h" #include "scalar.h" +#include "systable.h" #include "tglobal.h" #include "ttime.h" -#include "systable.h" #define generateDealNodeErrMsg(pCxt, code, ...) \ - (pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__) ? DEAL_RES_ERROR : DEAL_RES_ERROR) + (pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__), DEAL_RES_ERROR) typedef struct STranslateContext { SParseContext* pParseCxt; @@ -466,27 +466,66 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { break; case TSDB_DATA_TYPE_BOOL: pVal->datum.b = (0 == strcasecmp(pVal->literal, "true")); + *(bool*)&pVal->typeData = pVal->datum.b; break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_TINYINT:{ + char* endPtr = NULL; + pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + *(int8_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_SMALLINT:{ + char* endPtr = NULL; + pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + *(int16_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_INT:{ + char* endPtr = NULL; + pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + *(int32_t*)&pVal->typeData = pVal->datum.i; + break; + } case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + *(int64_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT:{ + char* endPtr = NULL; + pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + *(uint8_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_USMALLINT:{ + char* endPtr = NULL; + pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + *(uint16_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_UINT:{ + char* endPtr = NULL; + pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + *(uint32_t*)&pVal->typeData = pVal->datum.u; break; } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + *(uint64_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT:{ + char* endPtr = NULL; + pVal->datum.d = strtold(pVal->literal, &endPtr); + *(float*)&pVal->typeData = pVal->datum.d; break; } - case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { char* endPtr = NULL; pVal->datum.d = strtold(pVal->literal, &endPtr); + *(double*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_VARCHAR: @@ -504,6 +543,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } + *(int64_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_NCHAR: @@ -1676,11 +1716,12 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return TSDB_CODE_SUCCESS; } -static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, - bool alter) { +static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t daysPerFile = pOptions->daysPerFile; int32_t daysToKeep0 = pOptions->keep[0]; - if (alter && (-1 == daysPerFile || -1 == daysToKeep0)) { + if (-1 == daysPerFile && -1 == daysToKeep0) { + return TSDB_CODE_SUCCESS; + } else if (-1 == daysPerFile || -1 == daysToKeep0) { SDbCfgInfo dbCfg; int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); if (TSDB_CODE_SUCCESS != code) { @@ -1695,8 +1736,7 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa return TSDB_CODE_SUCCESS; } -static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, - bool alter) { +static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t code = checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); if (TSDB_CODE_SUCCESS == code) { @@ -1753,13 +1793,13 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRetentionsOption(pCxt, pOptions->pRetentions); } if (TSDB_CODE_SUCCESS == code) { - code = checkOptionsDependency(pCxt, pDbName, pOptions, alter); + code = checkOptionsDependency(pCxt, pDbName, pOptions); } return code; } static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) { - return checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions, false); + return checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions); } typedef int32_t (*FSerializeFunc)(void* pBuf, int32_t bufLen, void* pReq); @@ -1826,7 +1866,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, } static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) { - int32_t code = checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions, true); + int32_t code = checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -3136,11 +3176,11 @@ static int32_t rewriteShow(STranslateContext* pCxt, SQuery* pQuery) { return code; } -typedef struct SVgroupTablesBatch { +typedef struct SVgroupCreateTableBatch { SVCreateTbBatchReq req; SVgroupInfo info; char dbName[TSDB_DB_NAME_LEN]; -} SVgroupTablesBatch; +} SVgroupCreateTableBatch; static void destroyCreateTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->name); @@ -3148,7 +3188,7 @@ static void destroyCreateTbReq(SVCreateTbReq* pReq) { } static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* pStmt, const SVgroupInfo* pVgroupInfo, - SVgroupTablesBatch* pBatch) { + SVgroupCreateTableBatch* pBatch) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; strcpy(name.dbname, pStmt->dbName); @@ -3182,13 +3222,13 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* return TSDB_CODE_SUCCESS; } -static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) { +static int32_t serializeVgroupCreateTableBatch(SVgroupCreateTableBatch* pTbBatch, SArray* pBufArray) { int tlen; SCoder coder = {0}; int32_t ret = 0; tEncodeSize(tEncodeSVCreateTbBatchReq, &pTbBatch->req, tlen, ret); - tlen += sizeof(SMsgHead); //+ tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req)); + tlen += sizeof(SMsgHead); void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3214,7 +3254,7 @@ static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* return TSDB_CODE_SUCCESS; } -static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { +static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) { size_t size = taosArrayGetSize(pTbBatch->req.pArray); for (int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); @@ -3259,10 +3299,10 @@ static int32_t buildCreateTableDataBlock(int32_t acctId, const SCreateTableStmt* return TSDB_CODE_OUT_OF_MEMORY; } - SVgroupTablesBatch tbatch = {0}; - int32_t code = buildNormalTableBatchReq(acctId, pStmt, pInfo, &tbatch); + SVgroupCreateTableBatch tbatch = {0}; + int32_t code = buildNormalTableBatchReq(acctId, pStmt, pInfo, &tbatch); if (TSDB_CODE_SUCCESS == code) { - code = serializeVgroupTablesBatch(&tbatch, *pBufArray); + code = serializeVgroupCreateTableBatch(&tbatch, *pBufArray); } destroyCreateTbReqBatch(&tbatch); @@ -3307,9 +3347,9 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c req.ctb.suid = suid; req.ctb.pTag = row; - SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); + SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); if (pTableBatch == NULL) { - SVgroupTablesBatch tBatch = {0}; + SVgroupCreateTableBatch tBatch = {0}; tBatch.info = *pVgInfo; strcpy(tBatch.dbName, pDbName); @@ -3482,21 +3522,21 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla return code; } -static SArray* serializeVgroupsTablesBatch(int32_t acctId, SHashObj* pVgroupHashmap) { +static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgroupHashmap) { SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); if (NULL == pBufArray) { return NULL; } - int32_t code = TSDB_CODE_SUCCESS; - SVgroupTablesBatch* pTbBatch = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SVgroupCreateTableBatch* pTbBatch = NULL; do { pTbBatch = taosHashIterate(pVgroupHashmap, pTbBatch); if (pTbBatch == NULL) { break; } - serializeVgroupTablesBatch(pTbBatch, pBufArray); + serializeVgroupCreateTableBatch(pTbBatch, pBufArray); destroyCreateTbReqBatch(pTbBatch); } while (true); @@ -3521,7 +3561,143 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) } } - SArray* pBufArray = serializeVgroupsTablesBatch(pCxt->pParseCxt->acctId, pVgroupHashmap); + SArray* pBufArray = serializeVgroupsCreateTableBatch(pCxt->pParseCxt->acctId, pVgroupHashmap); + taosHashCleanup(pVgroupHashmap); + if (NULL == pBufArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return rewriteToVnodeModifOpStmt(pQuery, pBufArray); +} + +typedef struct SVgroupDropTableBatch { + SVDropTbBatchReq req; + SVgroupInfo info; + char dbName[TSDB_DB_NAME_LEN]; +} SVgroupDropTableBatch; + +static void addDropTbReqIntoVgroup(SHashObj* pVgroupHashmap, SDropTableClause* pClause, SVgroupInfo* pVgInfo) { + SVDropTbReq req = {.name = pClause->tableName, .igNotExists = pClause->ignoreNotExists}; + SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); + if (NULL == pTableBatch) { + SVgroupDropTableBatch tBatch = {0}; + tBatch.info = *pVgInfo; + tBatch.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); + taosArrayPush(tBatch.req.pArray, &req); + + taosHashPut(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &tBatch, sizeof(tBatch)); + } else { // add to the correct vgroup + taosArrayPush(pTableBatch->req.pArray, &req); + } +} + +static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableClause* pClause, bool* pIsSuperTable, + SHashObj* pVgroupHashmap) { + STableMeta* pTableMeta = NULL; + int32_t code = getTableMeta(pCxt, pClause->dbName, pClause->tableName, &pTableMeta); + + if (TSDB_CODE_SUCCESS == code && TSDB_SUPER_TABLE == pTableMeta->tableType) { + *pIsSuperTable = true; + goto over; + } + + *pIsSuperTable = false; + + SVgroupInfo info = {0}; + if (TSDB_CODE_SUCCESS == code) { + code = getTableHashVgroup(pCxt, pClause->dbName, pClause->tableName, &info); + } + if (TSDB_CODE_SUCCESS == code) { + addDropTbReqIntoVgroup(pVgroupHashmap, pClause, &info); + } + +over: + taosMemoryFreeClear(pTableMeta); + return code; +} + +static void destroyDropTbReqBatch(SVgroupDropTableBatch* pTbBatch) { taosArrayDestroy(pTbBatch->req.pArray); } + +static int32_t serializeVgroupDropTableBatch(SVgroupDropTableBatch* pTbBatch, SArray* pBufArray) { + int tlen; + SCoder coder = {0}; + + int32_t ret = 0; + tEncodeSize(tEncodeSVDropTbBatchReq, &pTbBatch->req, tlen, ret); + tlen += sizeof(SMsgHead); + void* buf = taosMemoryMalloc(tlen); + if (NULL == buf) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SMsgHead*)buf)->vgId = htonl(pTbBatch->info.vgId); + ((SMsgHead*)buf)->contLen = htonl(tlen); + void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, tlen - sizeof(SMsgHead), TD_ENCODER); + tEncodeSVDropTbBatchReq(&coder, &pTbBatch->req); + tCoderClear(&coder); + + SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == pVgData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pVgData->vg = pTbBatch->info; + pVgData->pData = buf; + pVgData->size = tlen; + pVgData->numOfTables = (int32_t)taosArrayGetSize(pTbBatch->req.pArray); + taosArrayPush(pBufArray, &pVgData); + + return TSDB_CODE_SUCCESS; +} + +static SArray* serializeVgroupsDropTableBatch(int32_t acctId, SHashObj* pVgroupHashmap) { + SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); + if (NULL == pBufArray) { + return NULL; + } + + int32_t code = TSDB_CODE_SUCCESS; + SVgroupDropTableBatch* pTbBatch = NULL; + do { + pTbBatch = taosHashIterate(pVgroupHashmap, pTbBatch); + if (pTbBatch == NULL) { + break; + } + + serializeVgroupDropTableBatch(pTbBatch, pBufArray); + destroyDropTbReqBatch(pTbBatch); + } while (true); + + return pBufArray; +} + +static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { + SDropTableStmt* pStmt = (SDropTableStmt*)pQuery->pRoot; + + SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pVgroupHashmap) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + bool isSuperTable = false; + SNode* pNode; + FOREACH(pNode, pStmt->pTables) { + int32_t code = buildDropTableVgroupHashmap(pCxt, (SDropTableClause*)pNode, &isSuperTable, pVgroupHashmap); + if (TSDB_CODE_SUCCESS != code) { + taosHashCleanup(pVgroupHashmap); + return code; + } + if (isSuperTable && LIST_LENGTH(pStmt->pTables) > 1) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DROP_STABLE); + } + } + + if (isSuperTable) { + taosHashCleanup(pVgroupHashmap); + return TSDB_CODE_SUCCESS; + } + + SArray* pBufArray = serializeVgroupsDropTableBatch(pCxt->pParseCxt->acctId, pVgroupHashmap); taosHashCleanup(pVgroupHashmap); if (NULL == pBufArray) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3567,6 +3743,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_CREATE_MULTI_TABLE_STMT: code = rewriteCreateMultiTable(pCxt, pQuery); break; + case QUERY_NODE_DROP_TABLE_STMT: + code = rewriteDropTable(pCxt, pQuery); + break; case QUERY_NODE_ALTER_TABLE_STMT: if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == ((SAlterTableStmt*)pQuery->pRoot)->alterType) { code = rewriteAlterTable(pCxt, pQuery); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index d753283008..2bf4071d33 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -126,6 +126,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "slimit/soffset only available for PARTITION BY query"; case TSDB_CODE_PAR_INVALID_TOPIC_QUERY: return "Invalid topic query"; + case TSDB_CODE_PAR_INVALID_DROP_STABLE: + return "Cannot drop super table in batch"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 8b60001172..4d06024c68 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -3021,11 +3021,11 @@ static YYACTIONTYPE yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ -{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +{ pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } yy_destructor(yypParser,232,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ -{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +{ pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } yy_destructor(yypParser,233,&yymsp[0].minor); break; case 2: /* account_options ::= */ @@ -3591,7 +3591,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } break; case 178: /* cmd ::= SHOW ACCOUNTS */ -{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +{ pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; case 179: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT, NULL, NULL); } @@ -3902,7 +3902,8 @@ static YYACTIONTYPE yy_reduce( yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 272: /* signed_literal ::= NULL */ -{ yymsp[0].minor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } +{ yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 291: /* expression ::= NK_LP expression NK_RP */ case 355: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==355); @@ -4352,13 +4353,12 @@ static void yy_syntax_error( #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ - if (pCxt->valid) { + if (TSDB_CODE_SUCCESS == pCxt->errCode) { if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); } - pCxt->valid = false; } /************ End %syntax_error code ******************************************/ ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index 96de3fb735..0e8adb978d 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -24,4 +24,9 @@ if(${BUILD_WINGETOPT}) PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" ) target_link_libraries(parserTest PUBLIC wingetopt) -endif() \ No newline at end of file +endif() + +add_test( + NAME parserTest + COMMAND parserTest +) diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 289888c71b..ab02ff9558 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -155,16 +155,22 @@ int32_t __catalogGetDBVgInfo(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, return 0; } +int32_t __catalogGetDBCfg(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { + return 0; +} + void initMetaDataEnv() { mockCatalogService.reset(new MockCatalogService()); static Stub stub; stub.set(catalogGetHandle, __catalogGetHandle); stub.set(catalogGetTableMeta, __catalogGetTableMeta); + stub.set(catalogGetSTableMeta, __catalogGetTableMeta); stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup); stub.set(catalogGetTableDistVgInfo, __catalogGetTableDistVgInfo); stub.set(catalogGetDBVgVersion, __catalogGetDBVgVersion); stub.set(catalogGetDBVgInfo, __catalogGetDBVgInfo); + stub.set(catalogGetDBCfg, __catalogGetDBCfg); // { // AddrAny any("libcatalog.so"); // std::map result; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index ef989e9190..2c85bfdf2e 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -111,9 +111,8 @@ class MockCatalogServiceImpl { } int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { - char db[TSDB_DB_NAME_LEN] = {0}; - tNameGetDbName(pTableName, db); - return copyTableVgroup(db, tNameGetTableName(pTableName), vgInfo); + vgInfo->vgId = 1; + return TSDB_CODE_SUCCESS; } int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** vgList) const { diff --git a/source/libs/parser/test/parExplainToSyncdbTest.cpp b/source/libs/parser/test/parExplainToSyncdbTest.cpp new file mode 100644 index 0000000000..c12912ef33 --- /dev/null +++ b/source/libs/parser/test/parExplainToSyncdbTest.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserExplainToSyncdbTest : public ParserTestBase {}; + +TEST_F(ParserExplainToSyncdbTest, explain) { + useDb("root", "test"); + + run("explain SELECT * FROM t1"); + + run("explain analyze SELECT * FROM t1"); + + run("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); +} + +// todo kill connection +// todo kill query +// todo kill stream +// todo merge vgroup +// todo redistribute vgroup +// todo reset query cache +// todo syncdb + +} // namespace ParserTest diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp new file mode 100644 index 0000000000..e58f50f52a --- /dev/null +++ b/source/libs/parser/test/parInitialATest.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserInitialATest : public ParserTestBase {}; + +TEST_F(ParserInitialATest, alterAccount) { + useDb("root", "test"); + + run("alter account ac_wxy pass '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT); +} + +TEST_F(ParserInitialATest, alterDnode) { + useDb("root", "test"); + + run("alter dnode 1 'resetLog'"); + + run("alter dnode 1 'debugFlag' '134'"); +} + +TEST_F(ParserInitialATest, alterDatabase) { + useDb("root", "test"); + + run("alter database wxy_db cachelast 1 fsync 200 wal 1"); +} + +// todo alter local +// todo alter stable +// todo alter table + +TEST_F(ParserInitialATest, alterUser) { + useDb("root", "test"); + + run("alter user wxy pass '123456'"); + + run("alter user wxy privilege 'write'"); +} + +TEST_F(ParserInitialATest, bug001) { + useDb("root", "test"); + + run("alter database db wal 0 # td-14436", TSDB_CODE_PAR_SYNTAX_ERROR); +} + +} // namespace ParserTest \ No newline at end of file diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp new file mode 100644 index 0000000000..540560fcd7 --- /dev/null +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserInitialCTest : public ParserTestBase {}; + +// todo compact + +TEST_F(ParserInitialCTest, createAccount) { + useDb("root", "test"); + + run("create account ac_wxy pass '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT); +} + +TEST_F(ParserInitialCTest, createBnode) { + useDb("root", "test"); + + run("create bnode on dnode 1"); +} + +TEST_F(ParserInitialCTest, createDatabase) { + useDb("root", "test"); + + run("create database wxy_db"); + + run("create database if not exists wxy_db " + "cachelast 2 " + "comp 1 " + "days 100 " + "fsync 100 " + "maxrows 1000 " + "minrows 100 " + "keep 1440 " + "precision 'ms' " + "replica 3 " + "wal 2 " + "vgroups 100 " + "single_stable 0 " + "retentions 15s:7d,1m:21d,15m:5y"); + + run("create database if not exists wxy_db " + "days 100m " + "keep 1440m,300h,400d "); +} + +TEST_F(ParserInitialCTest, createDnode) { + useDb("root", "test"); + + run("create dnode abc1 port 7000"); + + run("create dnode 1.1.1.1 port 9000"); +} + +// todo create function + +TEST_F(ParserInitialCTest, createIndexSma) { + useDb("root", "test"); + + run("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); +} + +TEST_F(ParserInitialCTest, createMnode) { + useDb("root", "test"); + + run("create mnode on dnode 1"); +} + +TEST_F(ParserInitialCTest, createQnode) { + useDb("root", "test"); + + run("create qnode on dnode 1"); +} + +TEST_F(ParserInitialCTest, createSnode) { + useDb("root", "test"); + + run("create snode on dnode 1"); +} + +TEST_F(ParserInitialCTest, createStable) { + useDb("root", "test"); + + run("create stable t1(ts timestamp, c1 int) TAGS(id int)"); + + run("create stable if not exists test.t1(" + "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " + "SMALLINT, " + "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), " + "c15 VARCHAR(50)) " + "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 " + "BINARY(20), a8 SMALLINT, " + "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " + "a15 VARCHAR(50)) " + "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); +} + +TEST_F(ParserInitialCTest, createStream) { + useDb("root", "test"); + + run("create stream s1 as select * from t1"); + + run("create stream if not exists s1 as select * from t1"); + + run("create stream s1 into st1 as select * from t1"); + + run("create stream if not exists s1 trigger window_close watermark 10s into st1 as select * from t1"); +} + +TEST_F(ParserInitialCTest, createTable) { + useDb("root", "test"); + + run("create table t1(ts timestamp, c1 int)"); + + run("create table if not exists test.t1(" + "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " + "SMALLINT, " + "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 " + "NCHAR(30), " + "c15 VARCHAR(50)) " + "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)"); + + run("create table if not exists test.t1(" + "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " + "SMALLINT, " + "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 " + "NCHAR(30), " + "c15 VARCHAR(50)) " + "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, " + "a5 FLOAT, a6 DOUBLE, a7 " + "BINARY(20), a8 SMALLINT, " + "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 " + "TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " + "a15 VARCHAR(50)) " + "TTL 100 COMMENT 'test create " + "table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); + + run("create table if not exists t1 using st1 tags(1, 'wxy')"); + + run("create table " + "if not exists test.t1 using test.st1 (tag1, tag2) tags(1, 'abc') " + "if not exists test.t2 using test.st1 (tag1, tag2) tags(2, 'abc') " + "if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc') "); +} + +TEST_F(ParserInitialCTest, createTopic) { + useDb("root", "test"); + + run("create topic tp1 as select * from t1"); + + run("create topic if not exists tp1 as select * from t1"); + + run("create topic tp1 as test"); + + run("create topic if not exists tp1 as test"); +} + +TEST_F(ParserInitialCTest, createUser) { + useDb("root", "test"); + + run("create user wxy pass '123456'"); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp new file mode 100644 index 0000000000..1153b238b1 --- /dev/null +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserInitialDTest : public ParserTestBase {}; + +// todo delete +// todo desc +// todo describe +// todo drop account + +TEST_F(ParserInitialDTest, dropBnode) { + useDb("root", "test"); + + run("drop bnode on dnode 1"); +} + +// todo drop database +// todo drop dnode +// todo drop function + +TEST_F(ParserInitialDTest, dropIndex) { + useDb("root", "test"); + + run("drop index index1 on t1"); +} + +TEST_F(ParserInitialDTest, dropMnode) { + useDb("root", "test"); + + run("drop mnode on dnode 1"); +} + +TEST_F(ParserInitialDTest, dropQnode) { + useDb("root", "test"); + + run("drop qnode on dnode 1"); +} + +TEST_F(ParserInitialDTest, dropSnode) { + useDb("root", "test"); + + run("drop snode on dnode 1"); +} + +// todo drop stable +// todo drop stream +// todo drop table + +TEST_F(ParserInitialDTest, dropTopic) { + useDb("root", "test"); + + run("drop topic tp1"); + + run("drop topic if exists tp1"); +} + +TEST_F(ParserInitialDTest, dropUser) { + useDb("root", "test"); + + run("drop user wxy"); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parInsertTest.cpp b/source/libs/parser/test/parInsertTest.cpp index 308152bf3e..7fafec8882 100644 --- a/source/libs/parser/test/parInsertTest.cpp +++ b/source/libs/parser/test/parInsertTest.cpp @@ -111,6 +111,7 @@ class InsertTest : public Test { cxt_.pMsg = errMagBuf_; cxt_.msgLen = max_err_len; code_ = TSDB_CODE_SUCCESS; + res_ = nullptr; } SVnodeModifOpStmt* getVnodeModifStmt(SQuery* pQuery) { return (SVnodeModifOpStmt*)pQuery->pRoot; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp new file mode 100644 index 0000000000..990c272612 --- /dev/null +++ b/source/libs/parser/test/parSelectTest.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserSelectTest : public ParserTestBase {}; + +TEST_F(ParserSelectTest, basic) { + useDb("root", "test"); + + run("select * from t1"); + + run("select * from test.t1"); + + run("select ts, c1 from t1"); + + run("select ts, t.c1 from (select * from t1) t"); + + run("select * from t1 tt1, t1 tt2 where tt1.c1 = tt2.c1"); +} + +TEST_F(ParserSelectTest, constant) { + useDb("root", "test"); + + run("select 123, 20.4, 'abc', \"wxy\", timestamp '2022-02-09 17:30:20', true, false, 10s from t1"); + + run("select 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", " + "timestamp '2022-02-09 17:30:20', true, false, 15s from t1"); + + run("select 123 + 45 from t1 where 2 - 1"); +} + +TEST_F(ParserSelectTest, expression) { + useDb("root", "test"); + + run("select ts + 10s, c1 + 10, concat(c2, 'abc') from t1"); + + run("select ts > 0, c1 < 20 and c2 = 'qaz' from t1"); + + run("select ts > 0, c1 between 10 and 20 and c2 = 'qaz' from t1"); +} + +TEST_F(ParserSelectTest, condition) { + useDb("root", "test"); + + run("select c1 from t1 where ts in (true, false)"); + + run("select * from t1 where c1 > 10 and c1 is not null"); +} + +TEST_F(ParserSelectTest, pseudoColumn) { + useDb("root", "test"); + + run("select _wstartts, _wendts, count(*) from t1 interval(10s)"); +} + +TEST_F(ParserSelectTest, multiResFunc) { + useDb("root", "test"); + + run("select last(*), first(*), last_row(*) from t1"); + + run("select last(c1, c2), first(t1.*), last_row(c3) from t1"); + + run("select last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); +} + +TEST_F(ParserSelectTest, clause) { + useDb("root", "test"); + + // group by clause + run("select count(*) cnt from t1 where c1 > 0"); + + run("select count(*), c2 cnt from t1 where c1 > 0 group by c2"); + + run("select count(*) cnt from t1 where c1 > 0 group by c2 having count(c1) > 10"); + + run("select count(*), c1, c2 + 10, c1 + c2 cnt from t1 where c1 > 0 group by c2, c1"); + + run("select count(*), c1 + 10, c2 cnt from t1 where c1 > 0 group by c1 + 10, c2"); + + // order by clause + run("select count(*) cnt from t1 where c1 > 0 group by c2 order by cnt"); + + run("select count(*) cnt from t1 where c1 > 0 group by c2 order by 1"); + + // distinct clause + // run("select distinct c1, c2 from t1 where c1 > 0 order by c1"); + + // run("select distinct c1 + 10, c2 from t1 where c1 > 0 order by c1 + 10, c2"); + + // run("select distinct c1 + 10 cc1, c2 cc2 from t1 where c1 > 0 order by cc1, c2"); + + // run("select distinct count(c2) from t1 where c1 > 0 group by c1 order by count(c2)"); +} + +TEST_F(ParserSelectTest, window) { + useDb("root", "test"); + + run("select count(*) from t1 interval(10s)"); +} + +TEST_F(ParserSelectTest, semanticError) { + useDb("root", "test"); + + // TSDB_CODE_PAR_INVALID_COLUMN + run("select c1, cc1 from t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); + + run("select t1.c1, t1.cc1 from t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_TABLE_NOT_EXIST + run("select * from t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + + run("select * from test.t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + + run("select t2.c1 from t1", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_AMBIGUOUS_COLUMN + run("select c2 from t1 tt1, t1 tt2 where tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_WRONG_VALUE_TYPE + run("select timestamp '2010' from t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION + run("select c2 from t1 tt1 join t1 tt2 on count(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, + PARSER_STAGE_TRANSLATE); + + run("select c2 from t1 where count(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + + run("select c2 from t1 group by count(*)", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT + run("select c2 from t1 order by 0", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); + + run("select c2 from t1 order by 2", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION + run("select count(*) cnt from t1 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); + + run("select count(*) cnt from t1 group by c2 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + run("select count(*), c1 cnt from t1 group by c2 having c2 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + run("select count(*) cnt from t1 group by c2 having c2 > 0 order by c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_NOT_SINGLE_GROUP + run("select count(*), c1 from t1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + + run("select count(*) from t1 order by c1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + + run("select c1 from t1 order by count(*)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION + run("select distinct c1, c2 from t1 where c1 > 0 order by ts", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + run("select distinct c1 from t1 where c1 > 0 order by count(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + run("select distinct c2 from t1 where c1 > 0 order by count(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + PARSER_STAGE_TRANSLATE); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parShowToUse.cpp b/source/libs/parser/test/parShowToUse.cpp new file mode 100644 index 0000000000..37d5d87a88 --- /dev/null +++ b/source/libs/parser/test/parShowToUse.cpp @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserShowToUseTest : public ParserTestBase {}; + +// todo show accounts +// todo show apps +// todo show connections +// todo show create database +// todo show create stable +// todo show create table + +TEST_F(ParserShowToUseTest, showDatabases) { + useDb("root", "test"); + + run("show databases"); +} + +TEST_F(ParserShowToUseTest, showDnodes) { + useDb("root", "test"); + + run("show dnodes"); +} + +TEST_F(ParserShowToUseTest, showFunctions) { + useDb("root", "test"); + + run("show functions"); +} + +// todo show licence + +TEST_F(ParserShowToUseTest, showIndexes) { + useDb("root", "test"); + + run("show indexes from t1"); + + run("show indexes from t1 from test"); +} + +TEST_F(ParserShowToUseTest, showMnodes) { + useDb("root", "test"); + + run("show mnodes"); +} + +TEST_F(ParserShowToUseTest, showModules) { + useDb("root", "test"); + + run("show modules"); +} + +TEST_F(ParserShowToUseTest, showQnodes) { + useDb("root", "test"); + + run("show qnodes"); +} + +// todo show queries +// todo show scores + +TEST_F(ParserShowToUseTest, showStables) { + useDb("root", "test"); + + run("show stables"); + + run("show test.stables"); + + run("show stables like 'c%'"); + + run("show test.stables like 'c%'"); +} + +TEST_F(ParserShowToUseTest, showStreams) { + useDb("root", "test"); + + run("show streams"); +} + +TEST_F(ParserShowToUseTest, showTables) { + useDb("root", "test"); + + run("show tables"); + + run("show test.tables"); + + run("show tables like 'c%'"); + + run("show test.tables like 'c%'"); +} + +// todo show topics + +TEST_F(ParserShowToUseTest, showUsers) { + useDb("root", "test"); + + run("show users"); +} + +// todo show variables + +TEST_F(ParserShowToUseTest, showVgroups) { + useDb("root", "test"); + + run("show vgroups"); + + run("show test.vgroups"); +} + +// todo show vnodes + +// todo split vgroup + +TEST_F(ParserShowToUseTest, useDatabase) { + useDb("root", "test"); + + run("use wxy_db"); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 28312d80a8..ebc83fb219 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -30,6 +30,8 @@ #include "parTestUtil.h" #include "parToken.h" +namespace ParserTest { + class ParserEnv : public testing::Environment { public: virtual void SetUp() { @@ -62,9 +64,11 @@ static void parseArg(int argc, char* argv[]) { } } +} // namespace ParserTest + int main(int argc, char* argv[]) { - testing::AddGlobalTestEnvironment(new ParserEnv()); + testing::AddGlobalTestEnvironment(new ParserTest::ParserEnv()); testing::InitGoogleTest(&argc, argv); - parseArg(argc, argv); + ParserTest::parseArg(argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index 8e7123ee95..a9ff756a95 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -23,17 +23,30 @@ using namespace std; using namespace testing; -#define DO_WITH_THROW(func, ...) \ - do { \ - int32_t code__ = func(__VA_ARGS__); \ - if (TSDB_CODE_SUCCESS != code__) { \ - throw runtime_error("sql:[" + stmtEnv_.sql_ + "] " #func " code:" + to_string(code__) + \ - ", strerror:" + string(tstrerror(code__)) + ", msg:" + string(stmtEnv_.msgBuf_.data())); \ - } \ +namespace ParserTest { + +#define DO_WITH_THROW(func, ...) \ + do { \ + int32_t code__ = func(__VA_ARGS__); \ + if (!checkResultCode(#func, code__)) { \ + if (TSDB_CODE_SUCCESS != code__) { \ + throw runtime_error("sql:[" + stmtEnv_.sql_ + "] " #func " code:" + to_string(code__) + \ + ", strerror:" + string(tstrerror(code__)) + ", msg:" + string(stmtEnv_.msgBuf_.data())); \ + } else { \ + throw runtime_error("sql:[" + stmtEnv_.sql_ + "] " #func " expect " + to_string(stmtEnv_.expect_) + \ + " actual " + to_string(code__)); \ + } \ + } else if (TSDB_CODE_SUCCESS != code__) { \ + throw TerminateFlag(); \ + } \ } while (0); bool g_isDump = false; +struct TerminateFlag : public exception { + const char* what() const throw() { return "success and terminate"; } +}; + class ParserTestBaseImpl { public: void useDb(const string& acctId, const string& db) { @@ -41,8 +54,8 @@ class ParserTestBaseImpl { caseEnv_.db_ = db; } - void run(const string& sql) { - reset(); + void run(const string& sql, int32_t expect, ParserStage checkStage) { + reset(expect, checkStage); try { SParseContext cxt = {0}; setParseContext(sql, &cxt); @@ -57,6 +70,9 @@ class ParserTestBaseImpl { if (g_isDump) { dump(); } + } catch (const TerminateFlag& e) { + // success and terminate + return; } catch (...) { dump(); throw; @@ -72,6 +88,8 @@ class ParserTestBaseImpl { struct stmtEnv { string sql_; array msgBuf_; + int32_t expect_; + string checkFunc_; }; struct stmtRes { @@ -80,9 +98,34 @@ class ParserTestBaseImpl { string calcConstAst_; }; - void reset() { + bool checkResultCode(const string& pFunc, int32_t resultCode) { + return !(stmtEnv_.checkFunc_.empty()) + ? (("*" == stmtEnv_.checkFunc_ || stmtEnv_.checkFunc_ == pFunc) ? stmtEnv_.expect_ == resultCode + : TSDB_CODE_SUCCESS == resultCode) + : true; + } + + string stageFunc(ParserStage stage) { + switch (stage) { + case PARSER_STAGE_PARSE: + return "parse"; + case PARSER_STAGE_TRANSLATE: + return "translate"; + case PARSER_STAGE_CALC_CONST: + return "calculateConstant"; + case PARSER_STAGE_ALL: + return "*"; + default: + break; + } + return "unknown"; + } + + void reset(int32_t expect, ParserStage checkStage) { stmtEnv_.sql_.clear(); stmtEnv_.msgBuf_.fill(0); + stmtEnv_.expect_ = expect; + stmtEnv_.checkFunc_ = stageFunc(checkStage); res_.parsedAst_.clear(); res_.translatedAst_.clear(); @@ -146,4 +189,8 @@ ParserTestBase::~ParserTestBase() {} void ParserTestBase::useDb(const std::string& acctId, const std::string& db) { impl_->useDb(acctId, db); } -void ParserTestBase::run(const std::string& sql) { return impl_->run(sql); } +void ParserTestBase::run(const std::string& sql, int32_t expect, ParserStage checkStage) { + return impl_->run(sql, expect, checkStage); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parTestUtil.h b/source/libs/parser/test/parTestUtil.h index de3b16aabc..0e8703b2c8 100644 --- a/source/libs/parser/test/parTestUtil.h +++ b/source/libs/parser/test/parTestUtil.h @@ -18,15 +18,21 @@ #include +#include "taoserror.h" + +namespace ParserTest { + class ParserTestBaseImpl; +enum ParserStage { PARSER_STAGE_PARSE, PARSER_STAGE_TRANSLATE, PARSER_STAGE_CALC_CONST, PARSER_STAGE_ALL }; + class ParserTestBase : public testing::Test { public: ParserTestBase(); virtual ~ParserTestBase(); void useDb(const std::string& acctId, const std::string& db); - void run(const std::string& sql); + void run(const std::string& sql, int32_t expect = TSDB_CODE_SUCCESS, ParserStage checkStage = PARSER_STAGE_ALL); private: std::unique_ptr impl_; @@ -34,4 +40,6 @@ class ParserTestBase : public testing::Test { extern bool g_isDump; +} // namespace ParserTest + #endif // PARSER_TEST_UTIL_H diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp deleted file mode 100644 index 3b00e696cc..0000000000 --- a/source/libs/parser/test/parserAstTest.cpp +++ /dev/null @@ -1,782 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include - -#include - -#include "parInt.h" -#include "parTestUtil.h" - -using namespace std; -using namespace testing; - -class ParserTest : public Test { - protected: - void setDatabase(const string& acctId, const string& db) { - acctId_ = acctId; - db_ = db; - } - - void bind(const char* sql) { - reset(); - cxt_.acctId = atoi(acctId_.c_str()); - cxt_.db = db_.c_str(); - sqlBuf_ = string(sql); - transform(sqlBuf_.begin(), sqlBuf_.end(), sqlBuf_.begin(), ::tolower); - cxt_.sqlLen = strlen(sql); - cxt_.pSql = sqlBuf_.c_str(); - } - - bool run(int32_t parseCode = TSDB_CODE_SUCCESS, int32_t translateCode = TSDB_CODE_SUCCESS) { - query_ = nullptr; - bool res = runImpl(parseCode, translateCode); - qDestroyQuery(query_); - if (!res || g_isDump) { - dump(); - } - return res; - } - - private: - static const int max_err_len = 1024; - - bool runImpl(int32_t parseCode, int32_t translateCode) { - int32_t code = parse(&cxt_, &query_); - if (code != TSDB_CODE_SUCCESS) { - parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; - return (code == parseCode); - } - if (TSDB_CODE_SUCCESS != parseCode) { - return false; - } - parsedAstStr_ = toString(query_->pRoot); - code = translate(&cxt_, query_); - if (code != TSDB_CODE_SUCCESS) { - translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; - return (code == translateCode); - } - translatedAstStr_ = toString(query_->pRoot); - code = calculateConstant(&cxt_, query_); - if (code != TSDB_CODE_SUCCESS) { - calcConstErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; - return false; - } - calcConstAstStr_ = toString(query_->pRoot); - return (TSDB_CODE_SUCCESS == translateCode); - } - - void dump() { - cout << "input sql : [" << cxt_.pSql << "]" << endl; - if (!parseErrStr_.empty()) { - cout << "parse error: " << parseErrStr_ << endl; - } - if (!parsedAstStr_.empty()) { - cout << "parse output: " << endl; - cout << parsedAstStr_ << endl; - } - if (!translateErrStr_.empty()) { - cout << "translate error: " << translateErrStr_ << endl; - } - if (!translatedAstStr_.empty()) { - cout << "translate output: " << endl; - cout << translatedAstStr_ << endl; - } - if (!calcConstErrStr_.empty()) { - cout << "calculateConstant error: " << calcConstErrStr_ << endl; - } - if (!calcConstAstStr_.empty()) { - cout << "calculateConstant output: " << endl; - cout << calcConstAstStr_ << endl; - } - } - - string toString(const SNode* pRoot, bool format = false) { - char* pStr = NULL; - int32_t len = 0; - int32_t code = nodesNodeToString(pRoot, format, &pStr, &len); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl; - throw "nodesNodeToString failed!"; - } - string str(pStr); - taosMemoryFreeClear(pStr); - return str; - } - - void reset() { - memset(&cxt_, 0, sizeof(cxt_)); - memset(errMagBuf_, 0, max_err_len); - cxt_.pMsg = errMagBuf_; - cxt_.msgLen = max_err_len; - parseErrStr_.clear(); - parsedAstStr_.clear(); - translateErrStr_.clear(); - translatedAstStr_.clear(); - calcConstErrStr_.clear(); - calcConstAstStr_.clear(); - } - - string acctId_; - string db_; - char errMagBuf_[max_err_len]; - string sqlBuf_; - SParseContext cxt_; - SQuery* query_; - string parseErrStr_; - string parsedAstStr_; - string translateErrStr_; - string translatedAstStr_; - string calcConstErrStr_; - string calcConstAstStr_; -}; - -TEST_F(ParserTest, createAccount) { - setDatabase("root", "test"); - - bind("create account ac_wxy pass '123456'"); - ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); -} - -TEST_F(ParserTest, alterAccount) { - setDatabase("root", "test"); - - bind("alter account ac_wxy pass '123456'"); - ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); -} - -TEST_F(ParserTest, createUser) { - setDatabase("root", "test"); - - bind("create user wxy pass '123456'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, alterUser) { - setDatabase("root", "test"); - - bind("alter user wxy pass '123456'"); - ASSERT_TRUE(run()); - - bind("alter user wxy privilege 'write'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropUser) { - setDatabase("root", "test"); - - bind("drop user wxy"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectSimple) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM test.t1"); - ASSERT_TRUE(run()); - - bind("SELECT ts, c1 FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT ts, t.c1 FROM (SELECT * FROM t1) t"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectConstant) { - setDatabase("root", "test"); - - bind("SELECT 123, 20.4, 'abc', \"wxy\", TIMESTAMP '2022-02-09 17:30:20', true, false, 10s FROM t1"); - ASSERT_TRUE(run()); - - bind( - "SELECT 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", " - "TIMESTAMP '2022-02-09 17:30:20', true, false, 15s FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT 123 + 45 FROM t1 where 2 - 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectExpression) { - setDatabase("root", "test"); - - bind("SELECT ts + 10s, c1 + 10, concat(c2, 'abc') FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT ts > 0, c1 < 20 AND c2 = 'qaz' FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT ts > 0, c1 BETWEEN 10 AND 20 AND c2 = 'qaz' FROM t1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectCondition) { - setDatabase("root", "test"); - - bind("SELECT c1 FROM t1 where ts in (true, false)"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 where c1 > 10 and c1 is not null"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectPseudoColumn) { - setDatabase("root", "test"); - - bind("SELECT _wstartts, _wendts, count(*) FROM t1 interval(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectMultiResFunc) { - setDatabase("root", "test"); - - // bind("SELECT last(*), first(*), last_row(*) FROM t1"); - // ASSERT_TRUE(run()); - - bind("SELECT last(c1, c2), first(t1.*), last_row(c3) FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectClause) { - setDatabase("root", "test"); - - // GROUP BY clause - bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0"); - ASSERT_TRUE(run()); - - bind("SELECT count(*), c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 HAVING count(c1) > 10"); - ASSERT_TRUE(run()); - - bind("SELECT count(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2, c1"); - ASSERT_TRUE(run()); - - bind("SELECT count(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); - ASSERT_TRUE(run()); - - // ORDER BY clause - bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY cnt"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY 1"); - ASSERT_TRUE(run()); - - // DISTINCT clause - bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY c1"); - ASSERT_TRUE(run()); - - bind("SELECT DISTINCT c1 + 10, c2 FROM t1 WHERE c1 > 0 ORDER BY c1 + 10, c2"); - ASSERT_TRUE(run()); - - bind("SELECT DISTINCT c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 ORDER BY cc1, c2"); - ASSERT_TRUE(run()); - - bind("SELECT DISTINCT count(c2) FROM t1 WHERE c1 > 0 GROUP BY c1 ORDER BY count(c2)"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectWindow) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1 interval(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectSyntaxError) { - setDatabase("root", "test"); - - bind("SELECTT * FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_FAILED)); - - bind("SELECT *"); - ASSERT_TRUE(run(TSDB_CODE_FAILED)); - - bind("SELECT *, * FROM test.t1"); - ASSERT_TRUE(run(TSDB_CODE_FAILED)); - - bind("SELECT * FROM test.t1 t WHER"); - ASSERT_TRUE(run(TSDB_CODE_FAILED)); -} - -TEST_F(ParserTest, selectSemanticError) { - setDatabase("root", "test"); - - // TSDB_CODE_PAR_INVALID_COLUMN - bind("SELECT c1, cc1 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN)); - - bind("SELECT t1.c1, t1.cc1 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN)); - - // TSDB_CODE_PAR_TABLE_NOT_EXIST - bind("SELECT * FROM t10"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); - - bind("SELECT * FROM test.t10"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); - - bind("SELECT t2.c1 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); - - // TSDB_CODE_PAR_AMBIGUOUS_COLUMN - bind("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_AMBIGUOUS_COLUMN)); - - // TSDB_CODE_PAR_WRONG_VALUE_TYPE - bind("SELECT 10n FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE)); - - bind("SELECT TIMESTAMP '2010' FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE)); - - // TSDB_CODE_PAR_INVALID_FUNTION - bind("SELECT cnt(*) FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FUNC_INVALID_FUNTION)); - - // TSDB_CODE_PAR_FUNTION_PARA_NUM - // TSDB_CODE_PAR_FUNTION_PARA_TYPE - - // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION - bind("SELECT c2 FROM t1 tt1 JOIN t1 tt2 ON count(*) > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); - - bind("SELECT c2 FROM t1 where count(*) > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); - - bind("SELECT c2 FROM t1 GROUP BY count(*)"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); - - // TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT - bind("SELECT c2 FROM t1 ORDER BY 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT)); - - bind("SELECT c2 FROM t1 ORDER BY 2"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT)); - - // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION - bind("SELECT count(*) cnt FROM t1 HAVING c1 > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); - - bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c1 > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); - - bind("SELECT count(*), c1 cnt FROM t1 GROUP BY c2 HAVING c2 > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); - - bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c2 > 0 ORDER BY c1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); - - // TSDB_CODE_PAR_NOT_SINGLE_GROUP - bind("SELECT count(*), c1 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); - - bind("SELECT count(*) FROM t1 ORDER BY c1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); - - bind("SELECT c1 FROM t1 ORDER BY count(*)"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); - - // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION - bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY ts"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); - - bind("SELECT DISTINCT c1 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); - - bind("SELECT DISTINCT c2 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); -} - -TEST_F(ParserTest, showUsers) { - setDatabase("root", "test"); - - bind("show users"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createDnode) { - setDatabase("root", "test"); - - bind("create dnode abc1 port 7000"); - ASSERT_TRUE(run()); - - bind("create dnode 1.1.1.1 port 9000"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showDnodes) { - setDatabase("root", "test"); - - bind("show dnodes"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, alterDnode) { - setDatabase("root", "test"); - - bind("alter dnode 1 'resetLog'"); - ASSERT_TRUE(run()); - - bind("alter dnode 1 'debugFlag' '134'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createDatabase) { - setDatabase("root", "test"); - - bind("create database wxy_db"); - ASSERT_TRUE(run()); - - bind( - "create database if not exists wxy_db " - "BLOCKS 100 " - "CACHE 100 " - "CACHELAST 2 " - "COMP 1 " - "DAYS 100 " - "FSYNC 100 " - "MAXROWS 1000 " - "MINROWS 100 " - "KEEP 100 " - "PRECISION 'ms' " - "QUORUM 1 " - "REPLICA 3 " - "TTL 100 " - "WAL 2 " - "VGROUPS 100 " - "SINGLE_STABLE 0 " - "STREAM_MODE 1 " - "RETENTIONS '15s:7d,1m:21d,15m:5y'"); - ASSERT_TRUE(run()); - - bind( - "create database if not exists wxy_db " - "DAYS 100m " - "KEEP 200m,300h,400d "); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, alterDatabase) { - setDatabase("root", "test"); - - bind("alter database wxy_db BLOCKS 200"); - ASSERT_TRUE(run()); - - bind( - "alter database wxy_db " - "BLOCKS 200 " - "CACHELAST 1 " - "FSYNC 200 " - "KEEP 200 " - "QUORUM 2 " - "WAL 1 "); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showDatabase) { - setDatabase("root", "test"); - - bind("show databases"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, useDatabase) { - setDatabase("root", "test"); - - bind("use wxy_db"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createTable) { - setDatabase("root", "test"); - - bind("create table t1(ts timestamp, c1 int)"); - ASSERT_TRUE(run()); - - bind( - "create table if not exists test.t1(" - "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " - "SMALLINT, " - "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), " - "c14 JSON, c15 VARCHAR(50)) " - "KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)"); - ASSERT_TRUE(run()); - - bind( - "create table if not exists test.t1(" - "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " - "SMALLINT, " - "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), " - "c14 JSON, c15 VARCHAR(50)) " - "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 " - "BINARY(20), a8 SMALLINT, " - "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " - "a14 JSON, a15 VARCHAR(50)) " - "KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); - ASSERT_TRUE(run()); - - bind("create table if not exists t1 using st1 tags(1, 'wxy')"); - ASSERT_TRUE(run()); - - bind( - "create table " - "if not exists test.t1 using test.st1 (tag1, tag2) tags(1, 'abc') " - "if not exists test.t2 using test.st1 (tag1, tag2) tags(2, 'abc') " - "if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc') " - "if not exists test.t4 using test.st1 (tag1, tag2) tags(3, null) " - "if not exists test.t5 using test.st1 (tag1, tag2) tags(null, 'abc') " - "if not exists test.t6 using test.st1 (tag1, tag2) tags(null, null)"); - ASSERT_TRUE(run()); - - bind("create stable t1(ts timestamp, c1 int) TAGS(id int)"); - ASSERT_TRUE(run()); - - bind( - "create stable if not exists test.t1(" - "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " - "SMALLINT, " - "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), " - "c14 JSON, c15 VARCHAR(50)) " - "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 " - "BINARY(20), a8 SMALLINT, " - "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " - "a14 JSON, a15 VARCHAR(50)) " - "KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showTables) { - setDatabase("root", "test"); - - bind("show tables"); - ASSERT_TRUE(run()); - - bind("show test.tables"); - ASSERT_TRUE(run()); - - bind("show tables like 'c%'"); - ASSERT_TRUE(run()); - - bind("show test.tables like 'c%'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showStables) { - setDatabase("root", "test"); - - bind("show stables"); - ASSERT_TRUE(run()); - - bind("show test.stables"); - ASSERT_TRUE(run()); - - bind("show stables like 'c%'"); - ASSERT_TRUE(run()); - - bind("show test.stables like 'c%'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showVgroups) { - setDatabase("root", "test"); - - bind("show vgroups"); - ASSERT_TRUE(run()); - - bind("show test.vgroups"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showMnodes) { - setDatabase("root", "test"); - - bind("show mnodes"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showModules) { - setDatabase("root", "test"); - - bind("show modules"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showQnodes) { - setDatabase("root", "test"); - - bind("show qnodes"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showFunctions) { - setDatabase("root", "test"); - - bind("show functions"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showIndexes) { - setDatabase("root", "test"); - - bind("show indexes from t1"); - ASSERT_TRUE(run()); - - bind("show indexes from t1 from test"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showStreams) { - setDatabase("root", "test"); - - bind("show streams"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createSmaIndex) { - setDatabase("root", "test"); - - bind("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropIndex) { - setDatabase("root", "test"); - - bind("drop index index1 on t1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createQnode) { - setDatabase("root", "test"); - - bind("create qnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropQnode) { - setDatabase("root", "test"); - - bind("drop qnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createBnode) { - setDatabase("root", "test"); - - bind("create bnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropBnode) { - setDatabase("root", "test"); - - bind("drop bnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createSnode) { - setDatabase("root", "test"); - - bind("create snode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropSnode) { - setDatabase("root", "test"); - - bind("drop snode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createMnode) { - setDatabase("root", "test"); - - bind("create mnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropMnode) { - setDatabase("root", "test"); - - bind("drop mnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createTopic) { - setDatabase("root", "test"); - - bind("create topic tp1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create topic if not exists tp1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create topic tp1 as test"); - ASSERT_TRUE(run()); - - bind("create topic if not exists tp1 as test"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropTopic) { - setDatabase("root", "test"); - - bind("drop topic tp1"); - ASSERT_TRUE(run()); - - bind("drop topic if exists tp1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createStream) { - setDatabase("root", "test"); - - bind("create stream s1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create stream if not exists s1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create stream s1 into st1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create stream if not exists s1 trigger window_close watermark 10s into st1 as select * from t1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, explain) { - setDatabase("root", "test"); - - bind("explain SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("explain analyze SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); - ASSERT_TRUE(run()); -} diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 18e59859ac..00f2808294 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -748,10 +748,10 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele SLogicNode* pRoot = NULL; int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot); if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot); + code = createChildLogicNode(pCxt, pSelect, createPartitionLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createPartitionLogicNode, &pRoot); + code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); @@ -944,9 +944,16 @@ static int32_t createSetOperatorLogicNode(SLogicPlanContext* pCxt, SSetOperator* } static int32_t getMsgType(ENodeType sqlType) { - return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) - ? TDMT_VND_CREATE_TABLE - : TDMT_VND_SUBMIT; + switch (sqlType) { + case QUERY_NODE_CREATE_TABLE_STMT: + case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + return TDMT_VND_CREATE_TABLE; + case QUERY_NODE_DROP_TABLE_STMT: + return TDMT_VND_DROP_TABLE; + default: + break; + } + return TDMT_VND_SUBMIT; } static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 042b914927..47dcd9ce86 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -206,6 +206,8 @@ static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { if (NULL == pLogicCond) { return TSDB_CODE_OUT_OF_MEMORY; } + pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogicCond->condType = LOGIC_COND_TYPE_AND; int32_t code = nodesListMakeAppend(&pLogicCond->pParameterList, *pSrc); if (TSDB_CODE_SUCCESS == code) { @@ -520,6 +522,7 @@ static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SN default: break; } + planError("cpdPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild))); return TSDB_CODE_PLAN_INTERNAL_ERROR; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0fd1cffc4c..901afce718 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -272,6 +272,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { } // pIndex is definitely not NULL, otherwise it is a bug if (NULL == pIndex) { + planError("doSetSlotId failed, invalid slot name %s", name); pCxt->errCode = TSDB_CODE_PLAN_INTERNAL_ERROR; return DEAL_RES_ERROR; } @@ -435,12 +436,15 @@ static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAdd pNodeAddr->epSet = vg->epSet; } -static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { +static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, + SPhysiNode** pPhyNode) { STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode( pCxt, pScanLogicNode->pMeta->tableInfo.precision, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); if (NULL == pTagScan) { return TSDB_CODE_OUT_OF_MEMORY; } + vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode); } @@ -514,7 +518,7 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SPhysiNode** pPhyNode) { switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: - return createTagScanPhysiNode(pCxt, pScanLogicNode, pPhyNode); + return createTagScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_TABLE: return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_SYSTEM_TABLE: diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 70a969584a..6336377279 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -104,8 +104,9 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; return TSDB_CODE_SUCCESS; } + int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); pVal->node.resType.type = pParam->buffer_type; - pVal->node.resType.bytes = NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes; + pVal->node.resType.bytes = inputSize; switch (pParam->buffer_type) { case TSDB_DATA_TYPE_BOOL: pVal->datum.b = *((bool*)pParam->buffer); @@ -130,7 +131,6 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_NCHAR: pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pVal->datum.p) { return TSDB_CODE_OUT_OF_MEMORY; @@ -138,6 +138,21 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { varDataSetLen(pVal->datum.p, pVal->node.resType.bytes); strncpy(varDataVal(pVal->datum.p), (const char*)pParam->buffer, pVal->node.resType.bytes); break; + case TSDB_DATA_TYPE_NCHAR: { + pVal->node.resType.bytes *= TSDB_NCHAR_SIZE; + pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + if (NULL == pVal->datum.p) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t output = 0; + if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, &output)) { + return errno; + } + varDataSetLen(pVal->datum.p, output); + pVal->node.resType.bytes = output; + break; + } case TSDB_DATA_TYPE_TIMESTAMP: pVal->datum.i = *((int64_t*)pParam->buffer); break; @@ -181,13 +196,20 @@ static EDealRes updatePlanQueryId(SNode* pNode, void* pContext) { int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId) { int32_t size = taosArrayGetSize(pPlan->pPlaceholderValues); - + int32_t code = 0; + if (colIdx < 0) { for (int32_t i = 0; i < size; ++i) { - setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, i), pParams + i); + code = setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, i), pParams + i); + if (code) { + return code; + } } } else { - setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, colIdx), pParams); + code = setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, colIdx), pParams); + if (code) { + return code; + } } if (colIdx < 0 || ((colIdx + 1) == size)) { diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index f0c59b969f..a21b36fef6 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -30,4 +30,9 @@ if(${BUILD_WINGETOPT}) PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" ) target_link_libraries(plannerTest PUBLIC wingetopt) -endif() \ No newline at end of file +endif() + +add_test( + NAME plannerTest + COMMAND plannerTest +) diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp new file mode 100644 index 0000000000..26b1fb8ece --- /dev/null +++ b/source/libs/planner/test/planBasicTest.cpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanBasicTest : public PlannerTestBase {}; + +TEST_F(PlanBasicTest, select) { + useDb("root", "test"); + + // run("select * from t1"); + // run("select 1 from t1"); + // run("select * from st1"); + run("select 1 from st1"); +} + +TEST_F(PlanBasicTest, where) { + useDb("root", "test"); + + run("select * from t1 where c1 > 10"); +} + +TEST_F(PlanBasicTest, join) { + useDb("root", "test"); + + run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + run("select t1.c1, t2.c2 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts"); +} \ No newline at end of file diff --git a/source/libs/planner/test/planDistinctTest.cpp b/source/libs/planner/test/planDistinctTest.cpp new file mode 100644 index 0000000000..58473dcff2 --- /dev/null +++ b/source/libs/planner/test/planDistinctTest.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanDistinctTest : public PlannerTestBase {}; + +TEST_F(PlanDistinctTest, basic) { + useDb("root", "test"); + + // distinct single col + run("select distinct c1 from t1"); + // distinct col list + run("select distinct c1, c2 from t1"); +} + +TEST_F(PlanDistinctTest, expr) { + useDb("root", "test"); + + run("select distinct c1, c2 + 10 from t1"); +} + +TEST_F(PlanDistinctTest, withOrderBy) { + useDb("root", "test"); + + run("select distinct c1 + 10 a from t1 order by a"); +} diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp new file mode 100644 index 0000000000..73fcffd729 --- /dev/null +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanGroupByTest : public PlannerTestBase {}; + +TEST_F(PlanGroupByTest, basic) { + useDb("root", "test"); + + run("select count(*) from t1"); + + run("select c1, max(c3), min(c3), count(*) from t1 group by c1"); + + run("select c1 + c3, c1 + count(*) from t1 where c2 = 'abc' group by c1, c3"); + + run("select c1 + c3, sum(c4 * c5) from t1 where concat(c2, 'wwww') = 'abcwww' group by c1 + c3"); + + run("select sum(ceil(c1)) from t1 group by ceil(c1)"); +} + +TEST_F(PlanGroupByTest, withOrderBy) { + useDb("root", "test"); + + // order by aggfunc + run("select count(*), sum(c1) from t1 order by sum(c1)"); + // order by alias of aggfunc + // run("select count(*), sum(c1) a from t1 order by a"); +} diff --git a/source/dnode/vnode/src/tsdb/tsdbScan.c b/source/libs/planner/test/planIntervalTest.cpp similarity index 50% rename from source/dnode/vnode/src/tsdb/tsdbScan.c rename to source/libs/planner/test/planIntervalTest.cpp index cc6fb473d0..6cc67a3446 100644 --- a/source/dnode/vnode/src/tsdb/tsdbScan.c +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -13,25 +13,29 @@ * along with this program. If not, see . */ -#if 0 -#include "tsdb.h" -#ifndef _TSDB_PLUGINS +#include "planTestUtil.h" +#include "planner.h" -int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } +using namespace std; -STsdbScanHandle* tsdbNewScanHandle() { return NULL; } +class PlanIntervalTest : public PlannerTestBase {}; -void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream) {} +TEST_F(PlanIntervalTest, basic) { + useDb("root", "test"); -int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } + run("select count(*) from t1 interval(10s)"); +} -int tsdbScanSBlockIdx(STsdbScanHandle* pScanHandle) { return 0; } +TEST_F(PlanIntervalTest, pseudoCol) { + useDb("root", "test"); -int tsdbScanSBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; } + run("select _wstartts, _wduration, _wendts, count(*) from t1 interval(10s)"); +} -int tsdbCloseScanFile(STsdbScanHandle* pScanHandle) { return 0; } +TEST_F(PlanIntervalTest, fill) { + useDb("root", "test"); -void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle) {} + run("select count(*) from t1 interval(10s) fill(linear)"); -#endif -#endif \ No newline at end of file + run("select count(*), sum(c1) from t1 interval(10s) fill(value, 10, 20)"); +} diff --git a/source/libs/planner/test/planJoinTest.cpp b/source/libs/planner/test/planJoinTest.cpp new file mode 100644 index 0000000000..4098d383f8 --- /dev/null +++ b/source/libs/planner/test/planJoinTest.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanJoinTest : public PlannerTestBase {}; + +TEST_F(PlanJoinTest, basic) { + useDb("root", "test"); + + run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + + run("select t1.*, t2.* from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + + // run("select t1.c1, t2.c1 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1 and t1.c2 = 'abc' and " + // "t2.c2 = 'qwe'"); +} diff --git a/source/libs/planner/test/planLimitTest.cpp b/source/libs/planner/test/planLimitTest.cpp new file mode 100644 index 0000000000..3b8b3180bd --- /dev/null +++ b/source/libs/planner/test/planLimitTest.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanLimitTest : public PlannerTestBase {}; + +TEST_F(PlanLimitTest, limit) { + useDb("root", "test"); + + run("select * from t1 limit 2"); + + run("select * from t1 limit 5 offset 2"); + + run("select * from t1 limit 2, 5"); +} + +TEST_F(PlanLimitTest, slimit) { + useDb("root", "test"); + + run("select * from t1 partition by c1 slimit 2"); + + run("select * from t1 partition by c1 slimit 5 soffset 2"); + + run("select * from t1 partition by c1 slimit 2, 5"); +} diff --git a/source/libs/planner/test/planOptTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp similarity index 100% rename from source/libs/planner/test/planOptTest.cpp rename to source/libs/planner/test/planOptimizeTest.cpp diff --git a/source/libs/planner/test/planOrderByTest.cpp b/source/libs/planner/test/planOrderByTest.cpp new file mode 100644 index 0000000000..d95d1bdf1d --- /dev/null +++ b/source/libs/planner/test/planOrderByTest.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanOrderByTest : public PlannerTestBase {}; + +TEST_F(PlanOrderByTest, basic) { + useDb("root", "test"); + + // order by key is in the projection list + run("select c1 from t1 order by c1"); + // order by key is not in the projection list + run("select c1 from t1 order by c2"); +} + +TEST_F(PlanOrderByTest, expr) { + useDb("root", "test"); + + run("select * from t1 order by c1 + 10, c2"); +} + +TEST_F(PlanOrderByTest, nullsOrder) { + useDb("root", "test"); + + run("select * from t1 order by c1 desc nulls first"); +} diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp new file mode 100644 index 0000000000..b70cb4d19a --- /dev/null +++ b/source/libs/planner/test/planOtherTest.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanOtherTest : public PlannerTestBase {}; + +TEST_F(PlanOtherTest, createTopic) { + useDb("root", "test"); + + run("create topic tp as SELECT * FROM st1"); +} + +TEST_F(PlanOtherTest, createStream) { + useDb("root", "test"); + + run("create stream if not exists s1 trigger window_close watermark 10s into st1 as select count(*) from t1 " + "interval(10s)"); +} + +TEST_F(PlanOtherTest, createSmaIndex) { + useDb("root", "test"); + + run("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) interval(10s)"); +} + +TEST_F(PlanOtherTest, explain) { + useDb("root", "test"); + + run("explain SELECT * FROM t1"); + + run("explain analyze SELECT * FROM t1"); + + run("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); +} \ No newline at end of file diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp new file mode 100644 index 0000000000..230500e702 --- /dev/null +++ b/source/libs/planner/test/planPartByTest.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanPartitionByTest : public PlannerTestBase {}; + +TEST_F(PlanPartitionByTest, basic) { + useDb("root", "test"); + + run("select * from t1 partition by c1"); +} + +TEST_F(PlanPartitionByTest, withAggFunc) { + useDb("root", "test"); + + run("select count(*) from t1 partition by c1"); +} + +TEST_F(PlanPartitionByTest, withInterval) { + useDb("root", "test"); + + // normal/child table + run("select count(*) from t1 partition by c1 interval(10s)"); + // super table + run("select count(*) from st1 partition by tag1, tag2 interval(10s)"); +} + +TEST_F(PlanPartitionByTest, withGroupBy) { + useDb("root", "test"); + + run("select count(*) from t1 partition by c1 group by c2"); +} diff --git a/source/libs/parser/test/parAlterTest.cpp b/source/libs/planner/test/planSessionTest.cpp similarity index 78% rename from source/libs/parser/test/parAlterTest.cpp rename to source/libs/planner/test/planSessionTest.cpp index 0e6a004138..3ec9c4c387 100644 --- a/source/libs/parser/test/parAlterTest.cpp +++ b/source/libs/planner/test/planSessionTest.cpp @@ -13,14 +13,15 @@ * along with this program. If not, see . */ -#include "parTestUtil.h" +#include "planTestUtil.h" +#include "planner.h" using namespace std; -class ParserAlterTest : public ParserTestBase {}; +class PlanSessionTest : public PlannerTestBase {}; -TEST_F(ParserAlterTest, stmt) { +TEST_F(PlanSessionTest, basic) { useDb("root", "test"); - run("create database db1"); + run("select count(*) from t1 session(ts, 10s)"); } diff --git a/source/dnode/vnode/src/vnd/vnodeInt.c b/source/libs/planner/test/planStateTest.cpp similarity index 64% rename from source/dnode/vnode/src/vnd/vnodeInt.c rename to source/libs/planner/test/planStateTest.cpp index 14dfc75ced..46988d6706 100644 --- a/source/dnode/vnode/src/vnd/vnodeInt.c +++ b/source/libs/planner/test/planStateTest.cpp @@ -13,12 +13,21 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE -#include "vnd.h" -// #include "vnodeInt.h" +#include "planTestUtil.h" +#include "planner.h" -int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; } +using namespace std; -int32_t vnodeCompact(SVnode *pVnode) { return 0; } +class PlanStateTest : public PlannerTestBase {}; -int32_t vnodeSync(SVnode *pVnode) { return 0; } \ No newline at end of file +TEST_F(PlanStateTest, basic) { + useDb("root", "test"); + + run("select count(*) from t1 state_window(c1)"); +} + +TEST_F(PlanStateTest, stateExpr) { + useDb("root", "test"); + + run("select count(*) from t1 state_window(c1 + 10)"); +} diff --git a/source/libs/planner/test/planStmtTest.cpp b/source/libs/planner/test/planStmtTest.cpp index edf1990704..6d56a3419e 100644 --- a/source/libs/planner/test/planStmtTest.cpp +++ b/source/libs/planner/test/planStmtTest.cpp @@ -42,13 +42,13 @@ class PlanStmtTest : public PlannerTestBase { // todo } -private: + private: TAOS_MULTI_BIND* pBindParams_; - int32_t paramNo_; + int32_t paramNo_; }; TEST_F(PlanStmtTest, stmt) { useDb("root", "test"); - run("SELECT * FROM t1 where c1 = ?"); + run("select * from t1 where c1 = ?"); } diff --git a/source/libs/planner/test/planSubqueryTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp new file mode 100644 index 0000000000..185f55db10 --- /dev/null +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanSubqeuryTest : public PlannerTestBase {}; + +TEST_F(PlanSubqeuryTest, basic) { + useDb("root", "test"); + + run("select * from (select * from t1)"); +} + +TEST_F(PlanSubqeuryTest, doubleGroupBy) { + useDb("root", "test"); + + run("select count(*) from (select c1 + c3 a, c1 + count(*) b from t1 where c2 = 'abc' group by c1, c3) where a > 100 " + "group by b"); +} diff --git a/source/libs/tdb/src/inc/tdbTxn.h b/source/libs/planner/test/planSysTbTest.cpp similarity index 65% rename from source/libs/tdb/src/inc/tdbTxn.h rename to source/libs/planner/test/planSysTbTest.cpp index ed9784614c..fff6bfcca4 100644 --- a/source/libs/tdb/src/inc/tdbTxn.h +++ b/source/libs/planner/test/planSysTbTest.cpp @@ -13,19 +13,22 @@ * along with this program. If not, see . */ -#ifndef _TDB_TXN_H_ -#define _TDB_TXN_H_ +#include "planTestUtil.h" +#include "planner.h" -#ifdef __cplusplus -extern "C" { -#endif +using namespace std; -int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, - int flags); -int tdbTxnClose(TXN *pTxn); +class PlanSysTableTest : public PlannerTestBase {}; -#ifdef __cplusplus +TEST_F(PlanSysTableTest, show) { + useDb("root", "test"); + + run("show tables"); + run("show stables"); } -#endif -#endif /*_TDB_TXN_H_*/ \ No newline at end of file +TEST_F(PlanSysTableTest, information) { + useDb("root", "information_schema"); + + run("show tables"); +} diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 2ecb59f0a1..9ddf2a4959 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -106,6 +106,7 @@ class PlannerTestBaseImpl { res_.splitLogicPlan_.clear(); res_.scaledLogicPlan_.clear(); res_.physiPlan_.clear(); + res_.physiSubplans_.clear(); } void dump() { diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp deleted file mode 100644 index 1128076823..0000000000 --- a/source/libs/planner/test/plannerTest.cpp +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include - -#include - -#include "cmdnodes.h" -#include "parser.h" -#include "planInt.h" - -using namespace std; -using namespace testing; - -class PlannerTest : public Test { - protected: - void setDatabase(const string& acctId, const string& db) { - acctId_ = acctId; - db_ = db; - } - - void bind(const char* sql) { - reset(); - cxt_.acctId = atoi(acctId_.c_str()); - cxt_.db = db_.c_str(); - sqlBuf_ = string(sql); - transform(sqlBuf_.begin(), sqlBuf_.end(), sqlBuf_.begin(), ::tolower); - cxt_.sqlLen = strlen(sql); - cxt_.pSql = sqlBuf_.c_str(); - } - - bool run(bool streamQuery = false) { - int32_t code = qParseQuerySql(&cxt_, &query_); - - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] qParseQuerySql code:" << code << ", strerror:" << tstrerror(code) - << ", msg:" << errMagBuf_ << endl; - return false; - } - - const string syntaxTreeStr = toString(query_->pRoot, false); - - SLogicNode* pLogicNode = nullptr; - SPlanContext cxt = {0}; - cxt.queryId = 1; - cxt.acctId = 0; - cxt.streamQuery = streamQuery; - - setPlanContext(query_, &cxt); - code = createLogicPlan(&cxt, &pLogicNode); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] createLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - cout << "====================sql : [" << cxt_.pSql << "]" << endl; - cout << "syntax tree : " << endl; - cout << syntaxTreeStr << endl; - cout << "unformatted logic plan : " << endl; - cout << toString((const SNode*)pLogicNode, false) << endl; - - code = optimizeLogicPlan(&cxt, pLogicNode); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] optimizeLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - SLogicSubplan* pLogicSubplan = nullptr; - code = splitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] splitLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - SQueryLogicPlan* pLogicPlan = NULL; - code = scaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] createPhysiPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - SArray* pExecNodeList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SQueryNodeAddr)); - code = createPhysiPlan(&cxt, pLogicPlan, &plan_, pExecNodeList); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] createPhysiPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - cout << "unformatted physical plan : " << endl; - cout << toString((const SNode*)plan_, false) << endl; - SNode* pNode; - FOREACH(pNode, plan_->pSubplans) { - SNode* pSubplan; - FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { - cout << "unformatted physical subplan : " << endl; - cout << toString(pSubplan, false) << endl; - } - } - - return true; - } - - private: - static const int max_err_len = 1024; - - void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { - if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { - pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; - pCxt->topicQuery = true; - } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { - SMCreateSmaReq req = {0}; - tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); - nodesStringToNode(req.ast, &pCxt->pAstRoot); - pCxt->streamQuery = true; - } else if (QUERY_NODE_CREATE_STREAM_STMT == nodeType(pQuery->pRoot)) { - SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pQuery->pRoot; - pCxt->pAstRoot = pStmt->pQuery; - pCxt->streamQuery = true; - pCxt->triggerType = pStmt->pOptions->triggerType; - pCxt->watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0); - } else { - pCxt->pAstRoot = pQuery->pRoot; - } - } - - void reset() { - memset(&cxt_, 0, sizeof(cxt_)); - memset(errMagBuf_, 0, max_err_len); - cxt_.pMsg = errMagBuf_; - cxt_.msgLen = max_err_len; - } - - string toString(const SNode* pRoot, bool format = true) { - char* pStr = NULL; - int32_t len = 0; - int32_t code = nodesNodeToString(pRoot, format, &pStr, &len); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl; - return string(); - } - string str(pStr); - taosMemoryFreeClear(pStr); - return str; - } - - string acctId_; - string db_; - char errMagBuf_[max_err_len]; - string sqlBuf_; - SParseContext cxt_; - SQuery* query_; - SQueryPlan* plan_; -}; - -TEST_F(PlannerTest, selectBasic) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectConstant) { - setDatabase("root", "test"); - - bind("SELECT 2-1 FROM t1"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectStableBasic) { - setDatabase("root", "test"); - - bind("SELECT * FROM st1"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectJoin) { - setDatabase("root", "test"); - - bind("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); - ASSERT_TRUE(run()); - - bind("SELECT t1.*, t2.* FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); - ASSERT_TRUE(run()); - - bind( - "SELECT t1.c1, t2.c1 FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1 and t1.c2 = 'abc' and " - "t2.c2 = 'qwe'"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectGroupBy) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT c1, max(c3), min(c3), count(*) FROM t1 GROUP BY c1"); - ASSERT_TRUE(run()); - - bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3"); - ASSERT_TRUE(run()); - - bind("SELECT c1 + c3, sum(c4 * c5) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); - ASSERT_TRUE(run()); - - bind("SELECT sum(ceil(c1)) FROM t1 GROUP BY ceil(c1)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectSubquery) { - setDatabase("root", "test"); - - bind( - "SELECT count(*) FROM (SELECT c1 + c3 a, c1 + count(*) b FROM t1 where c2 = 'abc' GROUP BY c1, c3) where a > 100 " - "group by b"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectInterval) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1 interval(10s)"); - ASSERT_TRUE(run()); - - bind("SELECT _wstartts, _wduration, _wendts, count(*) FROM t1 interval(10s)"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM t1 interval(10s) fill(linear)"); - ASSERT_TRUE(run()); - - bind("SELECT count(*), sum(c1) FROM t1 interval(10s) fill(value, 10, 20)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectSessionWindow) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1 session(ts, 10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectStateWindow) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1 state_window(c1)"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM t1 state_window(c1 + 10)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectPartitionBy) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1 partition by c1"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM t1 partition by c1"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM t1 partition by c1 group by c2"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM st1 partition by tag1, tag2 interval(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectOrderBy) { - setDatabase("root", "test"); - - bind("SELECT c1 FROM t1 order by c1"); - ASSERT_TRUE(run()); - - bind("SELECT c1 FROM t1 order by c2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 order by c1 + 10, c2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 order by c1 desc nulls first"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectGroupByOrderBy) { - setDatabase("root", "test"); - - bind("select count(*), sum(c1) from t1 order by sum(c1)"); - ASSERT_TRUE(run()); - - bind("select count(*), sum(c1) a from t1 order by a"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectDistinct) { - setDatabase("root", "test"); - - bind("SELECT distinct c1 FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT distinct c1, c2 + 10 FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT distinct c1 + 10 a FROM t1 order by a"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectLimit) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1 limit 2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 limit 5 offset 2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 limit 2, 5"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectSlimit) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1 partition by c1 slimit 2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 partition by c1 slimit 5 soffset 2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 partition by c1 slimit 2, 5"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, showTables) { - setDatabase("root", "test"); - - bind("show tables"); - ASSERT_TRUE(run()); - - setDatabase("root", "information_schema"); - - bind("show tables"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, showStables) { - setDatabase("root", "test"); - - bind("show stables"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, createTopic) { - setDatabase("root", "test"); - - bind("create topic tp as SELECT * FROM st1"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, createStream) { - setDatabase("root", "test"); - - bind( - "create stream if not exists s1 trigger window_close watermark 10s into st1 as select count(*) from t1 " - "interval(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, createSmaIndex) { - setDatabase("root", "test"); - - bind("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, explain) { - setDatabase("root", "test"); - - bind("explain SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("explain analyze SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); - ASSERT_TRUE(run()); -} diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index de73f1db0b..a2b1353093 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -23,6 +23,7 @@ extern "C" { #include "qworker.h" #include "tlockfree.h" #include "ttimer.h" +#include "tref.h" #define QW_DEFAULT_SCHEDULER_NUMBER 10000 #define QW_DEFAULT_TASK_NUMBER 10000 @@ -85,6 +86,12 @@ typedef struct SQWMsg { SQWConnInfo connInfo; } SQWMsg; +typedef struct SQWHbParam { + bool inUse; + int32_t qwrId; + int64_t refId; +} SQWHbParam; + typedef struct SQWHbInfo { SSchedulerHbRsp rsp; SQWConnInfo connInfo; @@ -137,7 +144,8 @@ typedef struct SQWSchStatus { } SQWSchStatus; // Qnode/Vnode level task management -typedef struct SQWorkerMgmt { +typedef struct SQWorker { + int64_t refId; SQWorkerCfg cfg; int8_t nodeType; int32_t nodeId; @@ -148,9 +156,17 @@ typedef struct SQWorkerMgmt { SHashObj *schHash; // key: schedulerId, value: SQWSchStatus SHashObj *ctxHash; // key: queryId+taskId, value: SQWTaskCtx SMsgCb msgCb; +} SQWorker; + +typedef struct SQWorkerMgmt { + SRWLatch lock; + int32_t qwRef; + int32_t qwNum; + SQWHbParam param[1024]; + int32_t paramIdx; } SQWorkerMgmt; -#define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId +#define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId #define QW_IDS() sId, qId, tId, rId #define QW_FPARAMS() mgmt, QW_IDS() @@ -209,13 +225,13 @@ typedef struct SQWorkerMgmt { } \ } while (0) -#define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__) -#define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__) +#define QW_ELOG(_param, ...) qError("QW:%p " _param, mgmt, __VA_ARGS__) +#define QW_DLOG(_param, ...) qDebug("QW:%p " _param, mgmt, __VA_ARGS__) -#define QW_DUMP(param, ...) \ +#define QW_DUMP(_param, ...) \ do { \ if (gQWDebug.dumpEnable) { \ - qDebug("QW:%p " param, mgmt, __VA_ARGS__); \ + qDebug("QW:%p " _param, mgmt, __VA_ARGS__); \ } \ } while (0) @@ -282,6 +298,14 @@ typedef struct SQWorkerMgmt { } \ } while (0) + +extern SQWorkerMgmt gQwMgmt; + +FORCE_INLINE SQWorker *qwAcquire(int64_t refId) { return (SQWorker *)taosAcquireRef(atomic_load_32(&gQwMgmt.qwRef), refId); } + +FORCE_INLINE int32_t qwRelease(int64_t refId) { return taosReleaseRef(gQwMgmt.qwRef, refId); } + + #ifdef __cplusplus } #endif diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index fb1950d16b..fbd9ce91bc 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -28,7 +28,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg); -int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req); +int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req); int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code); int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code); @@ -41,10 +41,10 @@ int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code); int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t num); void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); -int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp); +int32_t qwGetSchTasksStatus(SQWorker *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp); int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn); -int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo *pConn); +int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SQWConnInfo *pConn); #ifdef __cplusplus } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index b51bc5083e..e84f387dbe 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -10,6 +10,11 @@ #include "tname.h" SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; +SQWorkerMgmt gQwMgmt = { + .lock = 0, + .qwRef = -1, + .qwNum = 0, +}; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { if (!gQWDebug.statusEnable) { @@ -98,7 +103,7 @@ _return: void qwDbgDumpSchInfo(SQWSchStatus *sch, int32_t i) {} -void qwDbgDumpMgmtInfo(SQWorkerMgmt *mgmt) { +void qwDbgDumpMgmtInfo(SQWorker *mgmt) { if (!gQWDebug.dumpEnable) { return; } @@ -186,7 +191,7 @@ int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status) { return TSDB_CODE_SUCCESS; } -int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) { +int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -213,7 +218,7 @@ int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) { return TSDB_CODE_SUCCESS; } -int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { +int32_t qwAcquireSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { while (true) { QW_LOCK(rwType, &mgmt->schLock); *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); @@ -240,15 +245,15 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, return TSDB_CODE_SUCCESS; } -int32_t qwAcquireAddScheduler(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAcquireAddScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(mgmt, sId, rwType, sch, QW_NOT_EXIST_ADD); } -int32_t qwAcquireScheduler(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(mgmt, sId, rwType, sch, QW_NOT_EXIST_RET_ERR); } -void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } +void qwReleaseScheduler(int32_t rwType, SQWorker *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) { char id[sizeof(qId) + sizeof(tId)] = {0}; @@ -384,7 +389,7 @@ int32_t qwAddTaskCtx(QW_FPARAMS_DEF) { QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), fal int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { return qwAddTaskCtxImpl(QW_FPARAMS(), true, ctx); } -void qwReleaseTaskCtx(SQWorkerMgmt *mgmt, void *ctx) { taosHashRelease(mgmt->ctxHash, ctx); } +void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx) { taosHashRelease(mgmt->ctxHash, ctx); } void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { // Note: free/kill may in RC @@ -606,7 +611,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { QW_RET(code); } -int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { +int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; hbInfo->connInfo = sch->hbConnInfo; @@ -1262,7 +1267,7 @@ _return: QW_RET(TSDB_CODE_SUCCESS); } -int32_t qwProcessHbLinkBroken(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { +int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; SQWSchStatus * sch = NULL; @@ -1288,7 +1293,7 @@ int32_t qwProcessHbLinkBroken(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq QW_RET(TSDB_CODE_SUCCESS); } -int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { +int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; SQWSchStatus * sch = NULL; @@ -1333,7 +1338,19 @@ _return: } void qwProcessHbTimerEvent(void *param, void *tmrId) { - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)param; + SQWHbParam* hbParam = (SQWHbParam*)param; + if (hbParam->qwrId != atomic_load_32(&gQwMgmt.qwRef)) { + return; + } + + int64_t refId = hbParam->refId; + SQWorker *mgmt = qwAcquire(refId); + if (NULL == mgmt) { + QW_DLOG("qwAcquire %" PRIx64 "failed", refId); + taosMemoryFree(param); + return; + } + SQWSchStatus *sch = NULL; int32_t taskNum = 0; SQWHbInfo * rspList = NULL; @@ -1347,6 +1364,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { if (schNum <= 0) { QW_UNLOCK(QW_READ, &mgmt->schLock); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); + qwRelease(refId); return; } @@ -1355,6 +1373,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { QW_UNLOCK(QW_READ, &mgmt->schLock); QW_ELOG("calloc %d SQWHbInfo failed", schNum); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); + qwRelease(refId); return; } @@ -1396,6 +1415,74 @@ _return: taosMemoryFreeClear(rspList); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); + qwRelease(refId); +} + +void qwCloseRef(void) { + taosWLockLatch(&gQwMgmt.lock); + if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) { + taosCloseRef(gQwMgmt.qwRef); + gQwMgmt.qwRef= -1; + } + taosWUnLockLatch(&gQwMgmt.lock); +} + +void qwDestroyImpl(void *pMgmt) { + SQWorker *mgmt = (SQWorker *)pMgmt; + + taosTmrStopA(&mgmt->hbTimer); + taosTmrCleanUp(mgmt->timer); + + // TODO STOP ALL QUERY + + // TODO FREE ALL + + taosHashCleanup(mgmt->ctxHash); + taosHashCleanup(mgmt->schHash); + + taosMemoryFree(mgmt); + + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + + qwCloseRef(); +} + +int32_t qwOpenRef(void) { + taosWLockLatch(&gQwMgmt.lock); + if (gQwMgmt.qwRef < 0) { + gQwMgmt.qwRef= taosOpenRef(100, qwDestroyImpl); + if (gQwMgmt.qwRef < 0) { + taosWUnLockLatch(&gQwMgmt.lock); + qError("init qworker ref failed"); + QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + taosWUnLockLatch(&gQwMgmt.lock); + + return TSDB_CODE_SUCCESS; +} + +void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { + int32_t paramIdx = 0; + int32_t newParamIdx = 0; + + while (true) { + paramIdx = atomic_load_32(&gQwMgmt.paramIdx); + if (paramIdx == tListLen(gQwMgmt.param)) { + newParamIdx = 0; + } else { + newParamIdx = paramIdx + 1; + } + + if (paramIdx == atomic_val_compare_exchange_32(&gQwMgmt.paramIdx, paramIdx, newParamIdx)) { + break; + } + } + + gQwMgmt.param[paramIdx].qwrId = gQwMgmt.qwRef; + gQwMgmt.param[paramIdx].refId = refId; + + *pParam = &gQwMgmt.param[paramIdx]; } int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb) { @@ -1404,10 +1491,21 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW QW_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t code = 0; - SQWorkerMgmt *mgmt = taosMemoryCalloc(1, sizeof(SQWorkerMgmt)); + int32_t qwNum = atomic_add_fetch_32(&gQwMgmt.qwNum, 1); + if (1 == qwNum) { + memset(gQwMgmt.param, 0, sizeof(gQwMgmt.param)); + } + + int32_t code = qwOpenRef(); + if (code) { + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + QW_RET(code); + } + + SQWorker *mgmt = taosMemoryCalloc(1, sizeof(SQWorker)); if (NULL == mgmt) { - qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); + qError("calloc %d failed", (int32_t)sizeof(SQWorker)); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1449,16 +1547,25 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, mgmt, mgmt->timer); - if (NULL == mgmt->hbTimer) { - qError("start hb timer failed"); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - mgmt->nodeType = nodeType; mgmt->nodeId = nodeId; mgmt->msgCb = *pMsgCb; + mgmt->refId = taosAddRef(gQwMgmt.qwRef, mgmt); + if (mgmt->refId < 0) { + qError("taosAddRef qw failed, error:%s", tstrerror(terrno)); + QW_ERR_JRET(terrno); + } + + SQWHbParam *param = NULL; + qwSetHbParam(mgmt->refId, ¶m); + + mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void*)param, mgmt->timer); + if (NULL == mgmt->hbTimer) { + qError("start hb timer failed"); + QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + *qWorkerMgmt = mgmt; qDebug("qworker initialized for node, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); @@ -1467,13 +1574,17 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW _return: - taosHashCleanup(mgmt->schHash); - taosHashCleanup(mgmt->ctxHash); - - taosTmrCleanUp(mgmt->timer); - - taosMemoryFreeClear(mgmt); + if (mgmt->refId >= 0) { + qwRelease(mgmt->refId); + } else { + taosHashCleanup(mgmt->schHash); + taosHashCleanup(mgmt->ctxHash); + taosTmrCleanUp(mgmt->timer); + taosMemoryFreeClear(mgmt); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + } + QW_RET(code); } @@ -1482,22 +1593,14 @@ void qWorkerDestroy(void **qWorkerMgmt) { return; } - SQWorkerMgmt *mgmt = *qWorkerMgmt; + SQWorker *mgmt = *qWorkerMgmt; - taosTmrStopA(&mgmt->hbTimer); - taosTmrCleanUp(mgmt->timer); - - // TODO STOP ALL QUERY - - // TODO FREE ALL - - taosHashCleanup(mgmt->ctxHash); - taosHashCleanup(mgmt->schHash); - - taosMemoryFreeClear(*qWorkerMgmt); + if (taosRemoveRef(gQwMgmt.qwRef, mgmt->refId)) { + qError("remove qw from ref list failed, refId:%" PRIx64, mgmt->refId); + } } -int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp) { +int32_t qwGetSchTasksStatus(SQWorker *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp) { /* SQWSchStatus *sch = NULL; int32_t taskNum = 0; @@ -1544,7 +1647,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs return TSDB_CODE_SUCCESS; } -int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { +int32_t qwUpdateSchLastAccess(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; /* @@ -1557,7 +1660,7 @@ int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, ui return TSDB_CODE_SUCCESS; } -int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) { +int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) { SQWSchStatus * sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -1584,7 +1687,7 @@ int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_RET(code); } -int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { +int32_t qwCancelTask(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus * sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 411daf364a..d4f6c2fd00 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -320,7 +320,7 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { return TSDB_CODE_SUCCESS; } -int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo *pConn) { +int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SQWConnInfo *pConn) { SSchedulerHbReq req = {0}; req.header.vgId = mgmt->nodeId; req.sId = sId; @@ -363,7 +363,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SSubQueryMsg *msg = pMsg->pCont; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -405,7 +405,7 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQueryContinueReq *msg = (SQueryContinueReq *)pMsg->pCont; bool needStop = false; SQWTaskCtx *handles = NULL; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -436,7 +436,7 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SResReadyReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -478,7 +478,7 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; msg->sId = htobe64(msg->sId); uint64_t sId = msg->sId; @@ -499,7 +499,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } SResFetchReq *msg = pMsg->pCont; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -539,7 +539,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; int32_t code = 0; STaskCancelReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { @@ -579,7 +579,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; STaskDropReq *msg = pMsg->pCont; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task drop msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -621,7 +621,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SSchedulerHbReq req = {0}; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == pMsg->pCont) { QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 864894fa42..80e5669cc2 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3765,6 +3765,7 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); info->scalarMode = stat.scalarMode; + fltDebug("scalar mode: %d", info->scalarMode); if (!info->scalarMode) { FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index eb6be4ca3c..f5fab814ff 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -75,7 +75,15 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { if (valueNode->node.resType.type != type) { out.columnData->info.type = type; - out.columnData->info.bytes = tDataTypes[type].bytes; + if (IS_VAR_DATA_TYPE(type)) { + if (IS_VAR_DATA_TYPE(valueNode->node.resType.type)) { + out.columnData->info.bytes = valueNode->node.resType.bytes * TSDB_NCHAR_SIZE; + } else { + out.columnData->info.bytes = 64 * TSDB_NCHAR_SIZE; + } + } else { + out.columnData->info.bytes = tDataTypes[type].bytes; + } code = doConvertDataType(valueNode, &out); if (code != TSDB_CODE_SUCCESS) { @@ -598,7 +606,7 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1); memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); } else { - memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + nodesSetValueNodeValue(res, output.columnData->pData); } } @@ -638,7 +646,7 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { res->datum.p = output.columnData->pData; output.columnData->pData = NULL; } else { - memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + nodesSetValueNodeValue(res, output.columnData->pData); } nodesDestroyNode(*pNode); @@ -680,7 +688,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { res->datum.p = output.columnData->pData; output.columnData->pData = NULL; } else { - memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + nodesSetValueNodeValue(res, output.columnData->pData); } } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 0f6961018c..5906ee8970 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -85,7 +85,10 @@ typedef struct SSchedulerMgmt { uint64_t taskId; // sequential taksId uint64_t sId; // schedulerId SSchedulerCfg cfg; + SRWLatch lock; + bool exit; int32_t jobRef; + int32_t jobNum; SSchedulerStat stat; SHashObj *hbConnections; } SSchedulerMgmt; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 31d2da9380..a70d366e9b 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -21,7 +21,9 @@ #include "tref.h" #include "trpc.h" -SSchedulerMgmt schMgmt = {0}; +SSchedulerMgmt schMgmt = { + .jobRef = -1, +}; FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } @@ -70,6 +72,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql, int64_t startTs, bool syncSchedule) { int32_t code = 0; + int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); @@ -114,15 +117,17 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray tsem_init(&pJob->rspSem, 0, 0); - int64_t refId = taosAddRef(schMgmt.jobRef, pJob); + refId = taosAddRef(schMgmt.jobRef, pJob); if (refId < 0) { SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); SCH_ERR_JRET(terrno); } + atomic_add_fetch_32(&schMgmt.jobNum, 1); + if (NULL == schAcquireJob(refId)) { SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); - SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } pJob->refId = refId; @@ -137,7 +142,11 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray _return: - schFreeJobImpl(pJob); + if (refId < 0) { + schFreeJobImpl(pJob); + } else { + taosRemoveRef(schMgmt.jobRef, refId); + } SCH_RET(code); } @@ -245,6 +254,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_VND_CREATE_TABLE_RSP: + case TDMT_VND_DROP_TABLE_RSP: case TDMT_VND_SUBMIT_RSP: break; default: @@ -369,7 +379,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < childNum; ++n) { - SSubplan * child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); + SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); if (NULL == childTask || NULL == *childTask) { SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -401,7 +411,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < parentNum; ++n) { - SSubplan * parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); + SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); if (NULL == parentTask || NULL == *parentTask) { SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -491,7 +501,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SSchLevel level = {0}; SNodeListNode *plans = NULL; int32_t taskNum = 0; - SSchLevel * pLevel = NULL; + SSchLevel *pLevel = NULL; level.status = JOB_TASK_STATUS_NOT_START; @@ -1094,6 +1104,30 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); break; } + case TDMT_VND_DROP_TABLE_RSP: { + SVDropTbBatchRsp batchRsp = {0}; + if (msg) { + SCoder coder = {0}; + tCoderInit(&coder, TD_LITTLE_ENDIAN, msg, msgSize, TD_DECODER); + code = tDecodeSVDropTbBatchRsp(&coder, &batchRsp); + if (TSDB_CODE_SUCCESS == code && batchRsp.pArray) { + int32_t num = taosArrayGetSize(batchRsp.pArray); + for (int32_t i = 0; i < num; ++i) { + SVDropTbRsp *rsp = taosArrayGet(batchRsp.pArray, i); + if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { + tCoderClear(&coder); + SCH_ERR_JRET(rsp->code); + } + } + } + tCoderClear(&coder); + SCH_ERR_JRET(code); + } + + SCH_ERR_JRET(rspCode); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + break; + } case TDMT_VND_SUBMIT_RSP: { if (msg) { SSubmitRsp *rsp = (SSubmitRsp *)msg; @@ -1267,7 +1301,7 @@ int32_t schUpdateTaskExecNodeHandle(SSchTask *pTask, void *handle, int32_t rspCo int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { int32_t code = 0; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - SSchTask * pTask = NULL; + SSchTask *pTask = NULL; SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { @@ -1316,6 +1350,10 @@ int32_t schHandleCreateTableCallback(void *param, const SDataBuf *pMsg, int32_t return schHandleCallback(param, pMsg, TDMT_VND_CREATE_TABLE_RSP, code); } +int32_t schHandleDropTableCallback(void *param, const SDataBuf *pMsg, int32_t code) { + return schHandleCallback(param, pMsg, TDMT_VND_DROP_TABLE_RSP, code); +} + int32_t schHandleQueryCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code); } @@ -1412,6 +1450,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { case TDMT_VND_CREATE_TABLE: *fp = schHandleCreateTableCallback; break; + case TDMT_VND_DROP_TABLE: + *fp = schHandleDropTableCallback; + break; case TDMT_VND_SUBMIT: *fp = schHandleSubmitCallback; break; @@ -1617,8 +1658,8 @@ _return: int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { int32_t code = 0; SSchHbCallbackParam *param = NULL; - SMsgSendInfo * pMsgSendInfo = NULL; - SQueryNodeAddr * addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SMsgSendInfo *pMsgSendInfo = NULL; + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; @@ -1759,10 +1800,10 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { } SRpcCtxVal dst = {0}; - void * pIter = taosHashIterate(pSrc->args, NULL); + void *pIter = taosHashIterate(pSrc->args, NULL); while (pIter) { SRpcCtxVal *pVal = (SRpcCtxVal *)pIter; - int32_t * msgType = taosHashGetKey(pIter, NULL); + int32_t *msgType = taosHashGetKey(pIter, NULL); dst = *pVal; dst.val = NULL; @@ -1916,7 +1957,7 @@ _return: int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void * msg = NULL; + void *msg = NULL; int32_t code = 0; bool isCandidateAddr = false; bool persistHandle = false; @@ -1931,6 +1972,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, switch (msgType) { case TDMT_VND_CREATE_TABLE: + case TDMT_VND_DROP_TABLE: case TDMT_VND_SUBMIT: { msgSize = pTask->msgLen; msg = taosMemoryCalloc(1, msgSize); @@ -2239,6 +2281,19 @@ int32_t schCancelJob(SSchJob *pJob) { // TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST } +void schCloseJobRef(void) { + if (!atomic_load_8((int8_t*)&schMgmt.exit)) { + return; + } + + SCH_LOCK(SCH_WRITE, &schMgmt.lock); + if (atomic_load_32(&schMgmt.jobNum) <= 0 && schMgmt.jobRef >= 0) { + taosCloseRef(schMgmt.jobRef); + schMgmt.jobRef = -1; + } + SCH_UNLOCK(SCH_WRITE, &schMgmt.lock); +} + void schFreeJobImpl(void *job) { if (NULL == job) { return; @@ -2284,6 +2339,10 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob); qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); + + atomic_sub_fetch_32(&schMgmt.jobNum, 1); + + schCloseJobRef(); } static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, @@ -2368,7 +2427,7 @@ _return: } int32_t schedulerInit(SSchedulerCfg *cfg) { - if (schMgmt.jobRef) { + if (schMgmt.jobRef >= 0) { qError("scheduler already initialized"); return TSDB_CODE_QRY_INVALID_INPUT; } @@ -2673,7 +2732,7 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); for (int32_t m = 0; m < pLevel->taskNum; ++m) { - SSchTask * pTask = taosArrayGet(pLevel->subTasks, m); + SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); SQuerySubDesc subDesc = {.tid = pTask->taskId, .status = pTask->status}; taosArrayPush(pSub, &subDesc); @@ -2732,7 +2791,9 @@ void schedulerFreeTaskList(SArray *taskList) { } void schedulerDestroy(void) { - if (schMgmt.jobRef) { + atomic_store_8((int8_t*)&schMgmt.exit, 1); + + if (schMgmt.jobRef >= 0) { SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0); int64_t refId = 0; @@ -2745,9 +2806,6 @@ void schedulerDestroy(void) { pJob = taosIterateRef(schMgmt.jobRef, refId); } - - taosCloseRef(schMgmt.jobRef); - schMgmt.jobRef = 0; } if (schMgmt.hbConnections) { diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 71ac3d97ed..49da6fd8ed 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,6 +22,65 @@ extern "C" { #endif +typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); + +// exposed types +typedef struct STEnv TENV; +typedef struct STDB TDB; +typedef struct STDBC TDBC; +typedef struct STxn TXN; + +// TENV +int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv); +int tdbEnvClose(TENV *pEnv); +int tdbBegin(TENV *pEnv, TXN *pTxn); +int tdbCommit(TENV *pEnv, TXN *pTxn); + +// TDB +int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); +int tdbDbClose(TDB *pDb); +int tdbDbDrop(TDB *pDb); +int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); + +// TDBC +int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); +int tdbDbcClose(TDBC *pDbc); +int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); +int tdbDbcMoveToFirst(TDBC *pDbc); +int tdbDbcMoveToLast(TDBC *pDbc); +int tdbDbcMoveToNext(TDBC *pDbc); +int tdbDbcMoveToPrev(TDBC *pDbc); +int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); + +int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); +int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbDbcDrop(TDBC *pDbc); +int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); + +// TXN +#define TDB_TXN_WRITE 0x1 +#define TDB_TXN_READ_UNCOMMITTED 0x2 + +int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, + int flags); +int tdbTxnClose(TXN *pTxn); + +// other +void tdbFree(void *); + +struct STxn { + int flags; + int64_t txnId; + void *(*xMalloc)(void *, size_t); + void (*xFree)(void *, void *); + void *xArg; +}; + +// error code +enum { TDB_CODE_SUCCESS = 0, TDB_CODE_MAX }; + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0cee494ae8..2665dd1cf6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -19,17 +19,17 @@ #define TDB_BTREE_LEAF 0x2 struct SBTree { - SPgno root; - int keyLen; - int valLen; - SPager *pPager; - FKeyComparator kcmpr; - int pageSize; - int maxLocal; - int minLocal; - int maxLeaf; - int minLeaf; - void *pBuf; + SPgno root; + int keyLen; + int valLen; + SPager *pPager; + tdb_cmpr_fn_t kcmpr; + int pageSize; + int maxLocal; + int minLocal; + int maxLeaf; + int minLeaf; + void *pBuf; }; #define TDB_BTREE_PAGE_COMMON_HDR u8 flags; @@ -58,16 +58,6 @@ typedef struct { SBTree *pBt; } SBtreeInitPageArg; -typedef struct { - int kLen; - const u8 *pKey; - int vLen; - const u8 *pVal; - SPgno pgno; - u8 *pBuf; -} SCellDecoder; - -static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbBtreeOpenImpl(SBTree *pBt); static int tdbBtreeInitPage(SPage *pPage, void *arg, int init); @@ -76,11 +66,10 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); static int tdbBtreeBalance(SBTC *pBtc); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); -static int tdbBtcMoveToNext(SBTC *pBtc); static int tdbBtcMoveDownward(SBTC *pBtc); static int tdbBtcMoveUpward(SBTC *pBtc); -int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { +int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, tdb_cmpr_fn_t kcmpr, SBTree **ppBt) { SBTree *pBt; int ret; @@ -165,7 +154,7 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in // make sure enough space to hold the cell szBuf = kLen + vLen + 14; - pBuf = TDB_REALLOC(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize); + pBuf = tdbRealloc(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize); if (pBuf == NULL) { tdbBtcClose(&btc); ASSERT(0); @@ -243,7 +232,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL tdbBtreeDecodeCell(btc.pPage, pCell, &cd); if (ppKey) { - pTKey = TDB_REALLOC(*ppKey, cd.kLen); + pTKey = tdbRealloc(*ppKey, cd.kLen); if (pTKey == NULL) { tdbBtcClose(&btc); ASSERT(0); @@ -255,7 +244,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } if (ppVal) { - pTVal = TDB_REALLOC(*ppVal, cd.vLen); + pTVal = tdbRealloc(*ppVal, cd.vLen); if (pTVal == NULL) { tdbBtcClose(&btc); ASSERT(0); @@ -1018,6 +1007,7 @@ int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) { pBtc->iPage = -1; pBtc->pPage = NULL; pBtc->idx = -1; + memset(&pBtc->coder, 0, sizeof(SCellDecoder)); if (pTxn == NULL) { pBtc->pTxn = &pBtc->txn; @@ -1060,6 +1050,8 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { return 0; } } else { + ASSERT(0); +#if 0 // move from a position int iPage = 0; @@ -1077,6 +1069,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { tdbBtcMoveUpward(pBtc); } +#endif } // move downward @@ -1125,6 +1118,8 @@ int tdbBtcMoveToLast(SBTC *pBtc) { return 0; } } else { + ASSERT(0); +#if 0 int iPage = 0; // downward search @@ -1147,6 +1142,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { tdbBtcMoveUpward(pBtc); } +#endif } // move downward @@ -1185,7 +1181,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd); - pKey = TDB_REALLOC(*ppKey, cd.kLen); + pKey = tdbRealloc(*ppKey, cd.kLen); if (pKey == NULL) { return -1; } @@ -1196,9 +1192,9 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { if (ppVal) { // TODO: vLen may be zero - pVal = TDB_REALLOC(*ppVal, cd.vLen); + pVal = tdbRealloc(*ppVal, cd.vLen); if (pVal == NULL) { - TDB_FREE(pKey); + tdbFree(pKey); return -1; } @@ -1216,7 +1212,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return 0; } -static int tdbBtcMoveToNext(SBTC *pBtc) { +int tdbBtcMoveToNext(SBTC *pBtc) { int nCells; int ret; SCell *pCell; @@ -1262,6 +1258,43 @@ static int tdbBtcMoveToNext(SBTC *pBtc) { return 0; } +int tdbBtcMoveToPrev(SBTC *pBtc) { + if (pBtc->idx < 0) return -1; + + pBtc->idx--; + if (pBtc->idx >= 0) { + return 0; + } + + // move upward + for (;;) { + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } + + tdbBtcMoveUpward(pBtc); + pBtc->idx--; + if (pBtc->idx >= 0) { + break; + } + } + + // move downward + for (;;) { + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) break; + + tdbBtcMoveDownward(pBtc); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1; + } else { + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + } + } + + return 0; +} + static int tdbBtcMoveDownward(SBTC *pBtc) { int ret; SPgno pgno; @@ -1307,17 +1340,38 @@ static int tdbBtcMoveUpward(SBTC *pBtc) { return 0; } -static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { - int ret; - int nCells; - int c; - SBTree *pBt; - SCell *pCell; - SPager *pPager; - SCellDecoder cd = {0}; +int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen) { + SCell *pCell; - pBt = pBtc->pBt; - pPager = pBt->pPager; + if (pBtc->idx < 0 || pBtc->idx >= TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { + return -1; + } + + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + tdbBtreeDecodeCell(pBtc->pPage, pCell, &pBtc->coder); + + if (ppKey) { + *ppKey = (void *)pBtc->coder.pKey; + *kLen = pBtc->coder.kLen; + } + + if (ppVal) { + *ppVal = (void *)pBtc->coder.pVal; + *kLen = pBtc->coder.vLen; + } + + return 0; +} + +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { + int ret; + int nCells; + int c; + SCell *pCell; + SBTree *pBt = pBtc->pBt; + SPager *pPager = pBt->pPager; + const void *pTKey; + int tkLen; if (pBtc->iPage < 0) { // move from a clear cursor @@ -1334,6 +1388,8 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // for empty tree, just return with an invalid position if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) return 0; } else { + ASSERT(0); +#if 0 SPage *pPage; int idx; int iPage = 0; @@ -1368,11 +1424,12 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { if (pBtc->iPage == iPage) break; tdbBtcMoveUpward(pBtc); } +#endif } // search downward to the leaf for (;;) { - int lidx, ridx, midx; + int lidx, ridx; SPage *pPage; pPage = pBtc->pPage; @@ -1381,13 +1438,11 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { ridx = nCells - 1; ASSERT(nCells > 0); - ASSERT(pBtc->idx == -1); // compare first cell - midx = lidx; - pCell = tdbPageGetCell(pPage, midx); - tdbBtreeDecodeCell(pPage, pCell, &cd); - c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); + pBtc->idx = lidx; + tdbBtcGet(pBtc, &pTKey, &tkLen, NULL, NULL); + c = pBt->kcmpr(pKey, kLen, pTKey, tkLen); if (c <= 0) { ridx = lidx - 1; } else { @@ -1396,10 +1451,9 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // compare last cell if (lidx <= ridx) { - midx = ridx; - pCell = tdbPageGetCell(pPage, midx); - tdbBtreeDecodeCell(pPage, pCell, &cd); - c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); + pBtc->idx = ridx; + tdbBtcGet(pBtc, &pTKey, &tkLen, NULL, NULL); + c = pBt->kcmpr(pKey, kLen, pTKey, tkLen); if (c >= 0) { lidx = ridx + 1; } else { @@ -1411,24 +1465,15 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { for (;;) { if (lidx > ridx) break; - midx = (lidx + ridx) >> 1; - - pCell = tdbPageGetCell(pPage, midx); - ret = tdbBtreeDecodeCell(pPage, pCell, &cd); - if (ret < 0) { - // TODO: handle error - ASSERT(0); - return -1; - } - - // Compare the key values - c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); + pBtc->idx = (lidx + ridx) >> 1; + tdbBtcGet(pBtc, &pTKey, &tkLen, NULL, NULL); + c = pBt->kcmpr(pKey, kLen, pTKey, tkLen); if (c < 0) { // pKey < cd.pKey - ridx = midx - 1; + ridx = pBtc->idx - 1; } else if (c > 0) { // pKey > cd.pKey - lidx = midx + 1; + lidx = pBtc->idx + 1; } else { // pKey == cd.pKey break; @@ -1437,14 +1482,11 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // keep search downward or break if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { - pBtc->idx = midx; *pCRst = c; break; } else { - if (c <= 0) { - pBtc->idx = midx; - } else { - pBtc->idx = midx + 1; + if (c > 0) { + pBtc->idx += 1; } tdbBtcMoveDownward(pBtc); } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index f4468e7e82..aadb9238b0 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -24,7 +24,7 @@ struct STDBC { SBTC btc; }; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, TENV *pEnv, TDB **ppDb) { +int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb) { TDB *pDb; SPager *pPager; int ret; @@ -75,7 +75,7 @@ int tdbDbDrop(TDB *pDb) { return 0; } -int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { +int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn); } @@ -87,7 +87,7 @@ int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, vo return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); } -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc) { +int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn) { int ret; TDBC *pDbc = NULL; @@ -97,34 +97,53 @@ int tdbDbcOpen(TDB *pDb, TDBC **ppDbc) { return -1; } - tdbBtcOpen(&pDbc->btc, pDb->pBt, NULL); - - // TODO: move to first now, we can move to any key-value - // and in any direction, design new APIs. - ret = tdbBtcMoveToFirst(&pDbc->btc); - if (ret < 0) { - ASSERT(0); - return -1; - } + tdbBtcOpen(&pDbc->btc, pDb->pBt, pTxn); *ppDbc = pDbc; return 0; } -int tdbDbNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { +int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pDbc->btc, pKey, kLen, c); } + +int tdbDbcMoveToFirst(TDBC *pDbc) { return tdbBtcMoveToFirst(&pDbc->btc); } + +int tdbDbcMoveToLast(TDBC *pDbc) { return tdbBtcMoveToLast(&pDbc->btc); } + +int tdbDbcMoveToNext(TDBC *pDbc) { return tdbBtcMoveToNext(&pDbc->btc); } + +int tdbDbcMoveToPrev(TDBC *pDbc) { return tdbBtcMoveToPrev(&pDbc->btc); } + +int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) { + return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen); +} + +int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) { + // TODO + ASSERT(0); + return 0; +} + +int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen) { + // TODO + ASSERT(0); + return 0; +} + +int tdbDbcDrop(TDBC *pDbc) { + // TODO + ASSERT(0); + return 0; +} + +int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); } int tdbDbcClose(TDBC *pDbc) { if (pDbc) { + tdbBtcClose(&pDbc->btc); tdbOsFree(pDbc); } return 0; } - -int tdbDbcInsert(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 779435861e..d64b31234e 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -15,7 +15,7 @@ #include "tdbInt.h" -int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) { +int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv) { TENV *pEnv; int dsize; int zsize; @@ -49,14 +49,14 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) { pEnv->jfd = -1; - ret = tdbPCacheOpen(pageSize, cacheSize, &(pEnv->pCache)); + ret = tdbPCacheOpen(szPage, pages, &(pEnv->pCache)); if (ret < 0) { return -1; } pEnv->nPgrHash = 8; tsize = sizeof(SPager *) * pEnv->nPgrHash; - pEnv->pgrHash = TDB_REALLOC(pEnv->pgrHash, tsize); + pEnv->pgrHash = tdbRealloc(pEnv->pgrHash, tsize); if (pEnv->pgrHash == NULL) { return -1; } @@ -103,11 +103,6 @@ int tdbCommit(TENV *pEnv, TXN *pTxn) { return 0; } -int tdbRollback(TENV *pEnv, TXN *pTxn) { - ASSERT(0); - return 0; -} - SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) { u32 hash; SPager **ppPager; diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c index 5ca3578985..a2fce8925b 100644 --- a/source/libs/tdb/src/db/tdbOs.c +++ b/source/libs/tdb/src/db/tdbOs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tdbInt.h" +#include "tdbOs.h" #ifndef TDB_FOR_TDENGINE diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 0ccf7f5f92..646b83298e 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -135,7 +135,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) // 1. Search the hash table pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; while (pPage) { - if (TDB_IS_SAME_PAGE(&(pPage->pgid), pPgid)) break; + if (memcmp(pPage->pgid.fileid, pPgid->fileid, TDB_FILE_ID_LEN) == 0 && pPage->pgid.pgno == pPgid->pgno) break; pPage = pPage->pHashNext; } diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 09e126772b..3746d9358f 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -15,6 +15,26 @@ #include "tdbInt.h" +void *tdbRealloc(void *ptr, size_t size) { + void *nPtr; + if ((ptr) == NULL || ((int *)(ptr))[-1] < (size)) { + nPtr = tdbOsRealloc((ptr) ? (char *)(ptr) - sizeof(int) : NULL, (size) + sizeof(int)); + if (nPtr) { + ((int *)nPtr)[0] = (size); + nPtr = (char *)nPtr + sizeof(int); + } + } else { + nPtr = (ptr); + } + return nPtr; +} + +void tdbFree(void *p) { + if (p) { + tdbOsFree((char *)(p) - sizeof(int)); + } +} + int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { int64_t stDev = 0, stIno = 0; diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h deleted file mode 100644 index 2a9e8533d2..0000000000 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_BTREE_H_ -#define _TD_BTREE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SBTree SBTree; -typedef struct SBTC SBTC; -typedef struct SBtInfo { - SPgno root; - int nLevel; - int nData; -} SBtInfo; - -struct SBTC { - SBTree *pBt; - i8 iPage; - SPage *pPage; - int idx; - int idxStack[BTREE_MAX_DEPTH + 1]; - SPage *pgStack[BTREE_MAX_DEPTH + 1]; - TXN *pTxn; - TXN txn; -}; - -// SBTree -int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); -int tdbBtreeClose(SBTree *pBt); -int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); -int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); - -// SBTC -int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn); -int tdbBtcMoveToFirst(SBTC *pBtc); -int tdbBtcMoveToLast(SBTC *pBtc); -int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbBtcClose(SBTC *pBtc); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h deleted file mode 100644 index f9ff378633..0000000000 --- a/source/libs/tdb/src/inc/tdbDb.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_DB_H_ -#define _TD_TDB_DB_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct STDB TDB; -typedef struct STDBC TDBC; - -// TDB -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, TENV *pEnv, TDB **ppDb); -int tdbDbClose(TDB *pDb); -int tdbDbDrop(TDB *pDb); -int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); -int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); - -// TDBC -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc); -int tdbDbNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbDbcClose(TDBC *pDbc); -int tdbDbcInsert(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbEnv.h b/source/libs/tdb/src/inc/tdbEnv.h deleted file mode 100644 index 54f5818be1..0000000000 --- a/source/libs/tdb/src/inc/tdbEnv.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TDB_ENV_H_ -#define _TDB_ENV_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct STEnv { - char *rootDir; - char *jfname; - int jfd; - SPCache *pCache; - SPager *pgrList; - int nPager; - int nPgrHash; - SPager **pgrHash; -} TENV; - -int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv); -int tdbEnvClose(TENV *pEnv); -int tdbBegin(TENV *pEnv, TXN *pTxn); -int tdbCommit(TENV *pEnv, TXN *pTxn); -int tdbRollback(TENV *pEnv, TXN *pTxn); - -void tdbEnvAddPager(TENV *pEnv, SPager *pPager); -void tdbEnvRemovePager(TENV *pEnv, SPager *pPager); -SPager *tdbEnvGetPager(TENV *pEnv, const char *fname); - -#ifdef __cplusplus -} -#endif - -#endif /*_TDB_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index b85b7174bc..4c19dee03e 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -31,6 +31,12 @@ typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; +// SPgno +typedef u32 SPgno; + +#include "tdbOs.h" +#include "tdbUtil.h" + // p must be u8 * #define TDB_GET_U24(p) ((p)[0] * 65536 + *(u16 *)((p) + 1)) #define TDB_PUT_U24(p, v) \ @@ -41,10 +47,6 @@ typedef uint64_t u64; (p)[0] = (tv >> 16) & 0xff; \ } while (0) -// SPgno -typedef u32 SPgno; -#define TDB_IVLD_PGNO ((pgno_t)0) - // fileid #define TDB_FILE_ID_LEN 24 @@ -54,51 +56,21 @@ typedef struct { SPgno pgno; } SPgid; -#define TDB_IVLD_PGID (SPgid){0, TDB_IVLD_PGNO}; - -static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { - SPgid *pgid1 = (SPgid *)p1; - SPgid *pgid2 = (SPgid *)p2; - int rcode; - - rcode = memcmp(pgid1->fileid, pgid2->fileid, TDB_FILE_ID_LEN); - if (rcode) { - return rcode; - } else { - if (pgid1->pgno > pgid2->pgno) { - return 1; - } else if (pgid1->pgno < pgid2->pgno) { - return -1; - } else { - return 0; - } - } -} - -#define TDB_IS_SAME_PAGE(pPgid1, pPgid2) (tdbCmprPgId(pPgid1, pPgid2) == 0) - // pgsz_t #define TDB_MIN_PGSIZE 512 // 512B #define TDB_MAX_PGSIZE 16777216 // 16M #define TDB_DEFAULT_PGSIZE 4096 #define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) -// cache -#define TDB_DEFAULT_CACHE_SIZE (256 * 4096) // 1M - // dbname #define TDB_MAX_DBNAME_LEN 24 #define TDB_VARIANT_LEN ((int)-1) -typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); - #define TDB_JOURNAL_NAME "tdb.journal" #define TDB_FILENAME_LEN 128 -#define TDB_DEFAULT_FANOUT 6 - #define BTREE_MAX_DEPTH 20 #define TDB_FLAG_IS(flags, flag) ((flags) == (flag)) @@ -112,37 +84,253 @@ typedef struct SPCache SPCache; typedef struct SPage SPage; // transaction -#define TDB_TXN_WRITE 0x1 -#define TDB_TXN_READ_UNCOMMITTED 0x2 -typedef struct STxn { - int flags; - i64 txnId; - void *(*xMalloc)(void *, size_t); - void (*xFree)(void *, void *); - void *xArg; -} TXN; #define TDB_TXN_IS_WRITE(PTXN) ((PTXN)->flags & TDB_TXN_WRITE) #define TDB_TXN_IS_READ(PTXN) (!TDB_TXN_IS_WRITE(PTXN)) #define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED) -#include "tdbOs.h" +// tdbEnv.c ==================================== +void tdbEnvAddPager(TENV *pEnv, SPager *pPager); +void tdbEnvRemovePager(TENV *pEnv, SPager *pPager); +SPager *tdbEnvGetPager(TENV *pEnv, const char *fname); -#include "tdbUtil.h" +// tdbBtree.c ==================================== +typedef struct SBTree SBTree; +typedef struct SBTC SBTC; +typedef struct SBtInfo { + SPgno root; + int nLevel; + int nData; +} SBtInfo; -#include "tdbPCache.h" +typedef struct { + int kLen; + const u8 *pKey; + int vLen; + const u8 *pVal; + SPgno pgno; + u8 *pBuf; +} SCellDecoder; -#include "tdbPager.h" +struct SBTC { + SBTree *pBt; + i8 iPage; + SPage *pPage; + int idx; + int idxStack[BTREE_MAX_DEPTH + 1]; + SPage *pgStack[BTREE_MAX_DEPTH + 1]; + SCellDecoder coder; + TXN *pTxn; + TXN txn; +}; -#include "tdbBtree.h" +// SBTree +int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBTree **ppBt); +int tdbBtreeClose(SBTree *pBt); +int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); -#include "tdbEnv.h" +// SBTC +int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn); +int tdbBtcClose(SBTC *pBtc); +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); +int tdbBtcMoveToFirst(SBTC *pBtc); +int tdbBtcMoveToLast(SBTC *pBtc); +int tdbBtcMoveToNext(SBTC *pBtc); +int tdbBtcMoveToPrev(SBTC *pBtc); +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); -#include "tdbDb.h" +// tdbPager.c ==================================== -#include "tdbPage.h" +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); +int tdbPagerClose(SPager *pPager); +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); +int tdbPagerWrite(SPager *pPager, SPage *pPage); +int tdbPagerBegin(SPager *pPager, TXN *pTxn); +int tdbPagerCommit(SPager *pPager, TXN *pTxn); +int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, + TXN *pTxn); +void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); +int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); -#include "tdbTxn.h" +// tdbPCache.c ==================================== +#define TDB_PCACHE_PAGE \ + u8 isAnchor; \ + u8 isLocal; \ + u8 isDirty; \ + i32 nRef; \ + SPage *pCacheNext; \ + SPage *pFreeNext; \ + SPage *pHashNext; \ + SPage *pLruNext; \ + SPage *pLruPrev; \ + SPage *pDirtyNext; \ + SPager *pPager; \ + SPgid pgid; + +// For page ref +#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0) +#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1) +#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1) +#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef)) + +int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); +int tdbPCacheClose(SPCache *pCache); +SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn); +void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn); +int tdbPCacheGetPageSize(SPCache *pCache); + +// tdbPage.c ==================================== +typedef u8 SCell; + +// PAGE APIS implemented +typedef struct { + int szOffset; + int szPageHdr; + int szFreeCell; + // cell number + int (*getCellNum)(SPage *); + void (*setCellNum)(SPage *, int); + // cell content offset + int (*getCellBody)(SPage *); + void (*setCellBody)(SPage *, int); + // first free cell offset (0 means no free cells) + int (*getCellFree)(SPage *); + void (*setCellFree)(SPage *, int); + // total free bytes + int (*getFreeBytes)(SPage *); + void (*setFreeBytes)(SPage *, int); + // cell offset at idx + int (*getCellOffset)(SPage *, int); + void (*setCellOffset)(SPage *, int, int); + // free cell info + void (*getFreeCellInfo)(SCell *pCell, int *szCell, int *nxOffset); + void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); +} SPageMethods; + +#pragma pack(push, 1) + +// Page footer +typedef struct { + u8 cksm[4]; +} SPageFtr; +#pragma pack(pop) + +struct SPage { + tdb_spinlock_t lock; + int pageSize; + u8 *pData; + SPageMethods *pPageMethods; + // Fields below used by pager and am + u8 *pPageHdr; + u8 *pCellIdx; + u8 *pFreeStart; + u8 *pFreeEnd; + SPageFtr *pPageFtr; + int nOverflow; + SCell *apOvfl[4]; + int aiOvfl[4]; + int kLen; // key length of the page, -1 for unknown + int vLen; // value length of the page, -1 for unknown + int maxLocal; + int minLocal; + int (*xCellSize)(const SPage *, SCell *); + // Fields used by SPCache + TDB_PCACHE_PAGE +}; + +// For page lock +#define P_LOCK_SUCC 0 +#define P_LOCK_BUSY 1 +#define P_LOCK_FAIL -1 + +static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { + int ret; + if (tdbSpinlockTrylock(pLock) == 0) { + ret = P_LOCK_SUCC; + } else if (errno == EBUSY) { + ret = P_LOCK_BUSY; + } else { + ret = P_LOCK_FAIL; + } + return ret; +} + +#define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) +#define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock)) +#define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock)) +#define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock)) +#define TDB_TRY_LOCK_PAGE(pPage) tdbTryLockPage(&((pPage)->lock)) + +// APIs +#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) +#define TDB_PAGE_FREE_SIZE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) +#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) +#define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) + +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); +int tdbPageDropCell(SPage *pPage, int idx); +void tdbPageCopy(SPage *pFromPage, SPage *pToPage); +int tdbPageCapacity(int pageSize, int amHdrSize); + +static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { + SCell *pCell; + int iOvfl; + int lidx; + + ASSERT(idx >= 0 && idx < TDB_PAGE_TOTAL_CELLS(pPage)); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + pCell = pPage->apOvfl[iOvfl]; + return pCell; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } + } + + lidx = idx - iOvfl; + ASSERT(lidx >= 0 && lidx < pPage->pPageMethods->getCellNum(pPage)); + pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); + + return pCell; +} + +struct STEnv { + char *rootDir; + char *jfname; + int jfd; + SPCache *pCache; + SPager *pgrList; + int nPager; + int nPgrHash; + SPager **pgrHash; +}; + +struct SPager { + char *dbFileName; + char *jFileName; + int pageSize; + uint8_t fid[TDB_FILE_ID_LEN]; + tdb_fd_t fd; + tdb_fd_t jfd; + SPCache *pCache; + SPgno dbFileSize; + SPgno dbOrigSize; + SPage *pDirty; + u8 inTran; + SPager *pNext; // used by TENV + SPager *pHashNext; // used by TENV +}; #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h deleted file mode 100644 index a77cf6248c..0000000000 --- a/source/libs/tdb/src/inc/tdbPCache.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_PAGE_CACHE_H_ -#define _TD_PAGE_CACHE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#define TDB_PCACHE_PAGE \ - u8 isAnchor; \ - u8 isLocal; \ - u8 isDirty; \ - i32 nRef; \ - SPage *pCacheNext; \ - SPage *pFreeNext; \ - SPage *pHashNext; \ - SPage *pLruNext; \ - SPage *pLruPrev; \ - SPage *pDirtyNext; \ - SPager *pPager; \ - SPgid pgid; - -// For page ref -#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0) -#if 0 -#define TDB_REF_PAGE(pPage) (++(pPage)->nRef) -#define TDB_UNREF_PAGE(pPage) (--(pPage)->nRef) -#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef) -#else -#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1) -#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1) -#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef)) -#endif - -int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); -int tdbPCacheClose(SPCache *pCache); -SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn); -void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn); -int tdbPCacheGetPageSize(SPCache *pCache); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_PAGE_CACHE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h deleted file mode 100644 index 3707a922ee..0000000000 --- a/source/libs/tdb/src/inc/tdbPage.h +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TDB_PAGE_H_ -#define _TDB_PAGE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef u8 SCell; - -// PAGE APIS implemented -typedef struct { - int szOffset; - int szPageHdr; - int szFreeCell; - // cell number - int (*getCellNum)(SPage *); - void (*setCellNum)(SPage *, int); - // cell content offset - int (*getCellBody)(SPage *); - void (*setCellBody)(SPage *, int); - // first free cell offset (0 means no free cells) - int (*getCellFree)(SPage *); - void (*setCellFree)(SPage *, int); - // total free bytes - int (*getFreeBytes)(SPage *); - void (*setFreeBytes)(SPage *, int); - // cell offset at idx - int (*getCellOffset)(SPage *, int); - void (*setCellOffset)(SPage *, int, int); - // free cell info - void (*getFreeCellInfo)(SCell *pCell, int *szCell, int *nxOffset); - void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); -} SPageMethods; - -#pragma pack(push,1) - -// Page footer -typedef struct { - u8 cksm[4]; -} SPageFtr; -#pragma pack(pop) - -struct SPage { - tdb_spinlock_t lock; - int pageSize; - u8 *pData; - SPageMethods *pPageMethods; - // Fields below used by pager and am - u8 *pPageHdr; - u8 *pCellIdx; - u8 *pFreeStart; - u8 *pFreeEnd; - SPageFtr *pPageFtr; - int nOverflow; - SCell *apOvfl[4]; - int aiOvfl[4]; - int kLen; // key length of the page, -1 for unknown - int vLen; // value length of the page, -1 for unknown - int maxLocal; - int minLocal; - int (*xCellSize)(const SPage *, SCell *); - // Fields used by SPCache - TDB_PCACHE_PAGE -}; - -// For page lock -#define P_LOCK_SUCC 0 -#define P_LOCK_BUSY 1 -#define P_LOCK_FAIL -1 - -static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { - int ret; - if (tdbSpinlockTrylock(pLock) == 0) { - ret = P_LOCK_SUCC; - } else if (errno == EBUSY) { - ret = P_LOCK_BUSY; - } else { - ret = P_LOCK_FAIL; - } - return ret; -} - -#define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) -#define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock)) -#define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock)) -#define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock)) -#define TDB_TRY_LOCK_PAGE(pPage) tdbTryLockPage(&((pPage)->lock)) - -// APIs -#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) -#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) -#define TDB_PAGE_FREE_SIZE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) -#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) -#define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) - -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); -void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); -int tdbPageDropCell(SPage *pPage, int idx); -void tdbPageCopy(SPage *pFromPage, SPage *pToPage); -int tdbPageCapacity(int pageSize, int amHdrSize); - -static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { - SCell *pCell; - int iOvfl; - int lidx; - - ASSERT(idx >= 0 && idx < TDB_PAGE_TOTAL_CELLS(pPage)); - - iOvfl = 0; - for (; iOvfl < pPage->nOverflow; iOvfl++) { - if (pPage->aiOvfl[iOvfl] == idx) { - pCell = pPage->apOvfl[iOvfl]; - return pCell; - } else if (pPage->aiOvfl[iOvfl] > idx) { - break; - } - } - - lidx = idx - iOvfl; - ASSERT(lidx >= 0 && lidx < pPage->pPageMethods->getCellNum(pPage)); - pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); - - return pCell; -} - -#ifdef __cplusplus -} -#endif - -#endif /*_TDB_PAGE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h deleted file mode 100644 index c39f833c73..0000000000 --- a/source/libs/tdb/src/inc/tdbPager.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TDB_PAGER_H_ -#define _TDB_PAGER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -struct SPager { - char *dbFileName; - char *jFileName; - int pageSize; - uint8_t fid[TDB_FILE_ID_LEN]; - tdb_fd_t fd; - tdb_fd_t jfd; - SPCache *pCache; - SPgno dbFileSize; - SPgno dbOrigSize; - SPage *pDirty; - u8 inTran; - SPager *pNext; // used by TENV - SPager *pHashNext; // used by TENV -}; - -int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); -int tdbPagerClose(SPager *pPager); -int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); -int tdbPagerWrite(SPager *pPager, SPage *pPage); -int tdbPagerBegin(SPager *pPager, TXN *pTxn); -int tdbPagerCommit(SPager *pPager, TXN *pTxn); -int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, - TXN *pTxn); -void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); -int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); - -#ifdef __cplusplus -} -#endif - -#endif /*_TDB_PAGER_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index cffd3f2746..29a505fa78 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -31,27 +31,7 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size); -static inline void *tdbRealloc(void *ptr, size_t size) { - void *nPtr; - if ((ptr) == NULL || ((int *)(ptr))[-1] < (size)) { - nPtr = tdbOsRealloc((ptr) ? (char *)(ptr) - sizeof(int) : NULL, (size) + sizeof(int)); - if (nPtr) { - ((int *)nPtr)[0] = (size); - nPtr = (char *)nPtr + sizeof(int); - } - } else { - nPtr = (ptr); - } - return nPtr; -} -#define TDB_REALLOC(PTR, SIZE) tdbRealloc(PTR, SIZE) - -#define TDB_FREE(PTR) \ - do { \ - if (PTR) { \ - tdbOsFree((char *)(PTR) - sizeof(int)); \ - } \ - } while (0) +void *tdbRealloc(void *ptr, size_t size); static inline void *tdbDefaultMalloc(void *arg, size_t size) { void *ptr; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 49763ae937..1f6f28261f 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,7 +1,7 @@ #include #include "os.h" -#include "tdbInt.h" +#include "tdb.h" #include @@ -12,7 +12,7 @@ typedef struct SPoolMem { } SPoolMem; static SPoolMem *openPool() { - SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool)); + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); pPool->prev = pPool->next = pPool; pPool->size = 0; @@ -32,7 +32,7 @@ static void clearPool(SPoolMem *pPool) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - tdbOsFree(pMem); + taosMemoryFree(pMem); } while (1); assert(pPool->size == 0); @@ -40,7 +40,7 @@ static void clearPool(SPoolMem *pPool) { static void closePool(SPoolMem *pPool) { clearPool(pPool); - tdbOsFree(pPool); + taosMemoryFree(pPool); } static void *poolMalloc(void *arg, size_t size) { @@ -48,7 +48,7 @@ static void *poolMalloc(void *arg, size_t size) { SPoolMem *pPool = (SPoolMem *)arg; SPoolMem *pMem; - pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size); + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); if (pMem == NULL) { assert(0); } @@ -75,7 +75,7 @@ static void poolFree(void *arg, void *ptr) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - tdbOsFree(pMem); + taosMemoryFree(pMem); } static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { @@ -116,12 +116,12 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in } TEST(tdb_test, simple_test) { - int ret; - TENV *pEnv; - TDB *pDb; - FKeyComparator compFunc; - int nData = 10000000; - TXN txn; + int ret; + TENV *pEnv; + TDB *pDb; + tdb_cmpr_fn_t compFunc; + int nData = 10000000; + TXN txn; taosRemoveDir("tdb"); @@ -131,7 +131,7 @@ TEST(tdb_test, simple_test) { // Create a database compFunc = tKeyCmpr; - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb); + ret = tdbDbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -152,7 +152,7 @@ TEST(tdb_test, simple_test) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one @@ -189,7 +189,7 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); } - TDB_FREE(pVal); + tdbFree(pVal); } { // Iterate to query the DB data @@ -199,11 +199,11 @@ TEST(tdb_test, simple_test) { int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC); + ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); for (;;) { - ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; @@ -217,8 +217,8 @@ TEST(tdb_test, simple_test) { tdbDbcClose(pDBC); - TDB_FREE(pKey); - TDB_FREE(pVal); + tdbFree(pKey); + tdbFree(pVal); } } @@ -234,12 +234,12 @@ TEST(tdb_test, simple_test) { } TEST(tdb_test, simple_test2) { - int ret; - TENV *pEnv; - TDB *pDb; - FKeyComparator compFunc; - int nData = 1000000; - TXN txn; + int ret; + TENV *pEnv; + TDB *pDb; + tdb_cmpr_fn_t compFunc; + int nData = 1000000; + TXN txn; taosRemoveDir("tdb"); @@ -249,7 +249,7 @@ TEST(tdb_test, simple_test2) { // Create a database compFunc = tDefaultKeyCmpr; - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb); + ret = tdbDbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -269,7 +269,7 @@ TEST(tdb_test, simple_test2) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -280,11 +280,11 @@ TEST(tdb_test, simple_test2) { int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC); + ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); for (;;) { - ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; @@ -298,8 +298,8 @@ TEST(tdb_test, simple_test2) { tdbDbcClose(pDBC); - TDB_FREE(pKey); - TDB_FREE(pVal); + tdbFree(pKey); + tdbFree(pVal); } } diff --git a/source/libs/tdb/test/tdbUtilTest.cpp b/source/libs/tdb/test/tdbUtilTest.cpp index 5fd45daa3e..57372995cc 100644 --- a/source/libs/tdb/test/tdbUtilTest.cpp +++ b/source/libs/tdb/test/tdbUtilTest.cpp @@ -1,20 +1,20 @@ #include -#include "tdbInt.h" +#include "tdb.h" #include TEST(tdb_util_test, simple_test) { - int vEncode = 5000; - int vDecode; - int nEncode; - int nDecode; - u8 buffer[128]; + // int vEncode = 5000; + // int vDecode; + // int nEncode; + // int nDecode; + // uint8_t buffer[128]; - nEncode = tdbPutVarInt(buffer, vEncode); + // nEncode = tdbPutVarInt(buffer, vEncode); - nDecode = tdbGetVarInt(buffer, &vDecode); + // nDecode = tdbGetVarInt(buffer, &vDecode); - GTEST_ASSERT_EQ(nEncode, nDecode); - GTEST_ASSERT_EQ(vEncode, vDecode); + // GTEST_ASSERT_EQ(nEncode, nDecode); + // GTEST_ASSERT_EQ(vEncode, vDecode); } \ No newline at end of file diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index fecb5d9279..f776fb3764 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -101,18 +101,8 @@ void* rpcReallocCont(void* ptr, int contLen) { } void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { - SRpcMsg rpcMsg; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - - SMEpSet msg = {.epSet = *pEpSet}; - int32_t len = tSerializeSMEpSet(NULL, 0, &msg); - rpcMsg.pCont = rpcMallocCont(len); - tSerializeSMEpSet(rpcMsg.pCont, len, &msg); - - rpcMsg.code = TSDB_CODE_RPC_REDIRECT; - rpcMsg.handle = thandle; - - rpcSendResponse(&rpcMsg); + // deprecated api + assert(0); } int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c20303547c..80b5d88a90 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -914,8 +914,9 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tmsg_t msgType = pCtx->msgType; if ((pTransInst->retry != NULL && (pTransInst->retry(pResp->code))) || ((pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && msgType == TDMT_MND_CONNECT)) { - pCtx->retryCount += 1; + pMsg->sent = 0; pMsg->st = taosGetTimestampUs(); + pCtx->retryCount += 1; if (msgType == TDMT_MND_CONNECT && pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps) { pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; @@ -936,12 +937,14 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tDeserializeSMEpSet(pResp->pCont, pResp->contLen, &emsg); pCtx->epSet = emsg.epSet; } + addConnToPool(pThrd, pConn); + tTrace("use remote epset, current in use: %d, retry count%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, + TRANS_RETRY_COUNT_LIMIT); + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - addConnToPool(pThrd, pConn); return -1; } } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index f348f6c64c..b416b404a9 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -765,8 +765,10 @@ static void destroyConn(SSrvConn* conn, bool clear) { transDestroyBuffer(&conn->readBuf); if (clear) { tTrace("server conn %p to be destroyed", conn); - uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); - uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); + // uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); + uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); + // uv_close(conn->pTcp) + // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } } static void uvDestroyConn(uv_handle_t* handle) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a37b9750e1..87ef94f9b5 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -317,6 +317,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TSDB_STATE, "Invalid tsdb state") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST, "Table not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_SMA_NOT_EXIST, "SMA not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Hash value mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST, "Table does not exists") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index b1ab5253ad..f14885b72e 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -11,7 +11,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR, TSDB_DATA_TYPE_BOOL}; +int32_t optrIdxList[] = {2, 11, 6}; typedef struct { char* oper; @@ -32,7 +33,7 @@ OperInfo operInfo[] = { {"like", 2, false}, {"not like", 2, false}, {"match", 2, false}, - {"nmake", 2, false}, + {"nmatch", 2, false}, }; int32_t operatorList[] = {0, 1, 2, 3, 4, 5, 6, 7}; @@ -139,9 +140,7 @@ CaseCfg gCase[] = { {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1, -1}, // 22 - //{"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 1, 2}, - - {"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 1, 10, 1, 3, 0, 1, 2}, + {"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 1, 2}, }; @@ -161,6 +160,8 @@ typedef struct { int32_t bindRowNum; //row num for once bind int32_t bindColTypeNum; int32_t* bindColTypeList; + int32_t optrIdxListNum; + int32_t* optrIdxList; int32_t runTimes; int32_t caseIdx; // static case idx int32_t caseNum; // num in static case list @@ -178,8 +179,10 @@ CaseCtrl gCaseCtrl = { .rowNum = 0, .bindColNum = 0, .bindRowNum = 0, -// .bindColTypeNum = 0, -// .bindColTypeList = NULL, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, .checkParamNum = false, .printRes = true, .runTimes = 0, @@ -188,8 +191,11 @@ CaseCtrl gCaseCtrl = { .caseRunIdx = -1, // .caseRunNum = -1, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, + +// .optrIdxListNum = tListLen(optrIdxList), +// .optrIdxList = optrIdxList, +// .bindColTypeNum = tListLen(bindColTypeList), +// .bindColTypeList = bindColTypeList, .caseIdx = 22, .caseNum = 1, .caseRunNum = 1, @@ -210,10 +216,10 @@ CaseCtrl gCaseCtrl = { .checkParamNum = false, .printRes = true, .runTimes = 0, - .caseIdx = -1, - .caseNum = -1, + .caseIdx = 2, + .caseNum = 1, .caseRunIdx = -1, - .caseRunNum = -1, + .caseRunNum = 1, }; #endif @@ -318,7 +324,7 @@ void generateInsertSQL(BindData *data) { len += sprintf(data->sql + len, "ubigdata"); break; default: - printf("invalid col type:%d", data->pBind[c].buffer_type); + printf("!!!invalid col type:%d", data->pBind[c].buffer_type); exit(1); } } @@ -336,19 +342,23 @@ void generateInsertSQL(BindData *data) { len += sprintf(data->sql + len, ")"); if (gCaseCtrl.printStmtSql) { - printf("SQL: %s\n", data->sql); + printf("\tSQL: %s\n", data->sql); } } -void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType) { +void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType, int32_t idx) { OperInfo *pInfo = NULL; - - if (TSDB_DATA_TYPE_VARCHAR == dataType || TSDB_DATA_TYPE_NCHAR == dataType) { - pInfo = &operInfo[varoperatorList[rand() % tListLen(varoperatorList)]]; - } else { - pInfo = &operInfo[operatorList[rand() % tListLen(operatorList)]]; - } + if (gCaseCtrl.optrIdxListNum > 0) { + pInfo = &operInfo[gCaseCtrl.optrIdxList[idx]]; + } else { + if (TSDB_DATA_TYPE_VARCHAR == dataType || TSDB_DATA_TYPE_NCHAR == dataType) { + pInfo = &operInfo[varoperatorList[rand() % tListLen(varoperatorList)]]; + } else { + pInfo = &operInfo[operatorList[rand() % tListLen(operatorList)]]; + } + } + switch (pInfo->paramNum) { case 2: if (pInfo->enclose) { @@ -358,7 +368,7 @@ void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType) { } break; default: - printf("invalid paramNum:%d\n", pInfo->paramNum); + printf("!!!invalid paramNum:%d\n", pInfo->paramNum); exit(1); } } @@ -414,16 +424,16 @@ void generateQuerySQL(BindData *data, int32_t tblIdx) { len += sprintf(data->sql + len, "ubigdata"); break; default: - printf("invalid col type:%d", data->pBind[c].buffer_type); + printf("!!!invalid col type:%d", data->pBind[c].buffer_type); exit(1); } - bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type); + bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c); } } if (gCaseCtrl.printStmtSql) { - printf("SQL: %s\n", data->sql); + printf("\tSTMT SQL: %s\n", data->sql); } } @@ -551,7 +561,7 @@ int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; default: - printf("invalid col type:%d", dataType); + printf("!!!invalid col type:%d", dataType); exit(1); } @@ -709,7 +719,7 @@ void bpFetchRows(TAOS_RES *result, bool printr, int32_t *rows) { if (printr) { memset(temp, 0, sizeof(temp)); taos_print_row(temp, row, fields, num_fields); - printf("[%s]\n", temp); + printf("\t[%s]\n", temp); } } } @@ -718,7 +728,7 @@ void bpExecQuery(TAOS * taos, char* sql, bool printr, int32_t *rows) { TAOS_RES *result = taos_query(taos, sql); int code = taos_errno(result); if (code != 0) { - printf("failed to query table, reason:%s\n", taos_errstr(result)); + printf("!!!failed to query table, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -791,7 +801,7 @@ int32_t bpAppendValueString(char *buf, int type, void *value, int32_t valueLen, break; default: - printf("invalid data type:%d\n", type); + printf("!!!invalid data type:%d\n", type); exit(1); } } @@ -803,13 +813,13 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (gCurCase->bindRowNum > 1) { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { - printf("taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } else { for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { if (taos_stmt_bind_single_param_batch(stmt, bind++, i)) { - printf("taos_stmt_bind_single_param_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_bind_single_param_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -817,12 +827,12 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { - printf("taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } else { if (taos_stmt_bind_param(stmt, bind)) { - printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -834,12 +844,12 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { void bpCheckIsInsert(TAOS_STMT *stmt, int32_t insert) { int32_t isInsert = 0; if (taos_stmt_is_insert(stmt, &isInsert)) { - printf("taos_stmt_is_insert error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_is_insert error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (insert != isInsert) { - printf("is insert failed\n"); + printf("!!!is insert failed\n"); exit(1); } } @@ -847,12 +857,12 @@ void bpCheckIsInsert(TAOS_STMT *stmt, int32_t insert) { void bpCheckParamNum(TAOS_STMT *stmt) { int32_t num = 0; if (taos_stmt_num_params(stmt, &num)) { - printf("taos_stmt_num_params error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_num_params error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (gCurCase->bindColNum != num) { - printf("is insert failed\n"); + printf("!!!is insert failed\n"); exit(1); } } @@ -861,7 +871,7 @@ void bpCheckAffectedRows(TAOS_STMT *stmt, int32_t times) { int32_t rows = taos_stmt_affected_rows(stmt); int32_t insertNum = gCurCase->rowNum * gCurCase->tblNum * times; if (insertNum != rows) { - printf("affected rows %d mis-match with insert num %d\n", rows, insertNum); + printf("!!!affected rows %d mis-match with insert num %d\n", rows, insertNum); exit(1); } } @@ -869,7 +879,7 @@ void bpCheckAffectedRows(TAOS_STMT *stmt, int32_t times) { void bpCheckAffectedRowsOnce(TAOS_STMT *stmt, int32_t expectedNum) { int32_t rows = taos_stmt_affected_rows_once(stmt); if (expectedNum != rows) { - printf("affected rows %d mis-match with expected num %d\n", rows, expectedNum); + printf("!!!affected rows %d mis-match with expected num %d\n", rows, expectedNum); exit(1); } } @@ -904,16 +914,16 @@ void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_B } if (gCaseCtrl.printQuerySql) { - printf("Query SQL: %s\n", sql); + printf("\tQuery SQL: %s\n", sql); } bpExecQuery(taos, sql, gCaseCtrl.printRes, &sqlResNum); if (sqlResNum != stmtResNum) { - printf("sql res num %d mis-match stmt res num %d\n", sqlResNum, stmtResNum); + printf("!!!sql res num %d mis-match stmt res num %d\n", sqlResNum, stmtResNum); exit(1); } - printf("sql res num match stmt res num %d\n", stmtResNum); + printf("***sql res num match stmt res num %d\n", stmtResNum); } /* prepare [settbname [bind add]] exec */ @@ -923,7 +933,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -936,7 +946,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -951,14 +961,14 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -978,7 +988,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -993,7 +1003,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1007,14 +1017,14 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1033,7 +1043,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1046,7 +1056,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1061,13 +1071,13 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1087,7 +1097,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1100,7 +1110,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1115,12 +1125,12 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1141,7 +1151,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1154,7 +1164,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1169,7 +1179,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1179,12 +1189,12 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1206,7 +1216,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1221,7 +1231,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1235,12 +1245,12 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1264,7 +1274,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1277,7 +1287,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1292,13 +1302,13 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1328,7 +1338,7 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1344,12 +1354,12 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -4249,10 +4259,10 @@ void prepareCheckResultImpl(TAOS * taos, char *tname, bool printr, int expec if (rows == expected) { if (!silent) { - printf("%d rows are fetched as expected from %s\n", rows, tname); + printf("***%d rows are fetched as expected from %s\n", rows, tname); } } else { - printf("!!!expect %d rows, but %d rows are fetched from %s\n", expected, rows, tname); + printf("!!!expect rows %d mis-match rows %d fetched from %s\n", expected, rows, tname); exit(1); } } @@ -4302,7 +4312,7 @@ int sql_perf1(TAOS *taos) { result = taos_query(taos, sql[i]); int code = taos_errno(result); if (code != 0) { - printf("failed to query table, reason:%s\n", taos_errstr(result)); + printf("%d failed to query table, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -4539,7 +4549,7 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * blen += sprintf(buf + blen, ")"); if (gCaseCtrl.printCreateTblSql) { - printf("Create Table SQL:%s\n", buf); + printf("\tCreate Table SQL:%s\n", buf); } } @@ -4553,7 +4563,7 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { result = taos_query(taos, "create database demo keep 36500"); code = taos_errno(result); if (code != 0) { - printf("failed to create database, reason:%s\n", taos_errstr(result)); + printf("!!!failed to create database, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -4570,7 +4580,7 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); + printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -4583,7 +4593,7 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); + printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -4654,7 +4664,7 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { stmt = taos_stmt_init(taos); if (NULL == stmt) { - printf("taos_stmt_init failed, error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_init failed, error:%s\n", taos_stmt_errstr(stmt)); exit(1); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a8e2c22d42..6f91235d90 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -66,7 +66,7 @@ # --- stable ./test.sh -f tsim/stable/disk.sim -./test.sh -f tsim/stable/dnode3.sim +#./test.sh -f tsim/stable/dnode3.sim ./test.sh -f tsim/stable/metrics.sim ./test.sh -f tsim/stable/refcount.sim # ./test.sh -f tsim/stable/show.sim @@ -81,7 +81,7 @@ ./test.sh -f tsim/insert/backquote.sim -m ./test.sh -f tsim/parser/fourArithmetic-basic.sim -m ./test.sh -f tsim/query/interval-offset.sim -m -./test.sh -f tsim/tmq/basic3.sim -m +#./test.sh -f tsim/tmq/basic3.sim -m ./test.sh -f tsim/stable/vnode3.sim -m ./test.sh -f tsim/qnode/basic1.sim -m ./test.sh -f tsim/mnode/basic1.sim -m diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index 7a95a76df8..5f2f79982a 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -86,7 +86,7 @@ class TDTestCase: updatecfgDict["serverPort"] = serverPort updatecfgDict["firstEp"] = hostname + ':' + serverPort updatecfgDict["secondEp"] = hostname + ':' + serverPort - clientCfgDict["fqdn"] = hostname + updatecfgDict["fqdn"] = hostname print ("===================: ", updatecfgDict) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py new file mode 100644 index 0000000000..ccf83df952 --- /dev/null +++ b/tests/system-test/2-query/abs.py @@ -0,0 +1,483 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto(self ,origin_query , abs_query): + abs_result = tdSql.getResult(abs_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >=0: + elem = elem + else: + elem = -elem + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(abs_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("abs function value has not as expected , sql is \"%s\" "%abs_query ) + sys.exit(1) + else: + tdLog.info("abs value check pass , it work as expected ,sql is \"%s\" "%abs_query ) + + def test_errors(self): + error_sql_lists = [ + "select abs from t1", + # "select abs(-+--+c1) from t1", + # "select +-abs(c1) from t1", + # "select ++-abs(c1) from t1", + # "select ++--abs(c1) from t1", + # "select - -abs(c1)*0 from t1", + # "select abs(tbname+1) from t1 ", + "select abs(123--123)==1 from t1", + "select abs(c1) as 'd1' from t1", + "select abs(c1 ,c2 ) from t1", + "select abs(c1 ,NULL) from t1", + "select abs(,) from t1;", + "select abs(abs(c1) ab from t1)", + "select abs(c1) as int from t1", + "select abs from stb1", + # "select abs(-+--+c1) from stb1", + # "select +-abs(c1) from stb1", + # "select ++-abs(c1) from stb1", + # "select ++--abs(c1) from stb1", + # "select - -abs(c1)*0 from stb1", + # "select abs(tbname+1) from stb1 ", + "select abs(123--123)==1 from stb1", + "select abs(c1) as 'd1' from stb1", + "select abs(c1 ,c2 ) from stb1", + "select abs(c1 ,NULL) from stb1", + "select abs(,) from stb1;", + "select abs(abs(c1) ab from stb1)", + "select abs(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select abs(ts) from t1" , + "select abs(c7) from t1", + "select abs(c8) from t1", + "select abs(c9) from t1", + "select abs(ts) from ct1" , + "select abs(c7) from ct1", + "select abs(c8) from ct1", + "select abs(c9) from ct1", + "select abs(ts) from ct3" , + "select abs(c7) from ct3", + "select abs(c8) from ct3", + "select abs(c9) from ct3", + "select abs(ts) from ct4" , + "select abs(c7) from ct4", + "select abs(c8) from ct4", + "select abs(c9) from ct4", + "select abs(ts) from stb1" , + "select abs(c7) from stb1", + "select abs(c8) from stb1", + "select abs(c9) from stb1" , + + "select abs(ts) from stbbb1" , + "select abs(c7) from stbbb1", + + "select abs(ts) from tbname", + "select abs(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select abs(c1) from t1", + "select abs(c2) from t1", + "select abs(c3) from t1", + "select abs(c4) from t1", + "select abs(c5) from t1", + "select abs(c6) from t1", + + "select abs(c1) from ct1", + "select abs(c2) from ct1", + "select abs(c3) from ct1", + "select abs(c4) from ct1", + "select abs(c5) from ct1", + "select abs(c6) from ct1", + + "select abs(c1) from ct3", + "select abs(c2) from ct3", + "select abs(c3) from ct3", + "select abs(c4) from ct3", + "select abs(c5) from ct3", + "select abs(c6) from ct3", + + "select abs(c1) from stb1", + "select abs(c2) from stb1", + "select abs(c3) from stb1", + "select abs(c4) from stb1", + "select abs(c5) from stb1", + "select abs(c6) from stb1", + + "select abs(c6) as alisb from stb1", + "select abs(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_abs_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select abs(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c6) from ct3") + + # used for regular table + tdSql.query("select abs(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1) + tdSql.checkData(3 , 0, 3) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto( "select c1, c2, c3 , c4, c5 from t1", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from t1") + + # used for sub table + tdSql.query("select abs(c1) from ct1") + tdSql.checkData(0, 0, 8) + tdSql.checkData(1 , 0, 7) + tdSql.checkData(3 , 0, 5) + tdSql.checkData(5 , 0, 4) + + tdSql.query("select abs(c1) from ct1") + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct1", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct1") + self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct1;","select c1 from ct1" ) + + # used for stable table + + tdSql.query("select abs(c1) from stb1") + tdSql.checkRows(25) + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct4 ", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct4") + self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct4;" , "select c1 from ct4" ) + + + # used for not exists table + tdSql.error("select abs(c1) from stbbb1") + tdSql.error("select abs(c1) from tbname") + tdSql.error("select abs(c1) from ct5") + + # mix with common col + tdSql.query("select c1, abs(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,8) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0) + tdSql.query("select c1, abs(c1) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + tdSql.query("select c1, abs(c1) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + + # mix with common functions + tdSql.query("select c1, abs(c1),c5, floor(c5) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 , 6) + tdSql.checkData(3 , 2 ,6.66000) + tdSql.checkData(3 , 3 ,6.00000) + + tdSql.query("select c1, abs(c1),c5, floor(c5) from stb1 ") + + # mix with agg functions , not support + tdSql.error("select c1, abs(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, abs(c1),c5, count(c5) from ct1 ") + tdSql.error("select abs(c1), count(c5) from stb1 ") + tdSql.error("select abs(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # bug fix for compute + tdSql.query("select c1, abs(c1) -0 ,ceil(c1)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 8.000000000) + + tdSql.query(" select c1, abs(c1) -0 ,ceil(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 7.900000000) + + + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + + + def abs_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select abs(c1), abs(c2) ,abs(c3), abs(c4), abs(c5) ,abs(c6) from sub1_bound") + self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select abs(c1), abs(c2) ,abs(c3), abs(c3), abs(c2) ,abs(c1) from sub1_bound") + self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from sub1_bound;" , "select abs(c1) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from sub1_bound ") + tdSql.checkData(0,0,2147483647) + tdSql.checkData(0,1,9223372036854775807) + tdSql.checkData(0,2,32767) + tdSql.checkData(0,3,127) + tdSql.checkData(0,4,339999995214436424907732413799364296704.00000) + tdSql.checkData(0,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + tdSql.checkData(1,0,2147483647) + tdSql.checkData(1,1,9223372036854775807) + tdSql.checkData(1,2,32767) + tdSql.checkData(1,3,127) + tdSql.checkData(1,4,339999995214436424907732413799364296704.00000) + tdSql.checkData(1,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + tdSql.checkData(3,0,2147483646) + tdSql.checkData(3,1,9223372036854775806) + tdSql.checkData(3,2,32766) + tdSql.checkData(3,3,126) + tdSql.checkData(3,4,339999995214436424907732413799364296704.00000) + tdSql.checkData(3,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + + # check + - * / in functions + tdSql.query("select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ") + tdSql.checkData(0,0,2147483648.000000000) + tdSql.checkData(0,1,9223372036854775807) + tdSql.checkData(0,2,32767.000000000) + tdSql.checkData(0,3,63.500000000) + tdSql.checkData(0,4,169999997607218212453866206899682148352.000000000) + tdSql.checkData(0,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + + + tdSql.checkData(1,0,2147483646.000000000) + tdSql.checkData(1,1,9223372036854775808.000000000) + tdSql.checkData(1,2,32767.000000000) + tdSql.checkData(1,3,63.500000000) + tdSql.checkData(1,4,169999997607218212453866206899682148352.000000000) + + self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ") + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: abs basic query ============") + + self.basic_abs_function() + + tdLog.printNoPrefix("==========step5: abs boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: abs filter query ============") + + self.abs_func_filter() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py new file mode 100644 index 0000000000..4196a5a8ce --- /dev/null +++ b/tests/system-test/2-query/ceil.py @@ -0,0 +1,463 @@ +from math import ceil +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto(self ,origin_query , ceil_query): + pass + ceil_result = tdSql.getResult(ceil_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = ceil(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(ceil_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("ceil function value has not as expected , sql is \"%s\" "%ceil_query ) + sys.exit(1) + else: + tdLog.info("ceil value check pass , it work as expected ,sql is \"%s\" "%ceil_query ) + + def test_errors(self): + error_sql_lists = [ + "select ceil from t1", + # "select ceil(-+--+c1) from t1", + # "select +-ceil(c1) from t1", + # "select ++-ceil(c1) from t1", + # "select ++--ceil(c1) from t1", + # "select - -ceil(c1)*0 from t1", + # "select ceil(tbname+1) from t1 ", + "select ceil(123--123)==1 from t1", + "select ceil(c1) as 'd1' from t1", + "select ceil(c1 ,c2 ) from t1", + "select ceil(c1 ,NULL) from t1", + "select ceil(,) from t1;", + "select ceil(ceil(c1) ab from t1)", + "select ceil(c1) as int from t1", + "select ceil from stb1", + # "select ceil(-+--+c1) from stb1", + # "select +-ceil(c1) from stb1", + # "select ++-ceil(c1) from stb1", + # "select ++--ceil(c1) from stb1", + # "select - -ceil(c1)*0 from stb1", + # "select ceil(tbname+1) from stb1 ", + "select ceil(123--123)==1 from stb1", + "select ceil(c1) as 'd1' from stb1", + "select ceil(c1 ,c2 ) from stb1", + "select ceil(c1 ,NULL) from stb1", + "select ceil(,) from stb1;", + "select ceil(ceil(c1) ab from stb1)", + "select ceil(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select ceil(ts) from t1" , + "select ceil(c7) from t1", + "select ceil(c8) from t1", + "select ceil(c9) from t1", + "select ceil(ts) from ct1" , + "select ceil(c7) from ct1", + "select ceil(c8) from ct1", + "select ceil(c9) from ct1", + "select ceil(ts) from ct3" , + "select ceil(c7) from ct3", + "select ceil(c8) from ct3", + "select ceil(c9) from ct3", + "select ceil(ts) from ct4" , + "select ceil(c7) from ct4", + "select ceil(c8) from ct4", + "select ceil(c9) from ct4", + "select ceil(ts) from stb1" , + "select ceil(c7) from stb1", + "select ceil(c8) from stb1", + "select ceil(c9) from stb1" , + + "select ceil(ts) from stbbb1" , + "select ceil(c7) from stbbb1", + + "select ceil(ts) from tbname", + "select ceil(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select ceil(c1) from t1", + "select ceil(c2) from t1", + "select ceil(c3) from t1", + "select ceil(c4) from t1", + "select ceil(c5) from t1", + "select ceil(c6) from t1", + + "select ceil(c1) from ct1", + "select ceil(c2) from ct1", + "select ceil(c3) from ct1", + "select ceil(c4) from ct1", + "select ceil(c5) from ct1", + "select ceil(c6) from ct1", + + "select ceil(c1) from ct3", + "select ceil(c2) from ct3", + "select ceil(c3) from ct3", + "select ceil(c4) from ct3", + "select ceil(c5) from ct3", + "select ceil(c6) from ct3", + + "select ceil(c1) from stb1", + "select ceil(c2) from stb1", + "select ceil(c3) from stb1", + "select ceil(c4) from stb1", + "select ceil(c5) from stb1", + "select ceil(c6) from stb1", + + "select ceil(c6) as alisb from stb1", + "select ceil(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_ceil_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select ceil(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c6) from ct3") + + # used for regular table + tdSql.query("select ceil(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1) + tdSql.checkData(3 , 0, 3) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto( "select c1, c2, c3 , c4, c5 from t1", "select (c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) from t1") + + # used for sub table + tdSql.query("select ceil(c1) from ct1") + tdSql.checkData(0, 0, 8) + tdSql.checkData(1 , 0, 7) + tdSql.checkData(3 , 0, 5) + tdSql.checkData(5 , 0, 4) + + tdSql.query("select ceil(c1) from ct1") + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct1", "select (c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) from ct1") + self.check_result_auto("select ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(c1)))))))))) nest_col_func from ct1;","select c1 from ct1" ) + + # used for stable table + + tdSql.query("select ceil(c1) from stb1") + tdSql.checkRows(25) + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct4 ", "select (c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) from ct4") + self.check_result_auto("select ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(c1)))))))))) nest_col_func from ct4;" , "select c1 from ct4" ) + + + # used for not exists table + tdSql.error("select ceil(c1) from stbbb1") + tdSql.error("select ceil(c1) from tbname") + tdSql.error("select ceil(c1) from ct5") + + # mix with common col + tdSql.query("select c1, ceil(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,8) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0) + tdSql.query("select c1, ceil(c1) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + tdSql.query("select c1, ceil(c1) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + + # mix with common functions + tdSql.query("select c1, ceil(c1),c5, ceil(c5) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 , 6) + tdSql.checkData(3 , 2 ,6.66000) + tdSql.checkData(3 , 3 ,7.00000) + + tdSql.query("select c1, ceil(c1),c5, floor(c5) from stb1 ") + + # mix with agg functions , not support + tdSql.error("select c1, ceil(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, ceil(c1),c5, count(c5) from ct1 ") + tdSql.error("select ceil(c1), count(c5) from stb1 ") + tdSql.error("select ceil(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # bug fix for compute + tdSql.query("select c1, abs(c1) -0 ,ceil(c1)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 8.000000000) + + tdSql.query(" select c1, abs(c1) -0 ,ceil(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 7.900000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + + def ceil_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select ceil(c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) ,ceil(c6) from sub1_bound") + self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select ceil(c1), ceil(c2) ,ceil(c3), ceil(c3), ceil(c2) ,ceil(c1) from sub1_bound") + self.check_result_auto("select ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(c1)))))))))) nest_col_func from sub1_bound;" , "select ceil(c1) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select ceil(c1+0.2) ,ceil(c2) , ceil(c3+0.3) , ceil(c4-0.3), ceil(c5/2), ceil(c6/2) from sub1_bound ") + tdSql.checkData(0, 0, 2147483648.000000000) + tdSql.checkData(0, 2, 32768.000000000) + tdSql.checkData(0, 3, 127.000000000) + tdSql.checkData(0, 4, 169999997607218212453866206899682148352.000000000) + + tdSql.checkData(4, 0, -2147483642.000000000) + tdSql.checkData(4, 2, -32762.000000000) + tdSql.checkData(4, 3, -123.000000000) + tdSql.checkData(4, 4, -169499995645668991474575059260979281920.000000000) + + self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select ceil(c1+1) ,ceil(c2) , ceil(c3*1) , ceil(c4/2), ceil(c5)/2, ceil(c6) from sub1_bound ") + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: ceil basic query ============") + + self.basic_ceil_function() + + tdLog.printNoPrefix("==========step5: ceil boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: ceil filter query ============") + + self.abs_func_filter() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py new file mode 100644 index 0000000000..c955c24e05 --- /dev/null +++ b/tests/system-test/2-query/floor.py @@ -0,0 +1,462 @@ +from math import floor +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto(self ,origin_query , floor_query): + pass + floor_result = tdSql.getResult(floor_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = floor(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(floor_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("floor function value has not as expected , sql is \"%s\" "%floor_query ) + sys.exit(1) + else: + tdLog.info("floor value check pass , it work as expected ,sql is \"%s\" "%floor_query ) + + def test_errors(self): + error_sql_lists = [ + "select floor from t1", + # "select floor(-+--+c1) from t1", + # "select +-floor(c1) from t1", + # "select ++-floor(c1) from t1", + # "select ++--floor(c1) from t1", + # "select - -floor(c1)*0 from t1", + # "select floor(tbname+1) from t1 ", + "select floor(123--123)==1 from t1", + "select floor(c1) as 'd1' from t1", + "select floor(c1 ,c2 ) from t1", + "select floor(c1 ,NULL) from t1", + "select floor(,) from t1;", + "select floor(floor(c1) ab from t1)", + "select floor(c1) as int from t1", + "select floor from stb1", + # "select floor(-+--+c1) from stb1", + # "select +-floor(c1) from stb1", + # "select ++-floor(c1) from stb1", + # "select ++--floor(c1) from stb1", + # "select - -floor(c1)*0 from stb1", + # "select floor(tbname+1) from stb1 ", + "select floor(123--123)==1 from stb1", + "select floor(c1) as 'd1' from stb1", + "select floor(c1 ,c2 ) from stb1", + "select floor(c1 ,NULL) from stb1", + "select floor(,) from stb1;", + "select floor(floor(c1) ab from stb1)", + "select floor(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select floor(ts) from t1" , + "select floor(c7) from t1", + "select floor(c8) from t1", + "select floor(c9) from t1", + "select floor(ts) from ct1" , + "select floor(c7) from ct1", + "select floor(c8) from ct1", + "select floor(c9) from ct1", + "select floor(ts) from ct3" , + "select floor(c7) from ct3", + "select floor(c8) from ct3", + "select floor(c9) from ct3", + "select floor(ts) from ct4" , + "select floor(c7) from ct4", + "select floor(c8) from ct4", + "select floor(c9) from ct4", + "select floor(ts) from stb1" , + "select floor(c7) from stb1", + "select floor(c8) from stb1", + "select floor(c9) from stb1" , + + "select floor(ts) from stbbb1" , + "select floor(c7) from stbbb1", + + "select floor(ts) from tbname", + "select floor(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select floor(c1) from t1", + "select floor(c2) from t1", + "select floor(c3) from t1", + "select floor(c4) from t1", + "select floor(c5) from t1", + "select floor(c6) from t1", + + "select floor(c1) from ct1", + "select floor(c2) from ct1", + "select floor(c3) from ct1", + "select floor(c4) from ct1", + "select floor(c5) from ct1", + "select floor(c6) from ct1", + + "select floor(c1) from ct3", + "select floor(c2) from ct3", + "select floor(c3) from ct3", + "select floor(c4) from ct3", + "select floor(c5) from ct3", + "select floor(c6) from ct3", + + "select floor(c1) from stb1", + "select floor(c2) from stb1", + "select floor(c3) from stb1", + "select floor(c4) from stb1", + "select floor(c5) from stb1", + "select floor(c6) from stb1", + + "select floor(c6) as alisb from stb1", + "select floor(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_floor_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select floor(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c6) from ct3") + + # used for regular table + tdSql.query("select floor(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1) + tdSql.checkData(3 , 0, 3) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto( "select c1, c2, c3 , c4, c5 from t1", "select (c1), floor(c2) ,floor(c3), floor(c4), floor(c5) from t1") + + # used for sub table + tdSql.query("select floor(c1) from ct1") + tdSql.checkData(0, 0, 8) + tdSql.checkData(1 , 0, 7) + tdSql.checkData(3 , 0, 5) + tdSql.checkData(5 , 0, 4) + + tdSql.query("select floor(c1) from ct1") + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct1", "select (c1), floor(c2) ,floor(c3), floor(c4), floor(c5) from ct1") + self.check_result_auto("select floor(floor(floor(floor(floor(floor(floor(floor(floor(floor(c1)))))))))) nest_col_func from ct1;","select c1 from ct1" ) + + # used for stable table + + tdSql.query("select floor(c1) from stb1") + tdSql.checkRows(25) + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct4 ", "select (c1), floor(c2) ,floor(c3), floor(c4), floor(c5) from ct4") + self.check_result_auto("select floor(floor(floor(floor(floor(floor(floor(floor(floor(floor(c1)))))))))) nest_col_func from ct4;" , "select c1 from ct4" ) + + + # used for not exists table + tdSql.error("select floor(c1) from stbbb1") + tdSql.error("select floor(c1) from tbname") + tdSql.error("select floor(c1) from ct5") + + # mix with common col + tdSql.query("select c1, floor(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,8) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0) + tdSql.query("select c1, floor(c1) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + tdSql.query("select c1, floor(c1) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + + # mix with common functions + tdSql.query("select c1, floor(c1),c5, floor(c5) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 , 6) + tdSql.checkData(3 , 2 ,6.66000) + tdSql.checkData(3 , 3 ,6.00000) + + tdSql.query("select c1, floor(c1),c5, floor(c5) from stb1 ") + + # mix with agg functions , not support + tdSql.error("select c1, floor(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, floor(c1),c5, count(c5) from ct1 ") + tdSql.error("select floor(c1), count(c5) from stb1 ") + tdSql.error("select floor(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # bug fix for compute + tdSql.query("select c1, abs(c1) -0 ,floor(c1)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 8.000000000) + + tdSql.query(" select c1, abs(c1) -0 ,floor(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 6.900000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + + def floor_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select floor(c1), floor(c2) ,floor(c3), floor(c4), floor(c5) ,floor(c6) from sub1_bound") + self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select floor(c1), floor(c2) ,floor(c3), floor(c3), floor(c2) ,floor(c1) from sub1_bound") + self.check_result_auto("select floor(floor(floor(floor(floor(floor(floor(floor(floor(floor(c1)))))))))) nest_col_func from sub1_bound;" , "select floor(c1) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select floor(c1+0.2) ,floor(c2) , floor(c3+0.3) , floor(c4-0.3), floor(c5/2), floor(c6/2) from sub1_bound ") + tdSql.checkData(0, 0, 2147483647.000000000) + tdSql.checkData(0, 2, 32767.000000000) + tdSql.checkData(0, 3, 126.000000000) + tdSql.checkData(0, 4, 169999997607218212453866206899682148352.000000000) + + tdSql.checkData(4, 0, -2147483643.000000000) + tdSql.checkData(4, 2, -32763.000000000) + tdSql.checkData(4, 3, -124.000000000) + tdSql.checkData(4, 4, -169499995645668991474575059260979281920.000000000) + + self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select floor(c1+1) ,floor(c2) , floor(c3*1) , floor(c4/2), floor(c5)/2, floor(c6) from sub1_bound ") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: floor basic query ============") + + self.basic_floor_function() + + tdLog.printNoPrefix("==========step5: floor boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: floor filter query ============") + + self.abs_func_filter() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py new file mode 100644 index 0000000000..e6762b2d61 --- /dev/null +++ b/tests/system-test/2-query/log.py @@ -0,0 +1,652 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_log2(self ,origin_query , log_query): + + log_result = tdSql.getResult(log_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >0: + elem = math.log(elem,2) + elif elem <=0: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(log_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) + sys.exit(1) + else: + tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) + + def check_result_auto_log1(self ,origin_query , log_query): + log_result = tdSql.getResult(log_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >0: + elem = None + elif elem <=0: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(log_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) + sys.exit(1) + else: + tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) + def check_result_auto_log__10(self ,origin_query , log_query): + log_result = tdSql.getResult(log_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >0: + elem = None + elif elem <=0: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(log_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) + sys.exit(1) + else: + tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) + + def test_errors(self): + error_sql_lists = [ + "select log from t1", + # "select log(-+--+c1 ,2) from t1", + # "select +-log(c1,2) from t1", + # "select ++-log(c1,2) from t1", + # "select ++--log(c1,2) from t1", + # "select - -log(c1,2)*0 from t1", + # "select log(tbname+1,2) from t1 ", + "select log(123--123,2)==1 from t1", + "select log(c1,2) as 'd1' from t1", + "select log(c1 ,c2 ,2) from t1", + "select log(c1 ,NULL ,2) from t1", + "select log(, 2) from t1;", + "select log(log(c1, 2) ab from t1)", + "select log(c1 ,2 ) as int from t1", + "select log from stb1", + # "select log(-+--+c1) from stb1", + # "select +-log(c1) from stb1", + # "select ++-log(c1) from stb1", + # "select ++--log(c1) from stb1", + # "select - -log(c1)*0 from stb1", + # "select log(tbname+1) from stb1 ", + "select log(123--123 ,2)==1 from stb1", + "select log(c1 ,2) as 'd1' from stb1", + "select log(c1 ,c2 ,2 ) from stb1", + "select log(c1 ,NULL,2) from stb1", + "select log(,) from stb1;", + "select log(log(c1 , 2) ab from stb1)", + "select log(c1 , 2) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select log(ts ,2 ) from t1" , + "select log(c7,2 ) from t1", + "select log(c8,2 ) from t1", + "select log(c9,2 ) from t1", + "select log(ts,2 ) from ct1" , + "select log(c7,2 ) from ct1", + "select log(c8,2 ) from ct1", + "select log(c9,2 ) from ct1", + "select log(ts,2 ) from ct3" , + "select log(c7,2 ) from ct3", + "select log(c8,2 ) from ct3", + "select log(c9,2 ) from ct3", + "select log(ts,2 ) from ct4" , + "select log(c7,2 ) from ct4", + "select log(c8,2 ) from ct4", + "select log(c9,2 ) from ct4", + "select log(ts,2 ) from stb1" , + "select log(c7,2 ) from stb1", + "select log(c8,2 ) from stb1", + "select log(c9,2 ) from stb1" , + + "select log(ts,2 ) from stbbb1" , + "select log(c7,2 ) from stbbb1", + + "select log(ts,2 ) from tbname", + "select log(c9,2 ) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select log(c1,2 ) from t1", + "select log(c2,2 ) from t1", + "select log(c3,2 ) from t1", + "select log(c4,2 ) from t1", + "select log(c5,2 ) from t1", + "select log(c6,2 ) from t1", + + "select log(c1,2 ) from ct1", + "select log(c2,2 ) from ct1", + "select log(c3,2 ) from ct1", + "select log(c4,2 ) from ct1", + "select log(c5,2 ) from ct1", + "select log(c6,2 ) from ct1", + + "select log(c1,2 ) from ct3", + "select log(c2,2 ) from ct3", + "select log(c3,2 ) from ct3", + "select log(c4,2 ) from ct3", + "select log(c5,2 ) from ct3", + "select log(c6,2 ) from ct3", + + "select log(c1,2 ) from stb1", + "select log(c2,2 ) from stb1", + "select log(c3,2 ) from stb1", + "select log(c4,2 ) from stb1", + "select log(c5,2 ) from stb1", + "select log(c6,2 ) from stb1", + + "select log(c6,2) as alisb from stb1", + "select log(c6,2) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_log_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select log(c1 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c2 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c3 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c4 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c5 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c6 ,2) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select log(c1 ,2) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.000000000) + tdSql.checkData(3 , 0, 1.584962501) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,2), log(c2 ,2) ,log(c3, 2), log(c4 ,2), log(c5 ,2) from t1") + self.check_result_auto_log1( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,1), log(c2 ,1) ,log(c3, 1), log(c4 ,1), log(c5 ,1) from t1") + self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,-10), log(c2 ,-10) ,log(c3, -10), log(c4 ,-10), log(c5 ,-10) from t1") + + # used for sub table + tdSql.query("select c1 ,log(c1 ,3) from ct1") + tdSql.checkData(0, 1, 1.892789261) + tdSql.checkData(1 , 1, 1.771243749) + tdSql.checkData(3 , 1, 1.464973521) + tdSql.checkData(4 , 1, None) + + # test bug fix for log(c1,c2) + + tdSql.query("select c1, c2 ,log(c1,c2) from ct1") + tdSql.checkData(0 , 2, 0.182485070) + tdSql.checkData(1 , 2, 0.172791608) + tdSql.checkData(2 , 2, 0.161311499) + tdSql.checkData(3 , 2, 0.147315235) + tdSql.checkData(4 , 2, None) + + + self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) from ct1") + self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1,-10), log(c2,-10) ,log(c3,-10), log(c4,-10), log(c5,-10) from ct1") + + # nest query for log functions + tdSql.query("select c1 , log(c1,3) ,log(log(c1,3),3) , log(log(log(c1,3),3),3) from ct1;") + tdSql.checkData(0 , 0 , 8) + tdSql.checkData(0 , 1 , 1.892789261) + tdSql.checkData(0 , 2 , 0.580779541) + tdSql.checkData(0 , 3 , -0.494609470) + + tdSql.checkData(1 , 0 , 7) + tdSql.checkData(1 , 1 , 1.771243749) + tdSql.checkData(1 , 2 , 0.520367366) + tdSql.checkData(1 , 3 , -0.594586689) + + tdSql.checkData(4 , 0 , 0) + tdSql.checkData(4 , 1 , None) + tdSql.checkData(4 , 2 , None) + tdSql.checkData(4 , 3 , None) + + # # used for stable table + + tdSql.query("select log(c1, 2) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select log(c1, 2) from stbbb1") + tdSql.error("select log(c1, 2) from tbname") + tdSql.error("select log(c1, 2) from ct5") + + # mix with common col + tdSql.query("select c1, log(c1 ,2) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,3.000000000) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,None) + tdSql.query("select c1, log(c1,2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,2.321928095) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + tdSql.query("select c1, log(c1 ,2 ) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,2.321928095) + + # mix with common functions + tdSql.query("select c1, log(c1 ,2),c5, log(c5 ,2) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 , 2.584962501) + tdSql.checkData(3 , 2 ,6.66000) + tdSql.checkData(3 , 3 ,2.735522144) + + tdSql.query("select c1, log(c1,1),c5, floor(c5 ) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, log(c1 ,2),c5, count(c5) from stb1 ") + tdSql.error("select c1, log(c1 ,2),c5, count(c5) from ct1 ") + tdSql.error("select log(c1 ,2), count(c5) from stb1 ") + tdSql.error("select log(c1 ,2), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, log(c1 ,2) -0 ,log(c1-4 ,2)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 3.000000000) + tdSql.checkData(1, 2, 2.000000000) + + tdSql.query(" select c1, log(c1 ,2) -0 ,log(c1-0.1 ,2)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 3.000000000) + tdSql.checkData(1, 2, 2.881852653) + + tdSql.query("select c1, log(c1, -10), c2, log(c2, -10), c3, log(c3, -10) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, log(c1, 100000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, 0.112886248) + tdSql.checkData(1, 1, 0.105637255) + tdSql.checkData(4, 1, None) + + + tdSql.query("select c1, log(c1, 10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, 0.069468461) + tdSql.checkData(1, 1, 0.065007542) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(c1, 10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, log(c1, 10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, 0.036123599) + tdSql.checkData(1, 1, 0.033803922) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, 0.026561470) + tdSql.checkData(1, 1, 0.024855825) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, 0.022577250) + tdSql.checkData(1, 1, 0.021127451) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def log_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, log(c1, 2) from ct1") + tdSql.checkData(0, 1,3.000000000) + tdSql.query("select c1, log(c1, 2.0) from ct1") + tdSql.checkData(0, 1, 3.000000000) + + tdSql.query("select c1, log(1, 2.0) from ct1") + tdSql.checkData(0, 1, 0.000000000) + tdSql.checkRows(13) + + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, log(1, 2.0) from ct1") + tdSql.checkData(0, 1, 0.000000000) + tdSql.checkRows(13) + + # two cols start log(x,y) + tdSql.query("select c1,c2, log(c1,c2) from ct1") + tdSql.checkData(0, 2, 0.182485070) + tdSql.checkData(1, 2, 0.172791608) + tdSql.checkData(4, 2, None) + + tdSql.query("select c1,c2, log(c2,c1) from ct1") + tdSql.checkData(0, 2, 5.479900349) + tdSql.checkData(1, 2, 5.787318105) + tdSql.checkData(4, 2, None) + + tdSql.query("select c1, log(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 0.333333333) + tdSql.checkData(1, 1, 0.356207187) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(2.0 , ceil(abs(c1))) from ct1") + tdSql.checkData(0, 1, 0.333333333) + tdSql.checkData(1, 1, 0.356207187) + tdSql.checkData(4, 1, None) + + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + + def log_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) ,log(c6,2) from sub1_bound") + self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,-10), log(c2,-10) ,log(c3,-10), log(c4,-10), log(c5,-10) ,log(c6,-10) from sub1_bound") + + self.check_result_auto_log2( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c3,2), log(c2,2) ,log(c1,2) from sub1_bound") + + + self.check_result_auto_log2("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select log(abs(c1) ,2) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select log(abs(c1),2) ,log(abs(c2),2) , log(abs(c3),2) , log(abs(c4),2), log(abs(c5),2), log(abs(c6),2) from sub1_bound ") + tdSql.checkData(0,0,math.log(2147483647,2)) + tdSql.checkData(0,1,math.log(9223372036854775807 ,2)) + tdSql.checkData(0,2,math.log(32767,2)) + tdSql.checkData(0,3,math.log(127 ,2)) + tdSql.checkData(0,4,math.log(339999995214436424907732413799364296704.00000,2)) + tdSql.checkData(0,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000 ,2)) + tdSql.checkData(1,0,math.log(2147483647 ,2)) + tdSql.checkData(1,1,math.log(9223372036854775807 ,2)) + tdSql.checkData(1,2,math.log(32767 ,2)) + tdSql.checkData(1,3,math.log(127,2)) + tdSql.checkData(1,4,math.log(339999995214436424907732413799364296704.00000 ,2)) + tdSql.checkData(1,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000 ,2)) + tdSql.checkData(3,0,math.log(2147483646,2)) + tdSql.checkData(3,1,math.log(9223372036854775806,2)) + tdSql.checkData(3,2,math.log(32766,2)) + tdSql.checkData(3,3,math.log(126 ,2)) + tdSql.checkData(3,4,math.log(339999995214436424907732413799364296704.00000,2)) + tdSql.checkData(3,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000,2)) + + # check + - * / in functions + tdSql.query("select log(abs(c1+1) ,2) ,log(abs(c2),2) , log(abs(c3*1),2) , log(abs(c4/2),2), log(abs(c5) ,2)/2, log(abs(c6) ,2) from sub1_bound ") + tdSql.checkData(0,0,math.log(2147483648.000000000,2)) + tdSql.checkData(0,1,math.log(9223372036854775807,2)) + tdSql.checkData(0,2,math.log(32767.000000000,2)) + tdSql.checkData(0,3,math.log(63.500000000,2)) + tdSql.checkData(0,4,63.999401166) + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: log basic query ============") + + self.basic_log_function() + + tdLog.printNoPrefix("==========step5: big number log query ============") + + self.test_big_number() + + tdLog.printNoPrefix("==========step6: base number for log query ============") + + self.log_base_test() + + tdLog.printNoPrefix("==========step7: log boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step8: log filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py new file mode 100644 index 0000000000..223e56bce6 --- /dev/null +++ b/tests/system-test/2-query/round.py @@ -0,0 +1,467 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto(self ,origin_query , round_query): + pass + round_result = tdSql.getResult(round_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = round(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(round_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("round function value has not as expected , sql is \"%s\" "%round_query ) + sys.exit(1) + else: + tdLog.info("round value check pass , it work as expected ,sql is \"%s\" "%round_query ) + + def test_errors(self): + error_sql_lists = [ + "select round from t1", + # "select round(-+--+c1) from t1", + # "select +-round(c1) from t1", + # "select ++-round(c1) from t1", + # "select ++--round(c1) from t1", + # "select - -round(c1)*0 from t1", + # "select round(tbname+1) from t1 ", + "select round(123--123)==1 from t1", + "select round(c1) as 'd1' from t1", + "select round(c1 ,c2 ) from t1", + "select round(c1 ,NULL) from t1", + "select round(,) from t1;", + "select round(round(c1) ab from t1)", + "select round(c1) as int from t1", + "select round from stb1", + # "select round(-+--+c1) from stb1", + # "select +-round(c1) from stb1", + # "select ++-round(c1) from stb1", + # "select ++--round(c1) from stb1", + # "select - -round(c1)*0 from stb1", + # "select round(tbname+1) from stb1 ", + "select round(123--123)==1 from stb1", + "select round(c1) as 'd1' from stb1", + "select round(c1 ,c2 ) from stb1", + "select round(c1 ,NULL) from stb1", + "select round(,) from stb1;", + "select round(round(c1) ab from stb1)", + "select round(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select round(ts) from t1" , + "select round(c7) from t1", + "select round(c8) from t1", + "select round(c9) from t1", + "select round(ts) from ct1" , + "select round(c7) from ct1", + "select round(c8) from ct1", + "select round(c9) from ct1", + "select round(ts) from ct3" , + "select round(c7) from ct3", + "select round(c8) from ct3", + "select round(c9) from ct3", + "select round(ts) from ct4" , + "select round(c7) from ct4", + "select round(c8) from ct4", + "select round(c9) from ct4", + "select round(ts) from stb1" , + "select round(c7) from stb1", + "select round(c8) from stb1", + "select round(c9) from stb1" , + + "select round(ts) from stbbb1" , + "select round(c7) from stbbb1", + + "select round(ts) from tbname", + "select round(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select round(c1) from t1", + "select round(c2) from t1", + "select round(c3) from t1", + "select round(c4) from t1", + "select round(c5) from t1", + "select round(c6) from t1", + + "select round(c1) from ct1", + "select round(c2) from ct1", + "select round(c3) from ct1", + "select round(c4) from ct1", + "select round(c5) from ct1", + "select round(c6) from ct1", + + "select round(c1) from ct3", + "select round(c2) from ct3", + "select round(c3) from ct3", + "select round(c4) from ct3", + "select round(c5) from ct3", + "select round(c6) from ct3", + + "select round(c1) from stb1", + "select round(c2) from stb1", + "select round(c3) from stb1", + "select round(c4) from stb1", + "select round(c5) from stb1", + "select round(c6) from stb1", + + "select round(c6) as alisb from stb1", + "select round(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_round_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select round(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select round(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select round(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select round(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select round(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select round(c6) from ct3") + + # used for regular table + tdSql.query("select round(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1) + tdSql.checkData(3 , 0, 3) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto( "select c1, c2, c3 , c4, c5 from t1", "select (c1), round(c2) ,round(c3), round(c4), round(c5) from t1") + + # used for sub table + tdSql.query("select round(c1) from ct1") + tdSql.checkData(0, 0, 8) + tdSql.checkData(1 , 0, 7) + tdSql.checkData(3 , 0, 5) + tdSql.checkData(5 , 0, 4) + + tdSql.query("select round(c1) from ct1") + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct1", "select (c1), round(c2) ,round(c3), round(c4), round(c5) from ct1") + self.check_result_auto("select round(round(round(round(round(round(round(round(round(round(c1)))))))))) nest_col_func from ct1;","select c1 from ct1" ) + + # used for stable table + + tdSql.query("select round(c1) from stb1") + tdSql.checkRows(25) + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct4 ", "select (c1), round(c2) ,round(c3), round(c4), round(c5) from ct4") + self.check_result_auto("select round(round(round(round(round(round(round(round(round(round(c1)))))))))) nest_col_func from ct4;" , "select c1 from ct4" ) + + + # used for not exists table + tdSql.error("select round(c1) from stbbb1") + tdSql.error("select round(c1) from tbname") + tdSql.error("select round(c1) from ct5") + + # mix with common col + tdSql.query("select c1, round(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,8) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0) + tdSql.query("select c1, round(c1) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + tdSql.query("select c1, round(c1) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + + # mix with common functions + tdSql.query("select c1, round(c1),c5, round(c5) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 , 6) + tdSql.checkData(3 , 2 ,6.66000) + tdSql.checkData(3 , 3 ,7.00000) + + tdSql.checkData(6 , 0 , 4) + tdSql.checkData(6 , 1 , 4) + tdSql.checkData(6 , 2 ,4.44000) + tdSql.checkData(6 , 3 ,4.00000) + + tdSql.query("select c1, round(c1),c5, round(c5) from stb1 ") + + # mix with agg functions , not support + tdSql.error("select c1, round(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, round(c1),c5, count(c5) from ct1 ") + tdSql.error("select round(c1), count(c5) from stb1 ") + tdSql.error("select round(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # bug fix for compute + tdSql.query("select c1, abs(c1) -0 ,round(c1)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 8.000000000) + + tdSql.query(" select c1, abs(c1) -0 ,round(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 7.900000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) , round(abs(c1))-0.5 from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + tdSql.checkData(0,6,7.500000000) + + def round_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select round(c1), round(c2) ,round(c3), round(c4), round(c5) ,round(c6) from sub1_bound") + self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select round(c1), round(c2) ,round(c3), round(c3), round(c2) ,round(c1) from sub1_bound") + self.check_result_auto("select round(round(round(round(round(round(round(round(round(round(c1)))))))))) nest_col_func from sub1_bound;" , "select round(c1) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select round(c1+0.2) ,round(c2) , round(c3+0.3) , round(c4-0.3), round(c5/2), round(c6/2) from sub1_bound ") + tdSql.checkData(0, 0, 2147483647.000000000) + tdSql.checkData(0, 2, 32767.000000000) + tdSql.checkData(0, 3, 127.000000000) + tdSql.checkData(0, 4, 169999997607218212453866206899682148352.000000000) + + tdSql.checkData(4, 0, -2147483643.000000000) + tdSql.checkData(4, 2, -32763.000000000) + tdSql.checkData(4, 3, -123.000000000) + tdSql.checkData(4, 4, -169499995645668991474575059260979281920.000000000) + + self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select round(c1+1) ,round(c2) , round(c3*1) , round(c4/2), round(c5)/2, round(c6) from sub1_bound ") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: round basic query ============") + + self.basic_round_function() + + tdLog.printNoPrefix("==========step5: round boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: round filter query ============") + + self.abs_func_filter() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f007a09823..0b7a71bbe4 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -17,3 +17,9 @@ python3 ./test.py -f 2-query/Today.py #python3 ./test.py -f 2-query/cast.py + +python3 ./test.py -f 2-query/abs.py +python3 ./test.py -f 2-query/ceil.py +python3 ./test.py -f 2-query/floor.py +python3 ./test.py -f 2-query/round.py +python3 ./test.py -f 2-query/log.py \ No newline at end of file diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index db01b84d0e..73cdf7f59c 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -29,49 +29,49 @@ #define NC "\033[0m" #define min(a, b) (((a) < (b)) ? (a) : (b)) -#define MAX_SQL_STR_LEN (1024 * 1024) -#define MAX_ROW_STR_LEN (16 * 1024) -#define MAX_CONSUMER_THREAD_CNT (16) +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_CONSUMER_THREAD_CNT (16) typedef struct { - TdThread thread; - int32_t consumerId; + TdThread thread; + int32_t consumerId; - int32_t ifCheckData; - int64_t expectMsgCnt; - - int64_t consumeMsgCnt; - int64_t consumeRowCnt; - int32_t checkresult; + int32_t ifCheckData; + int64_t expectMsgCnt; - char topicString[1024]; - char keyString[1024]; + int64_t consumeMsgCnt; + int64_t consumeRowCnt; + int32_t checkresult; - int32_t numOfTopic; - char topics[32][64]; + char topicString[1024]; + char keyString[1024]; - int32_t numOfKey; - char key[32][64]; - char value[32][64]; + int32_t numOfTopic; + char topics[32][64]; + + int32_t numOfKey; + char key[32][64]; + char value[32][64]; tmq_t* tmq; tmq_list_t* topicList; - + } SThreadInfo; typedef struct { // input from argvs - char cdbName[32]; - char dbName[32]; - int32_t showMsgFlag; - int32_t showRowFlag; - int32_t consumeDelay; // unit s - int32_t numOfThread; - SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; + char cdbName[32]; + char dbName[32]; + int32_t showMsgFlag; + int32_t showRowFlag; + int32_t consumeDelay; // unit s + int32_t numOfThread; + SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; } SConfInfo; static SConfInfo g_stConfInfo; -TdFilePtr g_fp = NULL; +TdFilePtr g_fp = NULL; // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -93,7 +93,6 @@ static void printHelp() { exit(EXIT_SUCCESS); } - void initLogFile() { // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); char file[256]; @@ -106,36 +105,35 @@ void initLogFile() { g_fp = pFile; } - void saveConfigToLogFile() { time_t tTime = taosGetTimestampSec(); struct tm tm = *taosLocalTime(&tTime, NULL); taosFprintfFile(g_fp, "###################################################################\n"); - taosFprintfFile(g_fp, "# configDir: %s\n", configDir); - taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); - taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); - taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); - taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); - taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); - taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); + taosFprintfFile(g_fp, "# configDir: %s\n", configDir); + taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); + taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); + taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); + taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - taosFprintfFile(g_fp, " Topics: "); - for (int j = 0 ; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { - taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); + taosFprintfFile(g_fp, " Topics: "); + for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { + taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); } - taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, "\n"); taosFprintfFile(g_fp, " Key: "); - for (int k = 0 ; k < g_stConfInfo.stThreads[i].numOfKey; k++) { - taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); + for (int k = 0; k < g_stConfInfo.stThreads[i].numOfKey; k++) { + taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); } taosFprintfFile(g_fp, "\n"); } - + taosFprintfFile(g_fp, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); taosFprintfFile(g_fp, "###################################################################\n"); } @@ -168,7 +166,7 @@ void parseArgument(int32_t argc, char* argv[]) { } initLogFile(); - + taosFprintfFile(g_fp, "====parseArgument() success\n"); #if 1 @@ -203,26 +201,26 @@ void ltrim(char* str) { // return str; } -static int running = 1; +static int running = 1; static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) { - char buf[1024]; + char buf[1024]; int32_t totalRows = 0; - //printf("topic: %s\n", tmq_get_topic_name(msg)); - //printf("vg:%d\n", tmq_get_vgroup_id(msg)); - taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); - taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); - + // printf("topic: %s\n", tmq_get_topic_name(msg)); + // printf("vg:%d\n", tmq_get_vgroup_id(msg)); + taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); + taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); + while (1) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; - if (0 != g_stConfInfo.showRowFlag) { - TAOS_FIELD* fields = taos_fetch_fields(msg); + if (0 != g_stConfInfo.showRowFlag) { + TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); taos_print_row(buf, row, fields, numOfFields); - taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); - } - totalRows++; + taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); + } + totalRows++; } return totalRows; @@ -241,43 +239,43 @@ int queryDB(TAOS* taos, char* command) { return 0; } -static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) { +static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { printf("tmq_commit_cb_print() commit %d\n", resp); } -void build_consumer(SThreadInfo *pInfo) { +void build_consumer(SThreadInfo* pInfo) { tmq_conf_t* conf = tmq_conf_new(); - //tmq_conf_set(conf, "td.connect.ip", "localhost"); - //tmq_conf_set(conf, "td.connect.port", "6030"); + // tmq_conf_set(conf, "td.connect.ip", "localhost"); + // tmq_conf_set(conf, "td.connect.port", "6030"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL); // tmq_conf_set(conf, "group.id", "cgrp1"); for (int32_t i = 0; i < pInfo->numOfKey; i++) { tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); } - //tmq_conf_set(conf, "client.id", "c-001"); + // tmq_conf_set(conf, "client.id", "c-001"); - //tmq_conf_set(conf, "enable.auto.commit", "true"); - //tmq_conf_set(conf, "enable.auto.commit", "false"); + // tmq_conf_set(conf, "enable.auto.commit", "true"); + // tmq_conf_set(conf, "enable.auto.commit", "false"); + + // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + + // tmq_conf_set(conf, "auto.offset.reset", "none"); + // tmq_conf_set(conf, "auto.offset.reset", "earliest"); + // tmq_conf_set(conf, "auto.offset.reset", "latest"); - //tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - - //tmq_conf_set(conf, "auto.offset.reset", "none"); - //tmq_conf_set(conf, "auto.offset.reset", "earliest"); - //tmq_conf_set(conf, "auto.offset.reset", "latest"); - pInfo->tmq = tmq_consumer_new(conf, NULL, 0); return; } -void build_topic_list(SThreadInfo *pInfo) { +void build_topic_list(SThreadInfo* pInfo) { pInfo->topicList = tmq_list_new(); // tmq_list_append(topic_list, "test_stb_topic_1"); for (int32_t i = 0; i < pInfo->numOfTopic; i++) { @@ -286,41 +284,37 @@ void build_topic_list(SThreadInfo *pInfo) { return; } -int32_t saveConsumeResult(SThreadInfo *pInfo) { +int32_t saveConsumeResult(SThreadInfo* pInfo) { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", - g_stConfInfo.cdbName, - pInfo->consumerId, - pInfo->consumeMsgCnt, - pInfo->consumeRowCnt, - pInfo->checkresult); - + sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, + pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); + TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); exit(-1); } - + taos_free_result(pRes); return 0; } -void loop_consume(SThreadInfo *pInfo) { +void loop_consume(SThreadInfo* pInfo) { tmq_resp_err_t err; - + int64_t totalMsgs = 0; int64_t totalRows = 0; while (running) { TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); - if (tmqMsg) { + if (tmqMsg) { if (0 != g_stConfInfo.showMsgFlag) { totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId); } @@ -328,7 +322,7 @@ void loop_consume(SThreadInfo *pInfo) { taos_free_result(tmqMsg); totalMsgs++; - + if (totalMsgs >= pInfo->expectMsgCnt) { break; } @@ -336,7 +330,7 @@ void loop_consume(SThreadInfo *pInfo) { break; } } - + err = tmq_consumer_close(pInfo->tmq); if (err) { printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); @@ -346,27 +340,27 @@ void loop_consume(SThreadInfo *pInfo) { pInfo->consumeMsgCnt = totalMsgs; pInfo->consumeRowCnt = totalRows; - taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %"PRId64", consumeRowCnt: %"PRId64"\n", pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); - + taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n", + pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); } -void *consumeThreadFunc(void *param) { +void* consumeThreadFunc(void* param) { int32_t totalMsgs = 0; - SThreadInfo *pInfo = (SThreadInfo *)param; + SThreadInfo* pInfo = (SThreadInfo*)param; build_consumer(pInfo); build_topic_list(pInfo); - if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)){ + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { return NULL; } - + tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); if (err) { printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - + loop_consume(pInfo); tmq_commit(pInfo->tmq, NULL, 0); @@ -374,10 +368,10 @@ void *consumeThreadFunc(void *param) { err = tmq_unsubscribe(pInfo->tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - pInfo->consumeMsgCnt = -1; + pInfo->consumeMsgCnt = -1; return NULL; - } - + } + // save consume result into consumeresult table saveConsumeResult(pInfo); @@ -389,7 +383,7 @@ void parseConsumeInfo() { const char delim[2] = ","; const char ch = ':'; - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { token = strtok(g_stConfInfo.stThreads[i].topicString, delim); while (token != NULL) { // printf("%s\n", token ); @@ -397,10 +391,10 @@ void parseConsumeInfo() { ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); g_stConfInfo.stThreads[i].numOfTopic++; - + token = strtok(NULL, delim); } - + token = strtok(g_stConfInfo.stThreads[i].keyString, delim); while (token != NULL) { // printf("%s\n", token ); @@ -414,7 +408,7 @@ void parseConsumeInfo() { // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.stThreads[i].numOfKey++; } - + token = strtok(NULL, delim); } } @@ -422,48 +416,49 @@ void parseConsumeInfo() { int32_t getConsumeInfo() { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - + sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); - taosCloseFile(&g_fp); + taosCloseFile(&g_fp); taos_free_result(pRes); exit(-1); - } - - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(pRes); - TAOS_FIELD* fields = taos_fetch_fields(pRes); - - // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int - + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(pRes); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + + // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, + // ifcheckdata int + int32_t numOfThread = 0; while ((row = taos_fetch_row(pRes))) { - int32_t* lengths = taos_fetch_lengths(pRes); - - for (int i = 0; i < num_fields; ++i) { + int32_t* lengths = taos_fetch_lengths(pRes); + + for (int i = 0; i < num_fields; ++i) { if (row[i] == NULL || 0 == i) { continue; } - + if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t *)row[i]); + g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]); } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { - g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t *)row[i]); + g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t *)row[i]); + g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); } } - numOfThread ++; + numOfThread++; } g_stConfInfo.numOfThread = numOfThread; @@ -474,7 +469,6 @@ int32_t getConsumeInfo() { return 0; } - int main(int32_t argc, char* argv[]) { parseArgument(argc, argv); getConsumeInfo(); @@ -485,20 +479,21 @@ int main(int32_t argc, char* argv[]) { taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); // pthread_create one thread to consume - taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { - taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, (void *)(&(g_stConfInfo.stThreads[i]))); + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); } for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); } - //printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); - - taosFprintfFile(g_fp, "==== close tmqlog ====\n"); - taosCloseFile(&g_fp); - + // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + return 0; }