From f97adf54d333c75901b57c55cf820e017e9e5d24 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 13 May 2022 12:12:37 +0800 Subject: [PATCH 01/71] enh(sync): add syncStartStandBy --- include/libs/sync/sync.h | 1 + source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/src/syncMain.c | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ca10465ed4..551e0fc7b8 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -144,6 +144,7 @@ int32_t syncInit(); void syncCleanUp(); int64_t syncOpen(const SSyncInfo* pSyncInfo); void syncStart(int64_t rid); +void syncStartStandBy(int64_t rid); void syncStop(int64_t rid); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); ESyncState syncGetMyRole(int64_t rid); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 8a21eea7b7..9d090adda5 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -247,6 +247,7 @@ typedef struct SSyncNode { // open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); +void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); // ping -------------- diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index da23d8415b..0740a73fd0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -103,6 +103,16 @@ void syncStart(int64_t rid) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); } +void syncStartStandBy(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return; + } + syncNodeStartStandBy(pSyncNode); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + void syncStop(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -524,6 +534,17 @@ void syncNodeStart(SSyncNode* pSyncNode) { assert(ret == 0); } +void syncNodeStartStandBy(SSyncNode* pSyncNode) { + // state change + pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + syncNodeStopHeartbeatTimer(pSyncNode); + + // reset elect timer, long enough + int32_t electMS = TIMER_MAX_MS; + int32_t ret = syncNodeRestartElectTimer(pSyncNode, electMS); + ASSERT(ret == 0); +} + void syncNodeClose(SSyncNode* pSyncNode) { int32_t ret; assert(pSyncNode != NULL); From 65e9e9725241bb363ce4abb8e8e911509b84a07f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 13 May 2022 13:54:11 +0800 Subject: [PATCH 02/71] refactor: control the memory of the rpc queue --- include/common/tglobal.h | 3 ++ include/util/taoserror.h | 1 + include/util/tqueue.h | 2 +- source/common/src/tglobal.c | 9 ++++ source/dnode/mgmt/mgmt_qnode/src/qmWorker.c | 4 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 12 ++--- source/libs/executor/src/dataDispatcher.c | 6 +-- source/util/src/terror.c | 1 + source/util/src/tqueue.c | 57 +++++++++++++-------- 9 files changed, 61 insertions(+), 34 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index fd8cea449f..94b6ca551b 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -51,6 +51,7 @@ extern int32_t tsVnodeShmSize; extern int32_t tsQnodeShmSize; extern int32_t tsSnodeShmSize; extern int32_t tsBnodeShmSize; +extern int32_t tsNumOfShmThreads; // queue & threads extern int32_t tsNumOfRpcThreads; @@ -67,6 +68,8 @@ extern int32_t tsNumOfQnodeQueryThreads; extern int32_t tsNumOfQnodeFetchThreads; extern int32_t tsNumOfSnodeSharedThreads; extern int32_t tsNumOfSnodeUniqueThreads; +extern int64_t tsRpcQueueMemoryAllowed; +extern int64_t tsRpcQueueMemoryUsed; // monitor extern bool tsEnableMonitor; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index aa2b32daab..b73d39090b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -89,6 +89,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0115) #define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0116) #define TSDB_CODE_NEED_RETRY TAOS_DEF_ERROR_CODE(0, 0x0117) +#define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE TAOS_DEF_ERROR_CODE(0, 0x0118) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0140) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0141) diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 70db65d50f..fc02ea2491 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -59,7 +59,7 @@ void taosFreeQitem(void *pItem); void taosWriteQitem(STaosQueue *queue, void *pItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem); bool taosQueueEmpty(STaosQueue *queue); -int32_t taosQueueSize(STaosQueue *queue); +int32_t taosQueueItemSize(STaosQueue *queue); STaosQall *taosAllocateQall(); void taosFreeQall(STaosQall *qall); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d82378c315..53b59fa479 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -44,6 +44,7 @@ int32_t tsVnodeShmSize = TSDB_MAX_WAL_SIZE * 10 + 128; int32_t tsQnodeShmSize = TSDB_MAX_WAL_SIZE * 4 + 128; int32_t tsSnodeShmSize = TSDB_MAX_WAL_SIZE * 4 + 128; int32_t tsBnodeShmSize = TSDB_MAX_WAL_SIZE * 4 + 128; +int32_t tsNumOfShmThreads = 1; // queue & threads int32_t tsNumOfRpcThreads = 1; @@ -60,6 +61,8 @@ int32_t tsNumOfQnodeQueryThreads = 2; int32_t tsNumOfQnodeFetchThreads = 2; int32_t tsNumOfSnodeSharedThreads = 2; int32_t tsNumOfSnodeUniqueThreads = 2; +int64_t tsRpcQueueMemoryAllowed = 0; +int64_t tsRpcQueueMemoryUsed = 0; // monitor bool tsEnableMonitor = true; @@ -374,6 +377,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "qnodeShmSize", tsQnodeShmSize, TSDB_MAX_WAL_SIZE + 128, INT32_MAX, 0) != 0) return -1; if (cfgAddInt32(pCfg, "snodeShmSize", tsSnodeShmSize, TSDB_MAX_WAL_SIZE + 128, INT32_MAX, 0) != 0) return -1; if (cfgAddInt32(pCfg, "bnodeShmSize", tsBnodeShmSize, TSDB_MAX_WAL_SIZE + 128, INT32_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "mumOfShmThreads", tsNumOfShmThreads, 1, 1024, 0) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, 4); @@ -427,6 +431,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSnodeUniqueThreads = TRANGE(tsNumOfSnodeUniqueThreads, 2, 4); if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeUniqueThreads, 1, 1024, 0) != 0) return -1; + tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; + tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_WAL_SIZE * 10L, TSDB_MAX_WAL_SIZE * 10000L); + if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, 1, INT64_MAX, 0) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; @@ -566,6 +574,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; + tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index c95a4e8292..40b91cb93e 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -126,10 +126,10 @@ int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { switch (qtype) { case QUERY_QUEUE: - size = taosQueueSize(pMgmt->queryWorker.queue); + size = taosQueueItemSize(pMgmt->queryWorker.queue); break; case FETCH_QUEUE: - size = taosQueueSize(pMgmt->fetchWorker.queue); + size = taosQueueItemSize(pMgmt->fetchWorker.queue); break; default: break; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index cf8f4d5010..1f671179b6 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -397,22 +397,22 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { if (pVnode != NULL) { switch (qtype) { case WRITE_QUEUE: - size = taosQueueSize(pVnode->pWriteQ); + size = taosQueueItemSize(pVnode->pWriteQ); break; case SYNC_QUEUE: - size = taosQueueSize(pVnode->pSyncQ); + size = taosQueueItemSize(pVnode->pSyncQ); break; case APPLY_QUEUE: - size = taosQueueSize(pVnode->pApplyQ); + size = taosQueueItemSize(pVnode->pApplyQ); break; case QUERY_QUEUE: - size = taosQueueSize(pVnode->pQueryQ); + size = taosQueueItemSize(pVnode->pQueryQ); break; case FETCH_QUEUE: - size = taosQueueSize(pVnode->pFetchQ); + size = taosQueueItemSize(pVnode->pFetchQ); break; case MERGE_QUEUE: - size = taosQueueSize(pVnode->pMergeQ); + size = taosQueueItemSize(pVnode->pMergeQ); break; default: break; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a217701471..1d371d43f6 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -88,9 +88,9 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery; - if (taosQueueSize(pDispatcher->pDataBlocks) > capacity) { + if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) { qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity, - taosQueueSize(pDispatcher->pDataBlocks)); + taosQueueItemSize(pDispatcher->pDataBlocks)); return false; } @@ -106,7 +106,7 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { taosThreadMutexLock(&pDispatcher->mutex); - int32_t blockNums = taosQueueSize(pDispatcher->pDataBlocks); + int32_t blockNums = taosQueueItemSize(pDispatcher->pDataBlocks); int32_t status = (0 == blockNums ? DS_BUF_EMPTY : (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a1bc37e6cd..6c0a9b1324 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -95,6 +95,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") TAOS_DEFINE_ERROR(TSDB_CODE_DUP_KEY, "Cannot add duplicate keys to hash") TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY, "Retry needed") +TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE, "Out of memory in rpc queue") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 3c3a8460b9..0d96dcda0a 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -23,36 +23,36 @@ typedef struct STaosQnode STaosQnode; typedef struct STaosQnode { STaosQnode *next; STaosQueue *queue; + int32_t size; char item[]; } STaosQnode; typedef struct STaosQueue { - int32_t itemSize; - int32_t numOfItems; - int32_t threadId; - STaosQnode *head; - STaosQnode *tail; - STaosQueue *next; // for queue set - STaosQset *qset; // for queue set - void *ahandle; // for queue set - FItem itemFp; - FItems itemsFp; + int32_t memOfItems; + int32_t numOfItems; + int32_t threadId; + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set + FItem itemFp; + FItems itemsFp; TdThreadMutex mutex; } STaosQueue; typedef struct STaosQset { - STaosQueue *head; - STaosQueue *current; + STaosQueue *head; + STaosQueue *current; TdThreadMutex mutex; - int32_t numOfQueues; - int32_t numOfItems; - tsem_t sem; + int32_t numOfQueues; + int32_t numOfItems; + tsem_t sem; } STaosQset; typedef struct STaosQall { STaosQnode *current; STaosQnode *start; - int32_t itemSize; int32_t numOfItems; } STaosQall; @@ -118,15 +118,23 @@ bool taosQueueEmpty(STaosQueue *queue) { return empty; } -int32_t taosQueueSize(STaosQueue *queue) { +int32_t taosQueueItemSize(STaosQueue *queue) { taosThreadMutexLock(&queue->mutex); int32_t numOfItems = queue->numOfItems; taosThreadMutexUnlock(&queue->mutex); return numOfItems; } +int32_t taosQueueMemorySize(STaosQueue *queue) { + taosThreadMutexLock(&queue->mutex); + int32_t memOfItems = queue->memOfItems; + taosThreadMutexUnlock(&queue->mutex); + return memOfItems; +} + void *taosAllocateQitem(int32_t size) { STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); + pNode->size = size; if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -161,8 +169,9 @@ void taosWriteQitem(STaosQueue *queue, void *pItem) { } queue->numOfItems++; + queue->memOfItems += pNode->size; if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); - uTrace("item:%p is put into queue:%p, items:%d", pItem, queue, queue->numOfItems); + uTrace("item:%p is put into queue:%p, items:%d mem:%" PRId64, pItem, queue, queue->numOfItems, queue->memOfItems); taosThreadMutexUnlock(&queue->mutex); @@ -181,9 +190,11 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { queue->head = pNode->next; if (queue->head == NULL) queue->tail = NULL; queue->numOfItems--; + queue->memOfItems -= pNode->size; if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, 1); code = 1; - uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, items:%d mem:%" PRId64, *ppItem, queue, queue->numOfItems, + queue->memOfItems); } taosThreadMutexUnlock(&queue->mutex); @@ -207,12 +218,12 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { qall->current = queue->head; qall->start = queue->head; qall->numOfItems = queue->numOfItems; - qall->itemSize = queue->itemSize; code = qall->numOfItems; queue->head = NULL; queue->tail = NULL; queue->numOfItems = 0; + queue->memOfItems = 0; if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, qall->numOfItems); } @@ -377,9 +388,11 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI queue->head = pNode->next; if (queue->head == NULL) queue->tail = NULL; queue->numOfItems--; + queue->memOfItems -= pNode->size; atomic_sub_fetch_32(&qset->numOfItems, 1); code = 1; - uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, items:%d mem:%" PRId64, *ppItem, queue, queue->numOfItems, + queue->memOfItems); } taosThreadMutexUnlock(&queue->mutex); @@ -411,7 +424,6 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand qall->current = queue->head; qall->start = queue->head; qall->numOfItems = queue->numOfItems; - qall->itemSize = queue->itemSize; code = qall->numOfItems; if (ahandle) *ahandle = queue->ahandle; if (itemsFp) *itemsFp = queue->itemsFp; @@ -419,6 +431,7 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand queue->head = NULL; queue->tail = NULL; queue->numOfItems = 0; + queue->memOfItems = 0; atomic_sub_fetch_32(&qset->numOfItems, qall->numOfItems); for (int32_t j = 1; j < qall->numOfItems; ++j) { tsem_wait(&qset->sem); From 219e9553e7050cc14a6314d8a9fa6220d5f2ba9e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 13 May 2022 14:58:52 +0800 Subject: [PATCH 03/71] refactor: control the memory of the rpc queue --- source/util/src/tqueue.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 0d96dcda0a..d4c5a08011 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -28,7 +28,7 @@ typedef struct STaosQnode { } STaosQnode; typedef struct STaosQueue { - int32_t memOfItems; + int64_t memOfItems; int32_t numOfItems; int32_t threadId; STaosQnode *head; @@ -125,9 +125,9 @@ int32_t taosQueueItemSize(STaosQueue *queue) { return numOfItems; } -int32_t taosQueueMemorySize(STaosQueue *queue) { +int64_t taosQueueMemorySize(STaosQueue *queue) { taosThreadMutexLock(&queue->mutex); - int32_t memOfItems = queue->memOfItems; + int64_t memOfItems = queue->memOfItems; taosThreadMutexUnlock(&queue->mutex); return memOfItems; } From f815f6e44decc8e2b12fa86ed4e38707f9480f8d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 13 May 2022 15:45:16 +0800 Subject: [PATCH 04/71] refactor: control the memory of the rpc queue --- source/dnode/mnode/impl/src/mnode.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index cd24a97a93..3c75e557e8 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -66,11 +66,7 @@ static void mndPullupTrans(SMnode *pMnode) { static void mndCalMqRebalance(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_MQ_TIMER, - .pCont = pReq, - .contLen = contLen, - }; + SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } @@ -78,6 +74,7 @@ static void mndPullupTelem(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } static void *mndThreadFp(void *param) { From 267463863ac54ed8fae4aa66d51007a0ac7b4104 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 May 2022 09:32:15 +0000 Subject: [PATCH 05/71] feat: tag index --- source/common/src/ttypes.c | 3 +- source/dnode/vnode/src/inc/meta.h | 6 ++- source/dnode/vnode/src/meta/metaOpen.c | 46 ++++++++-------- source/dnode/vnode/src/meta/metaTable.c | 71 +++++++++++++++++++++++-- 4 files changed, 98 insertions(+), 28 deletions(-) diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 3fd2ef8e73..b3999a49e7 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -402,7 +402,8 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, - {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, + getStatics_nchr}, }; char tTokenTypeSwitcher[13] = { diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index f1917d5fea..60b3a889ef 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -96,8 +96,10 @@ typedef struct { #pragma pack(push, 1) typedef struct { tb_uid_t suid; - int16_t cid; - char data[]; + int32_t cid; + uint8_t isNull : 1; + uint8_t type : 7; + uint8_t data[]; // val + uid } STagIdxKey; #pragma pack(pop) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 8b3f555f4f..07422e3193 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -227,8 +227,7 @@ static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { STagIdxKey *pTagIdxKey1 = (STagIdxKey *)pKey1; STagIdxKey *pTagIdxKey2 = (STagIdxKey *)pKey2; - int8_t *p1, *p2; - int8_t type; + tb_uid_t uid1, uid2; int c; // compare suid @@ -245,31 +244,34 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return -1; } - // compare value - p1 = pTagIdxKey1->data; - p2 = pTagIdxKey2->data; - ASSERT(p1[0] == p2[0]); - type = p1[0]; + ASSERT(pTagIdxKey1->type == pTagIdxKey2->type); - p1++; - p2++; + // check NULL, NULL is always the smallest + if (pTagIdxKey1->isNull && !pTagIdxKey2->isNull) { + return -1; + } else if (!pTagIdxKey1->isNull && pTagIdxKey2->isNull) { + return 1; + } else if (!pTagIdxKey1->isNull && !pTagIdxKey2->isNull) { + // all not NULL, compr tag vals + c = doCompare(pTagIdxKey1->data, pTagIdxKey2->data, pTagIdxKey1->type, 0); + if (c) return c; - c = doCompare(p1, p2, type, 0); - if (c) return c; - - if (IS_VAR_DATA_TYPE(type)) { - p1 = p1 + varDataTLen(p1); - p2 = p2 + varDataTLen(p2); - } else { - p1 = p1 + tDataTypes[type].bytes; - p2 = p2 + tDataTypes[type].bytes; + if (IS_VAR_DATA_TYPE(pTagIdxKey1->type)) { + uid1 = *(tb_uid_t *)(pTagIdxKey1->data + varDataTLen(pTagIdxKey1->data)); + uid2 = *(tb_uid_t *)(pTagIdxKey2->data + varDataTLen(pTagIdxKey2->data)); + } else { + uid1 = *(tb_uid_t *)(pTagIdxKey1->data + tDataTypes[pTagIdxKey1->type].bytes); + uid2 = *(tb_uid_t *)(pTagIdxKey2->data + tDataTypes[pTagIdxKey2->type].bytes); + } } - // compare suid - if (*(tb_uid_t *)p1 > *(tb_uid_t *)p2) { - return 1; - } else if (*(tb_uid_t *)p1 < *(tb_uid_t *)p2) { + // compare uid + if (uid1 < uid2) { return -1; + } else if (uid1 > uid2) { + return 1; + } else { + return 0; } return 0; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 20248f39fb..d666bd22c1 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -22,7 +22,7 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry me = {0}; @@ -389,8 +389,73 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) { - // TODO +static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, + STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { + int32_t nTagData = 0; + + if (pTagData) { + if (IS_VAR_DATA_TYPE(type)) { + nTagData = varDataTLen(pTagData); + } else { + nTagData = tDataTypes[type].bytes; + } + } + *nTagIdxKey = sizeof(STagIdxKey) + nTagData + sizeof(tb_uid_t); + + *ppTagIdxKey = (STagIdxKey *)taosMemoryMalloc(*nTagIdxKey); + if (*ppTagIdxKey == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + (*ppTagIdxKey)->suid = suid; + (*ppTagIdxKey)->cid = cid; + (*ppTagIdxKey)->isNull = (pTagData == NULL) ? 1 : 0; + (*ppTagIdxKey)->type = type; + if (nTagData) memcpy((*ppTagIdxKey)->data, pTagData, nTagData); + *(tb_uid_t *)((*ppTagIdxKey)->data + nTagData) = uid; + + return 0; +} + +static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { + if (pTagIdxKey) taosMemoryFree(pTagIdxKey); +} + +static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { + void *pData = NULL; + int nData = 0; + STbDbKey tbDbKey = {0}; + SMetaEntry stbEntry = {0}; + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; + const void *pTagData = NULL; // + SDecoder dc = {0}; + + // get super table + tdbDbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); + tbDbKey.uid = pCtbEntry->ctbEntry.suid; + tbDbKey.version = *(int64_t *)pData; + tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + + tDecoderInit(&dc, pData, nData); + metaDecodeEntry(&dc, &stbEntry); + + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + pTagData = tdGetKVRowValOfCol((const SKVRow)pCtbEntry->ctbEntry.pTags, pTagColumn->colId); + + // update tag index + if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, pTagColumn->type, pCtbEntry->uid, + &pTagIdxKey, &nTagIdxKey) < 0) { + return -1; + } + tdbDbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); + metaDestroyTagIdxKey(pTagIdxKey); + + tDecoderClear(&dc); + tdbFree(pData); + return 0; } From d003f1b0d76d914c2b9b723009ccf2f56b13f64b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 13 May 2022 23:30:46 +0800 Subject: [PATCH 06/71] enh(index): add more UT --- .../executor/test/index_executor_tests.cpp | 26 +++++++++++++++++++ source/libs/index/test/jsonUT.cc | 13 ++++++++++ source/libs/transport/src/transCli.c | 12 ++++----- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/test/index_executor_tests.cpp b/source/libs/executor/test/index_executor_tests.cpp index 5f1bff45a3..5b03da034e 100644 --- a/source/libs/executor/test/index_executor_tests.cpp +++ b/source/libs/executor/test/index_executor_tests.cpp @@ -200,11 +200,37 @@ TEST(testCase, index_filter) { doFilterTag(opNode, result); EXPECT_EQ(1, taosArrayGetSize(result)); + taosArrayDestroy(result); + nodesDestroyNode(res); + } + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_INT, pLeft, pRight); + SArray *result = taosArrayInit(4, sizeof(uint64_t)); + doFilterTag(opNode, result); + EXPECT_EQ(0, taosArrayGetSize(result)); + taosArrayDestroy(result); + nodesDestroyNode(res); + } + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + SArray *result = taosArrayInit(4, sizeof(uint64_t)); + doFilterTag(opNode, result); + EXPECT_EQ(0, taosArrayGetSize(result)); + taosArrayDestroy(result); nodesDestroyNode(res); } } +// add other greater/lower/equal/in compare func test + TEST(testCase, index_filter_varify) { { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 5f471dba65..08d58da07f 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -403,6 +403,19 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { EXPECT_EQ(1000, taosArrayGetSize(result)); indexMultiTermQueryDestroy(mq); } + { + std::string colName("other_column"); + std::string colVal("100"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 1000; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } { std::string colName("test1"); std::string colVal("10"); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index cd7e3beb13..26f3f689aa 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -145,9 +145,9 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); } while (0) #define CONN_HOST_THREAD_INDEX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) -#define CONN_PERSIST_TIME(para) (para * 1000 * 10) -#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_PERSIST_TIME(para) (para * 1000 * 10) +#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ @@ -223,11 +223,11 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); #define CONN_RELEASE_BY_SERVER(conn) \ (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) -#define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) -#define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) +#define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) +#define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) -#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) +#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) static void* cliWorkThread(void* arg); From 42db1080c410df8023fe34739d604f93290afd5d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 13 May 2022 23:44:50 +0800 Subject: [PATCH 07/71] enh(index): add more UT --- source/libs/index/src/index.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index e0c24ac3bd..e460f00765 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -27,10 +27,33 @@ #endif #define INDEX_NUM_OF_THREADS 4 -#define INDEX_QUEUE_SIZE 200 +#define INDEX_QUEUE_SIZE 200 void* indexQhandle = NULL; +#define INDEX_DATA_BOOL_NULL 0x02 +#define INDEX_DATA_TINYINT_NULL 0x80 +#define INDEX_DATA_SMALLINT_NULL 0x8000 +#define INDEX_DATA_INT_NULL 0x80000000L +#define INDEX_DATA_BIGINT_NULL 0x8000000000000000L +#define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL + +#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF +#define INDEX_DATA_BINARY_NULL 0xFF +#define INDEX_DATA_JSON_NULL 0xFFFFFFFF +#define INDEX_DATA_JSON_null 0xFFFFFFFE +#define INDEX_DATA_JSON_NOT_NULL 0x01 + +#define INDEX_DATA_UTINYINT_NULL 0xFF +#define INDEX_DATA_USMALLINT_NULL 0xFFFF +#define INDEX_DATA_UINT_NULL 0xFFFFFFFF +#define INDEX_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + +#define INDEX_DATA_NULL_STR "NULL" +#define INDEX_DATA_NULL_STR_L "null" + void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); From e21588c627db0b1037cbbeac95dd127660a3e802 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 13 May 2022 23:49:22 +0800 Subject: [PATCH 08/71] enh(index): add NULL value and remove duplicate --- source/libs/index/src/index.c | 64 +++-------------------------------- 1 file changed, 4 insertions(+), 60 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index e460f00765..d56413f840 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -90,12 +90,6 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { return -1; } -#ifdef USE_LUCENE - index_t* index = index_open(path); - sIdx->index = index; -#endif - -#ifdef USE_INVERTED_INDEX // sIdx->cache = (void*)indexCacheCreate(sIdx); sIdx->tindex = indexTFileCreate(path); if (sIdx->tindex == NULL) { @@ -108,7 +102,6 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { taosThreadMutexInit(&sIdx->mtx, NULL); *index = sIdx; return 0; -#endif END: if (sIdx != NULL) { @@ -120,12 +113,6 @@ END: } void indexClose(SIndex* sIdx) { -#ifdef USE_LUCENE - index_close(sIdex->index); - sIdx->index = NULL; -#endif - -#ifdef USE_INVERTED_INDEX void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; @@ -137,31 +124,12 @@ void indexClose(SIndex* sIdx) { taosHashCleanup(sIdx->colObj); taosThreadMutexDestroy(&sIdx->mtx); indexTFileDestroy(sIdx->tindex); -#endif taosMemoryFree(sIdx->path); taosMemoryFree(sIdx); return; } int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { -#ifdef USE_LUCENE - index_document_t* doc = index_document_create(); - - char buf[16] = {0}; - sprintf(buf, "%d", uid); - - for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm* p = taosArrayGetP(fVals, i); - index_document_add(doc, (const char*)(p->key), p->nKey, (const char*)(p->val), p->nVal, 1); - } - index_document_add(doc, NULL, 0, buf, strlen(buf), 0); - - index_put(index->index, doc); - index_document_destroy(doc); -#endif - -#ifdef USE_INVERTED_INDEX - // TODO(yihao): reduce the lock range taosThreadMutexLock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { @@ -193,12 +161,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { return ret; } } - -#endif return 0; } int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { -#ifdef USE_INVERTED_INDEX EIndexOperatorType opera = multiQuerys->opera; // relation of querys SArray* iRslts = taosArrayInit(4, POINTER_BYTES); @@ -211,35 +176,14 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result } indexMergeFinalResults(iRslts, opera, result); indexInterResultsDestroy(iRslts); - -#endif return 0; } -int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { -#ifdef USE_INVERTED_INDEX +int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { return 1; } +int indexRebuild(SIndex* index, SIndexOpts* opts) { return 0; } -#endif - - return 1; -} -int indexRebuild(SIndex* index, SIndexOpts* opts) { -#ifdef USE_INVERTED_INDEX -#endif - - return 0; -} - -SIndexOpts* indexOptsCreate() { -#ifdef USE_LUCENE -#endif - return NULL; -} -void indexOptsDestroy(SIndexOpts* opts) { -#ifdef USE_LUCENE -#endif - return; -} +SIndexOpts* indexOptsCreate() { return NULL; } +void indexOptsDestroy(SIndexOpts* opts) { return; } /* * @param: oper * From feaf9d051997d01790a9bc993effc4e1f4b3132d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 14 May 2022 09:37:29 +0800 Subject: [PATCH 09/71] fix string to ts issue --- source/libs/scalar/src/scalar.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9843f0ac91..6c8326b09f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -27,6 +27,7 @@ void sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) { valueNode->datum.i = 0; } taosMemoryFree(timeStr); + valueNode->typeData = valueNode->datum.i; valueNode->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; valueNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; From 245f82ffe4bb76e68c447c76c133257b3de961cb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 14 May 2022 09:39:50 +0800 Subject: [PATCH 10/71] refactor: align 8 bytes in tqueue.c --- source/util/src/tqueue.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index d4c5a08011..eca529c632 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -24,6 +24,7 @@ typedef struct STaosQnode { STaosQnode *next; STaosQueue *queue; int32_t size; + int32_t reserved; char item[]; } STaosQnode; From e0a51d43e72b92421511656c51ec9494792d7425 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 14 May 2022 09:42:52 +0800 Subject: [PATCH 11/71] fix: 'union [all]' syntax problems --- include/libs/nodes/querynodes.h | 1 + source/libs/nodes/src/nodesUtilFuncs.c | 13 + source/libs/parser/inc/sql.y | 6 +- source/libs/parser/src/parTranslater.c | 7 +- source/libs/parser/src/sql.c | 1259 +++++++++-------- source/libs/parser/test/parSelectTest.cpp | 8 + source/libs/planner/test/planSubqueryTest.cpp | 9 +- 7 files changed, 675 insertions(+), 628 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 8232dbba0f..60f00daebe 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -345,6 +345,7 @@ bool nodesIsUnaryOp(const SOperatorNode* pOp); bool nodesIsArithmeticOp(const SOperatorNode* pOp); bool nodesIsComparisonOp(const SOperatorNode* pOp); bool nodesIsJsonOp(const SOperatorNode* pOp); +bool nodesIsRegularOp(const SOperatorNode* pOp); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index af7ebf5d10..c2bde4082c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1112,6 +1112,19 @@ bool nodesIsJsonOp(const SOperatorNode* pOp) { return false; } +bool nodesIsRegularOp(const SOperatorNode* pOp) { + switch (pOp->opType) { + case OP_TYPE_LIKE: + case OP_TYPE_NOT_LIKE: + case OP_TYPE_MATCH: + case OP_TYPE_NMATCH: + return true; + default: + break; + } + return false; +} + bool nodesIsTimeorderQuery(const SNode* pQuery) { return false; } bool nodesIsTimelineQuery(const SNode* pQuery) { return false; } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index a365faca19..9c83b600d2 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -868,9 +868,9 @@ query_expression_body(A) ::= query_expression_body(B) UNION query_expression_body(D). { A = createSetOperator(pCxt, SET_OP_TYPE_UNION, B, D); } query_primary(A) ::= query_specification(B). { A = B; } -//query_primary(A) ::= -// NK_LP query_expression_body(B) -// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B; } +query_primary(A) ::= + NK_LP query_expression_body(B) + order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B; } %type order_by_clause_opt { SNodeList* } %destructor order_by_clause_opt { nodesDestroyList($$); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c86c1ac2e9..425696bb28 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -635,6 +635,11 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType; } + if (nodesIsRegularOp(pOp)) { + if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || !IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + } + } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; } else if (nodesIsJsonOp(pOp)) { @@ -3842,7 +3847,7 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS if (pVal->node.resType.type == TSDB_DATA_TYPE_NULL) { // todo } else { - tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), + tdAddColToKVRow(pBuilder, pSchema->colId, nodesGetValueFromNode(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 06db8c909e..4293131338 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -134,17 +134,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 591 -#define YYNRULE 450 +#define YYNSTATE 603 +#define YYNRULE 451 #define YYNTOKEN 238 -#define YY_MAX_SHIFT 590 -#define YY_MIN_SHIFTREDUCE 877 -#define YY_MAX_SHIFTREDUCE 1326 -#define YY_ERROR_ACTION 1327 -#define YY_ACCEPT_ACTION 1328 -#define YY_NO_ACTION 1329 -#define YY_MIN_REDUCE 1330 -#define YY_MAX_REDUCE 1779 +#define YY_MAX_SHIFT 602 +#define YY_MIN_SHIFTREDUCE 890 +#define YY_MAX_SHIFTREDUCE 1340 +#define YY_ERROR_ACTION 1341 +#define YY_ACCEPT_ACTION 1342 +#define YY_NO_ACTION 1343 +#define YY_MIN_REDUCE 1344 +#define YY_MAX_REDUCE 1794 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -211,219 +211,225 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2104) +#define YY_ACTTAB_COUNT (2167) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 372, 72, 373, 1362, 380, 505, 373, 1362, 1613, 26, - /* 10 */ 211, 1449, 33, 31, 109, 1445, 332, 336, 1628, 1758, - /* 20 */ 290, 1452, 1143, 1609, 1617, 1615, 34, 32, 30, 29, - /* 30 */ 28, 1710, 1757, 1460, 90, 508, 1755, 89, 88, 87, - /* 40 */ 86, 85, 84, 83, 82, 81, 1644, 1141, 1505, 281, - /* 50 */ 30, 29, 28, 489, 280, 1707, 1758, 467, 12, 1503, - /* 60 */ 33, 31, 1268, 488, 1149, 504, 1613, 1599, 290, 140, - /* 70 */ 1143, 1451, 263, 1755, 504, 34, 32, 30, 29, 28, - /* 80 */ 1, 1609, 1616, 1615, 1656, 108, 22, 128, 1629, 491, - /* 90 */ 1631, 1632, 487, 508, 508, 1141, 34, 32, 30, 29, - /* 100 */ 28, 56, 587, 1230, 1644, 471, 12, 505, 33, 31, - /* 110 */ 1353, 489, 1149, 1142, 104, 504, 290, 1758, 1143, 100, - /* 120 */ 1599, 263, 1455, 106, 338, 36, 408, 126, 1, 1342, - /* 130 */ 1756, 472, 1771, 127, 1755, 1460, 505, 1417, 204, 1703, - /* 140 */ 466, 1330, 465, 1141, 1180, 1758, 371, 460, 100, 375, - /* 150 */ 587, 1165, 1230, 1231, 12, 413, 1144, 260, 140, 1599, - /* 160 */ 1149, 1142, 1755, 556, 1460, 99, 98, 97, 96, 95, - /* 170 */ 94, 93, 92, 91, 1236, 36, 1, 540, 1147, 1148, - /* 180 */ 457, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 484, 506, - /* 190 */ 1207, 1208, 1209, 1210, 1211, 1212, 539, 538, 587, 537, - /* 200 */ 536, 535, 1231, 326, 1144, 331, 1292, 330, 141, 1142, - /* 210 */ 25, 288, 1225, 1226, 1227, 1228, 1229, 1233, 1234, 1235, - /* 220 */ 583, 582, 492, 1236, 1352, 293, 1147, 1148, 1550, 1193, - /* 230 */ 1194, 1195, 1196, 1197, 1198, 1199, 484, 506, 1207, 1208, - /* 240 */ 1209, 1210, 1211, 1212, 1389, 454, 1290, 1291, 1293, 1294, - /* 250 */ 462, 458, 1144, 34, 32, 30, 29, 28, 141, 25, - /* 260 */ 288, 1225, 1226, 1227, 1228, 1229, 1233, 1234, 1235, 9, - /* 270 */ 8, 467, 1438, 1599, 1147, 1148, 1328, 1193, 1194, 1195, - /* 280 */ 1196, 1197, 1198, 1199, 484, 506, 1207, 1208, 1209, 1210, - /* 290 */ 1211, 1212, 33, 31, 34, 32, 30, 29, 28, 108, - /* 300 */ 290, 1436, 1143, 141, 563, 562, 561, 305, 907, 560, - /* 310 */ 559, 558, 110, 553, 552, 551, 550, 549, 548, 547, - /* 320 */ 546, 117, 1282, 1351, 1505, 302, 924, 1141, 923, 387, - /* 330 */ 295, 1628, 1541, 1543, 306, 1503, 1538, 106, 141, 141, - /* 340 */ 33, 31, 1213, 149, 1149, 61, 911, 912, 290, 1244, - /* 350 */ 1143, 469, 136, 1703, 1704, 925, 1708, 24, 542, 1644, - /* 360 */ 7, 34, 32, 30, 29, 28, 489, 34, 32, 30, - /* 370 */ 29, 28, 1599, 1505, 1758, 1141, 488, 1392, 294, 301, - /* 380 */ 1599, 50, 587, 451, 1503, 1167, 124, 140, 33, 31, - /* 390 */ 323, 1755, 1149, 1142, 1462, 1350, 290, 1656, 1143, 461, - /* 400 */ 257, 1629, 491, 1631, 1632, 487, 379, 508, 7, 375, - /* 410 */ 325, 321, 1013, 531, 530, 529, 1017, 528, 1019, 1020, - /* 420 */ 527, 1022, 524, 1141, 1028, 521, 1030, 1031, 518, 515, - /* 430 */ 587, 34, 32, 30, 29, 28, 1144, 422, 421, 300, - /* 440 */ 1149, 1142, 420, 1168, 1599, 105, 417, 124, 505, 416, - /* 450 */ 415, 414, 387, 58, 278, 1462, 7, 181, 1147, 1148, - /* 460 */ 337, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 484, 506, - /* 470 */ 1207, 1208, 1209, 1210, 1211, 1212, 1460, 1104, 587, 419, - /* 480 */ 418, 1232, 422, 421, 1144, 1106, 1349, 420, 141, 1142, - /* 490 */ 105, 417, 362, 436, 416, 415, 414, 1323, 377, 1588, - /* 500 */ 1180, 1143, 1237, 542, 1165, 444, 1147, 1148, 298, 1193, - /* 510 */ 1194, 1195, 1196, 1197, 1198, 1199, 484, 506, 1207, 1208, - /* 520 */ 1209, 1210, 1211, 1212, 147, 1613, 1141, 34, 32, 30, - /* 530 */ 29, 28, 1144, 1758, 434, 1599, 151, 150, 23, 1348, - /* 540 */ 1609, 1616, 1615, 1149, 313, 1105, 140, 432, 54, 141, - /* 550 */ 1755, 53, 508, 445, 1147, 1148, 1628, 1193, 1194, 1195, - /* 560 */ 1196, 1197, 1198, 1199, 484, 506, 1207, 1208, 1209, 1210, - /* 570 */ 1211, 1212, 33, 31, 259, 71, 1165, 975, 1322, 303, - /* 580 */ 290, 587, 1143, 355, 1644, 67, 367, 124, 1599, 134, - /* 590 */ 1169, 489, 1142, 1758, 977, 1462, 167, 1347, 1346, 1345, - /* 600 */ 1499, 488, 473, 1344, 368, 1599, 140, 1141, 133, 505, - /* 610 */ 1755, 471, 1341, 56, 404, 400, 396, 392, 166, 272, - /* 620 */ 1710, 347, 1656, 1267, 1149, 248, 1629, 491, 1631, 1632, - /* 630 */ 487, 505, 508, 124, 1456, 1144, 65, 1460, 467, 1710, - /* 640 */ 1, 1463, 57, 348, 1706, 164, 1599, 1599, 1599, 557, - /* 650 */ 555, 1758, 1599, 1340, 1542, 1543, 1453, 1147, 1148, 1460, - /* 660 */ 123, 1599, 587, 1705, 140, 1339, 108, 273, 1755, 271, - /* 670 */ 270, 923, 410, 1142, 366, 1338, 412, 361, 360, 359, - /* 680 */ 358, 357, 354, 353, 352, 351, 350, 346, 345, 344, - /* 690 */ 343, 342, 341, 340, 339, 492, 406, 505, 505, 411, - /* 700 */ 505, 1551, 1599, 163, 106, 158, 1170, 160, 467, 386, - /* 710 */ 1457, 1505, 1579, 1337, 1599, 206, 1144, 1336, 1335, 137, - /* 720 */ 1703, 1704, 1504, 1708, 1599, 1460, 1460, 156, 1460, 1334, - /* 730 */ 1628, 1333, 235, 911, 912, 1490, 108, 1437, 1147, 1148, - /* 740 */ 412, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 484, 506, - /* 750 */ 1207, 1208, 1209, 1210, 1211, 1212, 505, 6, 1644, 505, - /* 760 */ 505, 1166, 1599, 411, 505, 489, 1599, 1599, 502, 1128, - /* 770 */ 1129, 503, 225, 477, 106, 488, 304, 474, 1599, 1599, - /* 780 */ 1599, 545, 544, 1432, 1460, 471, 1379, 1460, 1460, 138, - /* 790 */ 1703, 1704, 1460, 1708, 1715, 1263, 1656, 43, 261, 75, - /* 800 */ 1629, 491, 1631, 1632, 487, 1628, 508, 114, 423, 1696, - /* 810 */ 1275, 1218, 172, 262, 1692, 170, 1167, 1167, 125, 174, - /* 820 */ 9, 8, 173, 241, 1374, 1758, 481, 176, 1266, 540, - /* 830 */ 175, 192, 42, 1644, 1331, 239, 49, 1152, 140, 48, - /* 840 */ 470, 178, 1755, 195, 177, 1372, 425, 446, 539, 538, - /* 850 */ 488, 537, 536, 535, 1599, 90, 152, 1619, 89, 88, - /* 860 */ 87, 86, 85, 84, 83, 82, 81, 428, 534, 1628, - /* 870 */ 1151, 1656, 1289, 35, 76, 1629, 491, 1631, 1632, 487, - /* 880 */ 35, 508, 948, 197, 1696, 1325, 1326, 1447, 283, 1692, - /* 890 */ 135, 1443, 184, 1621, 35, 483, 590, 1644, 443, 949, - /* 900 */ 533, 1343, 207, 1155, 489, 1418, 208, 214, 450, 1723, - /* 910 */ 229, 437, 42, 1238, 488, 112, 1628, 74, 1599, 455, - /* 920 */ 1200, 201, 101, 113, 1363, 405, 114, 1263, 579, 575, - /* 930 */ 571, 567, 228, 1500, 1099, 1656, 1154, 468, 128, 1629, - /* 940 */ 491, 1631, 1632, 487, 1644, 508, 1222, 216, 52, 51, - /* 950 */ 335, 470, 234, 146, 513, 497, 73, 113, 329, 223, - /* 960 */ 478, 488, 475, 222, 114, 1599, 1006, 1726, 1645, 2, - /* 970 */ 258, 115, 113, 319, 210, 315, 311, 143, 1165, 308, - /* 980 */ 312, 1628, 1656, 1772, 268, 76, 1629, 491, 1631, 1632, - /* 990 */ 487, 975, 508, 501, 1034, 1696, 1112, 1038, 230, 283, - /* 1000 */ 1692, 135, 269, 349, 1045, 148, 1540, 356, 141, 1644, - /* 1010 */ 369, 1043, 116, 1171, 364, 363, 489, 370, 449, 378, - /* 1020 */ 1724, 188, 1174, 381, 365, 155, 488, 1173, 157, 384, - /* 1030 */ 1599, 382, 383, 159, 1172, 385, 162, 55, 165, 1120, - /* 1040 */ 388, 183, 1628, 1435, 427, 1149, 407, 1656, 409, 277, - /* 1050 */ 76, 1629, 491, 1631, 1632, 487, 1450, 508, 169, 435, - /* 1060 */ 1696, 1583, 1446, 171, 283, 1692, 1770, 118, 119, 80, - /* 1070 */ 1644, 231, 232, 180, 1448, 1730, 1444, 489, 120, 121, - /* 1080 */ 182, 438, 439, 1170, 185, 430, 456, 488, 187, 190, - /* 1090 */ 424, 1599, 447, 495, 442, 179, 1628, 1727, 448, 5, - /* 1100 */ 1737, 464, 1736, 193, 453, 200, 196, 282, 1656, 452, - /* 1110 */ 459, 76, 1629, 491, 1631, 1632, 487, 4, 508, 46, - /* 1120 */ 202, 1696, 45, 107, 1644, 283, 1692, 1770, 1263, 1169, - /* 1130 */ 1711, 489, 37, 284, 16, 540, 1753, 479, 1717, 476, - /* 1140 */ 1549, 488, 493, 498, 131, 1599, 494, 500, 1548, 292, - /* 1150 */ 1628, 1677, 203, 499, 539, 538, 218, 537, 536, 535, - /* 1160 */ 233, 66, 1656, 220, 1461, 76, 1629, 491, 1631, 1632, - /* 1170 */ 487, 64, 508, 511, 1433, 1696, 236, 227, 1644, 283, - /* 1180 */ 1692, 1770, 1628, 1773, 44, 489, 586, 279, 1754, 130, - /* 1190 */ 1714, 209, 242, 240, 238, 488, 249, 243, 1593, 1599, - /* 1200 */ 1628, 1592, 307, 1589, 309, 310, 1137, 1138, 144, 314, - /* 1210 */ 1644, 1587, 316, 317, 1586, 318, 1656, 489, 320, 77, - /* 1220 */ 1629, 491, 1631, 1632, 487, 1585, 508, 488, 1644, 1696, - /* 1230 */ 322, 1599, 1628, 1695, 1692, 489, 1584, 324, 1569, 327, - /* 1240 */ 1115, 328, 145, 1114, 1563, 488, 1562, 333, 1656, 1599, - /* 1250 */ 1628, 244, 1629, 491, 1631, 1632, 487, 334, 508, 1561, - /* 1260 */ 1644, 1560, 1082, 1533, 1532, 1531, 1656, 486, 1530, 77, - /* 1270 */ 1629, 491, 1631, 1632, 487, 1529, 508, 488, 1644, 1696, - /* 1280 */ 1528, 1599, 1527, 480, 1692, 489, 1526, 1525, 1524, 1523, - /* 1290 */ 1522, 1521, 1520, 1519, 1518, 488, 1517, 1628, 1656, 1599, - /* 1300 */ 1516, 256, 1629, 491, 1631, 1632, 487, 485, 508, 482, - /* 1310 */ 1668, 111, 1628, 1515, 1514, 1513, 1656, 1512, 1511, 77, - /* 1320 */ 1629, 491, 1631, 1632, 487, 1644, 508, 1510, 1084, 1696, - /* 1330 */ 1509, 1508, 489, 1507, 1693, 1506, 1391, 1359, 153, 102, - /* 1340 */ 1644, 374, 488, 914, 132, 376, 1599, 489, 913, 1358, - /* 1350 */ 154, 1577, 1571, 103, 1555, 1546, 161, 488, 1439, 1390, - /* 1360 */ 1388, 1599, 1386, 1656, 287, 391, 252, 1629, 491, 1631, - /* 1370 */ 1632, 487, 389, 508, 1384, 395, 399, 390, 1656, 942, - /* 1380 */ 1382, 257, 1629, 491, 1631, 1632, 487, 1628, 508, 393, - /* 1390 */ 394, 397, 398, 401, 403, 402, 1371, 1370, 1357, 1441, - /* 1400 */ 79, 1048, 1440, 1049, 463, 1380, 274, 1375, 168, 554, - /* 1410 */ 974, 973, 972, 971, 556, 1644, 968, 1628, 426, 967, - /* 1420 */ 1373, 1356, 486, 966, 429, 275, 276, 1355, 431, 78, - /* 1430 */ 433, 1576, 488, 1122, 1570, 122, 1599, 440, 1554, 1553, - /* 1440 */ 47, 1545, 3, 13, 441, 1644, 186, 59, 189, 14, - /* 1450 */ 191, 35, 489, 1656, 40, 129, 256, 1629, 491, 1631, - /* 1460 */ 1632, 487, 488, 508, 1628, 1669, 1599, 199, 194, 289, - /* 1470 */ 1288, 198, 20, 1628, 1281, 60, 1619, 21, 38, 205, - /* 1480 */ 8, 1260, 11, 1656, 39, 15, 257, 1629, 491, 1631, - /* 1490 */ 1632, 487, 1644, 508, 139, 1259, 1311, 1316, 1310, 489, - /* 1500 */ 285, 1644, 1315, 1314, 286, 17, 1202, 142, 489, 488, - /* 1510 */ 27, 212, 1188, 1599, 1544, 1159, 291, 10, 488, 219, - /* 1520 */ 1628, 1223, 1599, 1201, 496, 18, 19, 215, 490, 213, - /* 1530 */ 1656, 1618, 217, 257, 1629, 491, 1631, 1632, 487, 1656, - /* 1540 */ 508, 1286, 251, 1629, 491, 1631, 1632, 487, 1644, 508, - /* 1550 */ 1628, 62, 221, 63, 224, 489, 67, 1659, 512, 510, - /* 1560 */ 1204, 507, 299, 41, 514, 488, 1035, 1032, 516, 1599, - /* 1570 */ 1029, 517, 519, 1023, 520, 522, 523, 525, 1644, 1021, - /* 1580 */ 526, 1012, 1628, 68, 1027, 489, 1656, 532, 69, 253, - /* 1590 */ 1629, 491, 1631, 1632, 487, 488, 508, 1044, 70, 1599, - /* 1600 */ 1041, 1026, 1040, 1025, 940, 1024, 541, 981, 226, 543, - /* 1610 */ 1644, 962, 1628, 961, 957, 1042, 1656, 489, 960, 245, - /* 1620 */ 1629, 491, 1631, 1632, 487, 959, 508, 488, 958, 956, - /* 1630 */ 955, 1599, 978, 976, 952, 951, 950, 947, 946, 1387, - /* 1640 */ 1644, 566, 945, 564, 565, 1385, 569, 489, 1656, 568, - /* 1650 */ 570, 254, 1629, 491, 1631, 1632, 487, 488, 508, 1628, - /* 1660 */ 1383, 1599, 572, 573, 574, 1381, 576, 577, 578, 1369, - /* 1670 */ 580, 581, 1368, 1354, 1628, 584, 585, 588, 1656, 1145, - /* 1680 */ 237, 246, 1629, 491, 1631, 1632, 487, 1644, 508, 1329, - /* 1690 */ 589, 1329, 1329, 1329, 489, 1329, 1329, 1329, 1329, 1329, - /* 1700 */ 1329, 1329, 1644, 1329, 488, 1329, 1628, 1329, 1599, 489, - /* 1710 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 488, - /* 1720 */ 1329, 1329, 1329, 1599, 1329, 1656, 1329, 1329, 255, 1629, - /* 1730 */ 491, 1631, 1632, 487, 1644, 508, 1329, 1329, 1329, 1329, - /* 1740 */ 1656, 489, 1329, 247, 1629, 491, 1631, 1632, 487, 1329, - /* 1750 */ 508, 488, 1329, 1329, 1329, 1599, 1329, 1329, 1628, 1329, - /* 1760 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 1770 */ 1329, 1329, 1656, 1329, 1329, 1640, 1629, 491, 1631, 1632, - /* 1780 */ 487, 1329, 508, 1329, 1329, 1329, 1644, 1329, 1628, 1329, - /* 1790 */ 1329, 1329, 1329, 489, 1329, 1329, 1329, 1329, 1329, 1329, - /* 1800 */ 1329, 1329, 1329, 488, 1329, 1329, 1329, 1599, 1329, 1329, - /* 1810 */ 1329, 1329, 1329, 1329, 1329, 1329, 1644, 1329, 1329, 1329, - /* 1820 */ 1628, 1329, 1329, 489, 1656, 1329, 1329, 1639, 1629, 491, - /* 1830 */ 1631, 1632, 487, 488, 508, 1628, 1329, 1599, 1329, 1329, - /* 1840 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1644, 1329, - /* 1850 */ 1329, 1329, 1329, 1329, 1656, 489, 1329, 1638, 1629, 491, - /* 1860 */ 1631, 1632, 487, 1644, 508, 488, 1329, 1329, 1329, 1599, - /* 1870 */ 489, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 1880 */ 488, 1329, 1628, 1329, 1599, 1329, 1656, 1329, 1329, 266, - /* 1890 */ 1629, 491, 1631, 1632, 487, 1628, 508, 1329, 1329, 1329, - /* 1900 */ 1329, 1656, 1329, 1329, 265, 1629, 491, 1631, 1632, 487, - /* 1910 */ 1644, 508, 1329, 1329, 1329, 1329, 1329, 489, 1329, 1329, - /* 1920 */ 1329, 1329, 1329, 1644, 297, 296, 1329, 488, 1329, 1329, - /* 1930 */ 489, 1599, 1329, 1329, 1157, 1329, 1329, 1329, 1329, 1329, - /* 1940 */ 488, 1329, 1329, 1329, 1599, 1329, 1628, 1329, 1656, 1329, - /* 1950 */ 1329, 267, 1629, 491, 1631, 1632, 487, 1329, 508, 1150, - /* 1960 */ 1329, 1656, 1329, 1329, 264, 1629, 491, 1631, 1632, 487, - /* 1970 */ 1329, 508, 1329, 1329, 1644, 1329, 1149, 1329, 1329, 1329, - /* 1980 */ 1329, 489, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 1990 */ 1329, 488, 1329, 1329, 1329, 1599, 1329, 1329, 1329, 1329, - /* 2000 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 2010 */ 1329, 1329, 1656, 1329, 509, 250, 1629, 491, 1631, 1632, - /* 2020 */ 487, 1329, 508, 1329, 1329, 1153, 1329, 1329, 1329, 1329, - /* 2030 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 2040 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 2050 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 2060 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1158, 1329, - /* 2070 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 2080 */ 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 2090 */ 1161, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, 1329, - /* 2100 */ 1329, 506, 1207, 1208, + /* 0 */ 382, 76, 383, 1376, 390, 517, 383, 1376, 1627, 28, + /* 10 */ 221, 1463, 35, 33, 113, 1459, 342, 346, 1642, 1772, + /* 20 */ 300, 1466, 1156, 1623, 1631, 1629, 36, 34, 32, 31, + /* 30 */ 30, 1724, 1771, 1474, 94, 520, 1769, 93, 92, 91, + /* 40 */ 90, 89, 88, 87, 86, 85, 1658, 1154, 1519, 291, + /* 50 */ 32, 31, 30, 501, 290, 1721, 1772, 477, 14, 1517, + /* 60 */ 35, 33, 1281, 500, 1162, 516, 1627, 1613, 300, 144, + /* 70 */ 1156, 348, 273, 1769, 397, 36, 34, 32, 31, 30, + /* 80 */ 1, 1623, 1630, 1629, 1670, 112, 24, 132, 1643, 503, + /* 90 */ 1645, 1646, 499, 520, 520, 1154, 36, 34, 32, 31, + /* 100 */ 30, 61, 599, 1243, 270, 481, 14, 517, 35, 33, + /* 110 */ 595, 594, 1162, 1155, 108, 516, 300, 304, 1156, 104, + /* 120 */ 1178, 273, 1469, 110, 936, 128, 418, 130, 2, 1356, + /* 130 */ 54, 482, 1786, 1476, 1772, 1474, 517, 69, 210, 1717, + /* 140 */ 476, 1344, 475, 1154, 1193, 1772, 1772, 145, 104, 416, + /* 150 */ 599, 1769, 1243, 1244, 14, 423, 1157, 1467, 146, 1770, + /* 160 */ 1162, 1155, 1769, 1769, 1474, 103, 102, 101, 100, 99, + /* 170 */ 98, 97, 96, 95, 1249, 38, 2, 552, 1160, 1161, + /* 180 */ 54, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, + /* 190 */ 1220, 1221, 1222, 1223, 1224, 1225, 551, 550, 599, 549, + /* 200 */ 548, 547, 1244, 493, 1157, 341, 1305, 340, 147, 1155, + /* 210 */ 27, 298, 1238, 1239, 1240, 1241, 1242, 1246, 1247, 1248, + /* 220 */ 131, 1180, 504, 1249, 1431, 303, 1160, 1161, 1564, 1206, + /* 230 */ 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, 1220, 1221, + /* 240 */ 1222, 1223, 1224, 1225, 1403, 464, 1303, 1304, 1306, 1307, + /* 250 */ 55, 54, 1157, 36, 34, 32, 31, 30, 147, 27, + /* 260 */ 298, 1238, 1239, 1240, 1241, 1242, 1246, 1247, 1248, 63, + /* 270 */ 288, 477, 554, 187, 1160, 1161, 1342, 1206, 1207, 1208, + /* 280 */ 1209, 1210, 1211, 1212, 496, 518, 1220, 1221, 1222, 1223, + /* 290 */ 1224, 1225, 35, 33, 36, 34, 32, 31, 30, 112, + /* 300 */ 300, 1452, 1156, 147, 575, 574, 573, 315, 147, 572, + /* 310 */ 571, 570, 114, 565, 564, 563, 562, 561, 560, 559, + /* 320 */ 558, 121, 1295, 1235, 1519, 312, 54, 1154, 988, 1658, + /* 330 */ 305, 1642, 1555, 1557, 316, 1517, 501, 110, 138, 310, + /* 340 */ 35, 33, 1226, 471, 1162, 990, 920, 128, 300, 1513, + /* 350 */ 1156, 479, 140, 1717, 1718, 1476, 1722, 26, 397, 1658, + /* 360 */ 8, 36, 34, 32, 31, 30, 501, 36, 34, 32, + /* 370 */ 31, 30, 470, 467, 1772, 1154, 500, 1406, 313, 147, + /* 380 */ 1613, 53, 599, 461, 924, 925, 128, 144, 35, 33, + /* 390 */ 333, 1769, 1162, 1155, 1476, 516, 300, 1670, 1156, 1182, + /* 400 */ 267, 1643, 503, 1645, 1646, 499, 381, 520, 9, 385, + /* 410 */ 335, 331, 1026, 543, 542, 541, 1030, 540, 1032, 1033, + /* 420 */ 539, 1035, 536, 1154, 1041, 533, 1043, 1044, 530, 527, + /* 430 */ 599, 36, 34, 32, 31, 30, 1157, 432, 431, 128, + /* 440 */ 1162, 1155, 430, 472, 468, 109, 427, 1477, 517, 426, + /* 450 */ 425, 424, 1183, 153, 147, 39, 9, 1724, 1160, 1161, + /* 460 */ 347, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, + /* 470 */ 1220, 1221, 1222, 1223, 1224, 1225, 1474, 59, 599, 387, + /* 480 */ 58, 1720, 432, 431, 1157, 1178, 1465, 430, 147, 1155, + /* 490 */ 109, 427, 1519, 446, 426, 425, 424, 1337, 311, 546, + /* 500 */ 1193, 1156, 1367, 1517, 372, 454, 1160, 1161, 308, 1206, + /* 510 */ 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, 1220, 1221, + /* 520 */ 1222, 1223, 1224, 1225, 212, 1627, 1154, 36, 34, 32, + /* 530 */ 31, 30, 1157, 1772, 389, 1613, 1519, 385, 504, 1366, + /* 540 */ 1623, 1630, 1629, 1162, 1565, 1365, 144, 1518, 157, 156, + /* 550 */ 1769, 1613, 520, 455, 1160, 1161, 1642, 1206, 1207, 1208, + /* 560 */ 1209, 1210, 1211, 1212, 496, 518, 1220, 1221, 1222, 1223, + /* 570 */ 1224, 1225, 35, 33, 269, 517, 1178, 198, 1336, 61, + /* 580 */ 300, 599, 1156, 365, 1658, 517, 377, 357, 1613, 429, + /* 590 */ 428, 501, 1155, 1772, 1613, 1364, 173, 514, 1363, 1362, + /* 600 */ 1470, 500, 483, 1474, 378, 1613, 144, 1154, 137, 517, + /* 610 */ 1769, 481, 1361, 1474, 414, 410, 406, 402, 172, 282, + /* 620 */ 1181, 358, 1670, 1280, 1162, 258, 1643, 503, 1645, 1646, + /* 630 */ 499, 517, 520, 1117, 937, 1157, 936, 1474, 477, 1450, + /* 640 */ 2, 1119, 62, 396, 1613, 170, 1360, 1613, 1613, 569, + /* 650 */ 567, 1772, 1179, 11, 10, 1556, 1557, 1160, 1161, 1474, + /* 660 */ 127, 1613, 599, 938, 146, 1359, 112, 283, 1769, 281, + /* 670 */ 280, 7, 420, 1155, 376, 1358, 422, 371, 370, 369, + /* 680 */ 368, 367, 364, 363, 362, 361, 360, 356, 355, 354, + /* 690 */ 353, 352, 351, 350, 349, 1613, 554, 517, 517, 421, + /* 700 */ 517, 1118, 485, 169, 110, 164, 1451, 166, 477, 1471, + /* 710 */ 1593, 1355, 515, 1245, 1613, 1354, 1157, 1353, 1352, 141, + /* 720 */ 1717, 1718, 444, 1722, 1613, 1474, 1474, 162, 1474, 1351, + /* 730 */ 1642, 1350, 1349, 1257, 1250, 442, 112, 1348, 1160, 1161, + /* 740 */ 422, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, + /* 750 */ 1220, 1221, 1222, 1223, 1224, 1225, 517, 568, 1658, 517, + /* 760 */ 1613, 1347, 1552, 421, 1613, 501, 1613, 1613, 235, 155, + /* 770 */ 25, 314, 924, 925, 110, 500, 1729, 1276, 1613, 1613, + /* 780 */ 1613, 1613, 1288, 1724, 1474, 481, 1613, 1474, 1180, 142, + /* 790 */ 1717, 1718, 245, 1722, 1602, 1504, 1670, 336, 552, 79, + /* 800 */ 1643, 503, 1645, 1646, 499, 1642, 520, 1719, 178, 1710, + /* 810 */ 1613, 176, 180, 272, 1706, 179, 182, 551, 550, 181, + /* 820 */ 549, 548, 547, 1393, 557, 1772, 1446, 184, 1279, 1231, + /* 830 */ 183, 1388, 118, 1658, 1386, 1180, 1141, 1142, 146, 323, + /* 840 */ 501, 1276, 1769, 45, 129, 433, 201, 46, 271, 251, + /* 850 */ 500, 11, 10, 435, 1613, 1633, 438, 1339, 1340, 37, + /* 860 */ 481, 249, 52, 488, 556, 51, 1642, 1461, 37, 1457, + /* 870 */ 37, 1670, 456, 190, 79, 1643, 503, 1645, 1646, 499, + /* 880 */ 437, 520, 158, 1302, 1710, 224, 203, 486, 272, 1706, + /* 890 */ 116, 1635, 75, 117, 1658, 445, 495, 545, 1357, 1251, + /* 900 */ 1772, 480, 71, 453, 118, 1165, 54, 1164, 1213, 186, + /* 910 */ 1112, 500, 45, 144, 216, 1613, 1449, 1769, 1432, 1642, + /* 920 */ 525, 440, 117, 465, 447, 226, 434, 118, 207, 1377, + /* 930 */ 509, 185, 1670, 232, 415, 80, 1643, 503, 1645, 1646, + /* 940 */ 499, 119, 520, 78, 1019, 1710, 117, 1658, 1659, 293, + /* 950 */ 1706, 139, 244, 1514, 501, 50, 1740, 478, 49, 215, + /* 960 */ 1047, 961, 1051, 213, 500, 218, 220, 1058, 1613, 460, + /* 970 */ 1737, 1168, 1642, 1167, 57, 56, 345, 3, 962, 152, + /* 980 */ 1178, 1056, 318, 322, 339, 1670, 120, 278, 262, 1643, + /* 990 */ 503, 1645, 1646, 499, 988, 520, 268, 279, 359, 329, + /* 1000 */ 1658, 325, 321, 149, 240, 1125, 1554, 480, 552, 154, + /* 1010 */ 366, 374, 379, 1184, 380, 373, 388, 500, 375, 1187, + /* 1020 */ 391, 1613, 161, 392, 1186, 163, 473, 551, 550, 393, + /* 1030 */ 549, 548, 547, 165, 147, 394, 1185, 168, 1670, 1345, + /* 1040 */ 395, 80, 1643, 503, 1645, 1646, 499, 1642, 520, 60, + /* 1050 */ 489, 1710, 398, 417, 171, 293, 1706, 139, 419, 1464, + /* 1060 */ 94, 175, 1460, 93, 92, 91, 90, 89, 88, 87, + /* 1070 */ 86, 85, 177, 84, 1162, 1658, 1738, 122, 123, 287, + /* 1080 */ 1642, 188, 501, 1462, 241, 1458, 124, 1597, 125, 448, + /* 1090 */ 1183, 191, 500, 452, 193, 449, 1613, 457, 196, 466, + /* 1100 */ 1751, 1750, 507, 6, 1731, 206, 474, 242, 1658, 462, + /* 1110 */ 463, 458, 5, 1670, 135, 501, 80, 1643, 503, 1645, + /* 1120 */ 1646, 499, 292, 520, 199, 500, 1710, 208, 202, 1613, + /* 1130 */ 293, 1706, 1785, 111, 1276, 469, 1182, 1741, 40, 490, + /* 1140 */ 487, 1744, 1642, 1725, 1788, 18, 1670, 294, 510, 80, + /* 1150 */ 1643, 503, 1645, 1646, 499, 1563, 520, 505, 506, 1710, + /* 1160 */ 1562, 511, 512, 293, 1706, 1785, 228, 302, 209, 230, + /* 1170 */ 1658, 243, 68, 1475, 1767, 70, 523, 501, 1691, 1447, + /* 1180 */ 246, 237, 598, 47, 134, 252, 289, 500, 259, 248, + /* 1190 */ 1768, 1613, 253, 214, 1607, 250, 1642, 1606, 484, 217, + /* 1200 */ 317, 1603, 319, 320, 219, 1642, 491, 1150, 1670, 1151, + /* 1210 */ 150, 80, 1643, 503, 1645, 1646, 499, 324, 520, 1601, + /* 1220 */ 326, 1710, 327, 328, 1658, 293, 1706, 1785, 1600, 330, + /* 1230 */ 1599, 501, 332, 1658, 1598, 334, 1728, 1583, 151, 337, + /* 1240 */ 501, 500, 338, 1128, 1127, 1613, 343, 344, 1575, 1574, + /* 1250 */ 500, 1577, 1576, 1095, 1613, 1547, 1546, 1545, 1544, 1642, + /* 1260 */ 481, 1543, 1670, 1542, 1541, 132, 1643, 503, 1645, 1646, + /* 1270 */ 499, 1670, 520, 1540, 258, 1643, 503, 1645, 1646, 499, + /* 1280 */ 1539, 520, 1538, 1537, 1536, 1535, 1534, 1658, 1533, 1532, + /* 1290 */ 1531, 1530, 1529, 115, 501, 1528, 1527, 1526, 1525, 1642, + /* 1300 */ 1772, 1524, 1523, 1097, 500, 1522, 1521, 1520, 1613, 159, + /* 1310 */ 1787, 1405, 1373, 144, 106, 136, 384, 1769, 1642, 386, + /* 1320 */ 1372, 1591, 927, 1585, 926, 1670, 1569, 1658, 81, 1643, + /* 1330 */ 503, 1645, 1646, 499, 501, 520, 1560, 160, 1710, 400, + /* 1340 */ 107, 167, 1709, 1706, 500, 1453, 1658, 1404, 1613, 1402, + /* 1350 */ 1400, 401, 955, 498, 405, 1398, 1396, 1385, 1384, 409, + /* 1360 */ 399, 404, 403, 500, 413, 1670, 407, 1613, 81, 1643, + /* 1370 */ 503, 1645, 1646, 499, 1642, 520, 408, 411, 1710, 412, + /* 1380 */ 1371, 1455, 492, 1706, 1670, 1062, 1454, 266, 1643, 503, + /* 1390 */ 1645, 1646, 499, 497, 520, 494, 1682, 1061, 987, 986, + /* 1400 */ 985, 984, 1658, 602, 566, 1394, 568, 83, 981, 501, + /* 1410 */ 284, 174, 1389, 980, 979, 285, 436, 239, 1387, 500, + /* 1420 */ 286, 439, 1370, 1613, 441, 1369, 443, 82, 1590, 105, + /* 1430 */ 1135, 1584, 48, 1642, 1568, 591, 587, 583, 579, 238, + /* 1440 */ 1670, 450, 192, 81, 1643, 503, 1645, 1646, 499, 1567, + /* 1450 */ 520, 1559, 64, 1710, 195, 4, 37, 15, 1707, 200, + /* 1460 */ 1301, 1658, 43, 77, 205, 133, 233, 204, 501, 197, + /* 1470 */ 22, 1633, 1294, 451, 65, 10, 23, 16, 500, 42, + /* 1480 */ 1273, 1272, 1613, 211, 41, 297, 126, 1642, 143, 1330, + /* 1490 */ 1319, 307, 306, 17, 1325, 1324, 19, 295, 148, 1670, + /* 1500 */ 513, 1170, 267, 1643, 503, 1645, 1646, 499, 1329, 520, + /* 1510 */ 1328, 296, 29, 1236, 1215, 1658, 222, 1642, 1214, 1201, + /* 1520 */ 12, 20, 498, 502, 223, 459, 1163, 1299, 194, 1558, + /* 1530 */ 229, 1632, 500, 21, 234, 508, 1613, 225, 227, 66, + /* 1540 */ 67, 231, 1673, 1162, 13, 1658, 1133, 1217, 189, 519, + /* 1550 */ 1642, 44, 501, 1670, 1172, 71, 266, 1643, 503, 1645, + /* 1560 */ 1646, 499, 500, 520, 524, 1683, 1613, 522, 309, 299, + /* 1570 */ 1048, 526, 528, 1045, 529, 531, 1042, 1036, 1658, 532, + /* 1580 */ 1642, 521, 534, 1670, 535, 501, 267, 1643, 503, 1645, + /* 1590 */ 1646, 499, 1166, 520, 1034, 500, 537, 1025, 538, 1613, + /* 1600 */ 544, 1057, 301, 72, 1040, 1054, 1039, 1038, 1658, 1037, + /* 1610 */ 1642, 73, 477, 74, 1053, 501, 1670, 994, 953, 267, + /* 1620 */ 1643, 503, 1645, 1646, 499, 500, 520, 1055, 553, 1613, + /* 1630 */ 555, 236, 975, 974, 973, 1171, 972, 970, 1658, 1401, + /* 1640 */ 112, 971, 969, 968, 991, 501, 1670, 989, 965, 254, + /* 1650 */ 1643, 503, 1645, 1646, 499, 500, 520, 1174, 964, 1613, + /* 1660 */ 481, 1642, 578, 963, 960, 959, 958, 576, 518, 1220, + /* 1670 */ 1221, 1399, 577, 582, 581, 580, 1670, 1642, 110, 261, + /* 1680 */ 1643, 503, 1645, 1646, 499, 1397, 520, 584, 585, 1658, + /* 1690 */ 586, 1395, 588, 210, 1717, 476, 501, 475, 590, 589, + /* 1700 */ 1772, 1383, 592, 593, 1382, 1658, 500, 1368, 596, 597, + /* 1710 */ 1613, 1158, 501, 144, 247, 600, 601, 1769, 1343, 1343, + /* 1720 */ 1343, 1343, 500, 1343, 1642, 1343, 1613, 1670, 1343, 1343, + /* 1730 */ 263, 1643, 503, 1645, 1646, 499, 1343, 520, 1343, 1343, + /* 1740 */ 1343, 1343, 1642, 1670, 1343, 1343, 255, 1643, 503, 1645, + /* 1750 */ 1646, 499, 1658, 520, 1343, 1343, 1343, 1642, 1343, 501, + /* 1760 */ 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 500, + /* 1770 */ 1658, 1343, 1343, 1613, 1343, 1343, 1343, 501, 1343, 1343, + /* 1780 */ 1343, 1343, 1343, 1343, 1343, 1658, 1343, 500, 1343, 1343, + /* 1790 */ 1670, 1613, 501, 264, 1643, 503, 1645, 1646, 499, 1343, + /* 1800 */ 520, 1343, 500, 1343, 1642, 1343, 1613, 1343, 1670, 1343, + /* 1810 */ 1343, 256, 1643, 503, 1645, 1646, 499, 1343, 520, 1343, + /* 1820 */ 1642, 1343, 1343, 1670, 1343, 1343, 265, 1643, 503, 1645, + /* 1830 */ 1646, 499, 1658, 520, 1343, 1343, 1343, 1343, 1343, 501, + /* 1840 */ 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1658, 500, + /* 1850 */ 1343, 1343, 1343, 1613, 1343, 501, 1343, 1343, 1343, 1343, + /* 1860 */ 1343, 1343, 1343, 1343, 1343, 500, 1343, 1642, 1343, 1613, + /* 1870 */ 1670, 1343, 1343, 257, 1643, 503, 1645, 1646, 499, 1343, + /* 1880 */ 520, 1343, 1343, 1343, 1343, 1343, 1670, 1343, 1343, 1654, + /* 1890 */ 1643, 503, 1645, 1646, 499, 1658, 520, 1642, 1343, 1343, + /* 1900 */ 1343, 1343, 501, 1343, 1343, 1343, 1343, 1343, 1343, 1343, + /* 1910 */ 1343, 1343, 500, 1343, 1343, 1343, 1613, 1343, 1343, 1343, + /* 1920 */ 1343, 1343, 1343, 1343, 1343, 1658, 1343, 1343, 1343, 1343, + /* 1930 */ 1343, 1343, 501, 1670, 1343, 1343, 1653, 1643, 503, 1645, + /* 1940 */ 1646, 499, 500, 520, 1343, 1343, 1613, 1642, 1343, 1343, + /* 1950 */ 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, + /* 1960 */ 1343, 1343, 1343, 1670, 1343, 1343, 1652, 1643, 503, 1645, + /* 1970 */ 1646, 499, 1343, 520, 1343, 1658, 1343, 1642, 1343, 1343, + /* 1980 */ 1343, 1343, 501, 1343, 1343, 1343, 1343, 1343, 1343, 1343, + /* 1990 */ 1343, 1343, 500, 1343, 1343, 1343, 1613, 1343, 1343, 1343, + /* 2000 */ 1343, 1343, 1343, 1343, 1343, 1658, 1343, 1343, 1343, 1343, + /* 2010 */ 1343, 1343, 501, 1670, 1343, 1343, 276, 1643, 503, 1645, + /* 2020 */ 1646, 499, 500, 520, 1343, 1343, 1613, 1642, 1343, 1343, + /* 2030 */ 1343, 1343, 1343, 1343, 1343, 1343, 1642, 1343, 1343, 1343, + /* 2040 */ 1343, 1343, 1343, 1670, 1343, 1343, 275, 1643, 503, 1645, + /* 2050 */ 1646, 499, 1343, 520, 1343, 1658, 1343, 1343, 1343, 1343, + /* 2060 */ 1343, 1343, 501, 1343, 1658, 1343, 1343, 1343, 1343, 1343, + /* 2070 */ 1343, 501, 500, 1343, 1343, 1343, 1613, 1343, 1343, 1343, + /* 2080 */ 1343, 500, 1343, 1343, 1343, 1613, 1343, 1343, 1343, 1343, + /* 2090 */ 1642, 1343, 1343, 1670, 1343, 1343, 277, 1643, 503, 1645, + /* 2100 */ 1646, 499, 1670, 520, 1343, 274, 1643, 503, 1645, 1646, + /* 2110 */ 499, 1343, 520, 1343, 1343, 1343, 1343, 1343, 1658, 1343, + /* 2120 */ 1343, 1343, 1343, 1343, 1343, 501, 1343, 1343, 1343, 1343, + /* 2130 */ 1343, 1343, 1343, 1343, 1343, 500, 1343, 1343, 1343, 1613, + /* 2140 */ 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, + /* 2150 */ 1343, 1343, 1343, 1343, 1343, 1343, 1670, 1343, 1343, 260, + /* 2160 */ 1643, 503, 1645, 1646, 499, 1343, 520, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 244, 251, 246, 247, 244, 248, 246, 247, 290, 321, @@ -433,372 +439,375 @@ static const YYCODETYPE yy_lookahead[] = { /* 40 */ 27, 28, 29, 30, 31, 32, 269, 47, 269, 273, /* 50 */ 14, 15, 16, 276, 275, 332, 336, 248, 58, 280, /* 60 */ 12, 13, 14, 286, 64, 20, 290, 290, 20, 349, - /* 70 */ 22, 241, 58, 353, 20, 12, 13, 14, 15, 16, + /* 70 */ 22, 248, 58, 353, 57, 12, 13, 14, 15, 16, /* 80 */ 80, 305, 306, 307, 307, 276, 2, 310, 311, 312, /* 90 */ 313, 314, 315, 317, 317, 47, 12, 13, 14, 15, - /* 100 */ 16, 253, 102, 89, 269, 296, 58, 248, 12, 13, - /* 110 */ 241, 276, 64, 113, 266, 20, 20, 336, 22, 260, - /* 120 */ 290, 58, 274, 314, 248, 80, 267, 240, 80, 242, - /* 130 */ 349, 354, 355, 254, 353, 276, 248, 258, 329, 330, - /* 140 */ 331, 0, 333, 47, 81, 336, 245, 312, 260, 248, - /* 150 */ 102, 20, 89, 139, 58, 267, 156, 281, 349, 290, - /* 160 */ 64, 113, 353, 41, 276, 24, 25, 26, 27, 28, + /* 100 */ 16, 253, 102, 89, 281, 296, 58, 248, 12, 13, + /* 110 */ 249, 250, 64, 113, 266, 20, 20, 261, 22, 260, + /* 120 */ 20, 58, 274, 314, 22, 269, 267, 240, 80, 242, + /* 130 */ 80, 354, 355, 277, 336, 276, 248, 251, 329, 330, + /* 140 */ 331, 0, 333, 47, 81, 336, 336, 349, 260, 47, + /* 150 */ 102, 353, 89, 139, 58, 267, 156, 271, 349, 349, + /* 160 */ 64, 113, 353, 353, 276, 24, 25, 26, 27, 28, /* 170 */ 29, 30, 31, 32, 160, 80, 80, 92, 178, 179, - /* 180 */ 143, 181, 182, 183, 184, 185, 186, 187, 188, 189, + /* 180 */ 80, 181, 182, 183, 184, 185, 186, 187, 188, 189, /* 190 */ 190, 191, 192, 193, 194, 195, 111, 112, 102, 114, - /* 200 */ 115, 116, 139, 81, 156, 155, 178, 157, 208, 113, + /* 200 */ 115, 116, 139, 58, 156, 155, 178, 157, 208, 113, /* 210 */ 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, - /* 220 */ 249, 250, 286, 160, 241, 289, 178, 179, 292, 181, + /* 220 */ 254, 20, 286, 160, 258, 289, 178, 179, 292, 181, /* 230 */ 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, /* 240 */ 192, 193, 194, 195, 0, 217, 218, 219, 220, 221, - /* 250 */ 213, 214, 156, 12, 13, 14, 15, 16, 208, 196, - /* 260 */ 197, 198, 199, 200, 201, 202, 203, 204, 205, 1, - /* 270 */ 2, 248, 0, 290, 178, 179, 238, 181, 182, 183, + /* 250 */ 4, 80, 156, 12, 13, 14, 15, 16, 208, 196, + /* 260 */ 197, 198, 199, 200, 201, 202, 203, 204, 205, 165, + /* 270 */ 166, 248, 57, 169, 178, 179, 238, 181, 182, 183, /* 280 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, /* 290 */ 194, 195, 12, 13, 12, 13, 14, 15, 16, 276, - /* 300 */ 20, 0, 22, 208, 60, 61, 62, 63, 4, 65, + /* 300 */ 20, 0, 22, 208, 60, 61, 62, 63, 208, 65, /* 310 */ 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, - /* 320 */ 76, 77, 81, 241, 269, 278, 20, 47, 22, 57, - /* 330 */ 275, 241, 285, 286, 296, 280, 276, 314, 208, 208, - /* 340 */ 12, 13, 14, 283, 64, 4, 42, 43, 20, 81, - /* 350 */ 22, 328, 329, 330, 331, 49, 333, 2, 57, 269, + /* 320 */ 76, 77, 81, 178, 269, 278, 80, 47, 47, 269, + /* 330 */ 275, 241, 285, 286, 296, 280, 276, 314, 268, 261, + /* 340 */ 12, 13, 14, 20, 64, 64, 4, 269, 20, 279, + /* 350 */ 22, 328, 329, 330, 331, 277, 333, 2, 57, 269, /* 360 */ 80, 12, 13, 14, 15, 16, 276, 12, 13, 14, - /* 370 */ 15, 16, 290, 269, 336, 47, 286, 0, 261, 275, - /* 380 */ 290, 3, 102, 293, 280, 20, 269, 349, 12, 13, - /* 390 */ 151, 353, 64, 113, 277, 241, 20, 307, 22, 20, + /* 370 */ 15, 16, 312, 143, 336, 47, 286, 0, 261, 208, + /* 380 */ 290, 3, 102, 293, 42, 43, 269, 349, 12, 13, + /* 390 */ 151, 353, 64, 113, 277, 20, 20, 307, 22, 20, /* 400 */ 310, 311, 312, 313, 314, 315, 245, 317, 80, 248, /* 410 */ 171, 172, 93, 94, 95, 96, 97, 98, 99, 100, /* 420 */ 101, 102, 103, 47, 105, 106, 107, 108, 109, 110, - /* 430 */ 102, 12, 13, 14, 15, 16, 156, 60, 61, 261, - /* 440 */ 64, 113, 65, 20, 290, 68, 69, 269, 248, 72, - /* 450 */ 73, 74, 57, 165, 166, 277, 80, 169, 178, 179, + /* 430 */ 102, 12, 13, 14, 15, 16, 156, 60, 61, 269, + /* 440 */ 64, 113, 65, 213, 214, 68, 69, 277, 248, 72, + /* 450 */ 73, 74, 20, 55, 208, 80, 80, 308, 178, 179, /* 460 */ 260, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 470 */ 190, 191, 192, 193, 194, 195, 276, 79, 102, 255, - /* 480 */ 256, 139, 60, 61, 156, 87, 241, 65, 208, 113, - /* 490 */ 68, 69, 75, 296, 72, 73, 74, 148, 14, 0, - /* 500 */ 81, 22, 160, 57, 20, 248, 178, 179, 273, 181, + /* 470 */ 190, 191, 192, 193, 194, 195, 276, 79, 102, 14, + /* 480 */ 82, 332, 60, 61, 156, 20, 241, 65, 208, 113, + /* 490 */ 68, 69, 269, 296, 72, 73, 74, 148, 275, 91, + /* 500 */ 81, 22, 241, 280, 75, 248, 178, 179, 273, 181, /* 510 */ 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - /* 520 */ 192, 193, 194, 195, 55, 290, 47, 12, 13, 14, - /* 530 */ 15, 16, 156, 336, 21, 290, 119, 120, 196, 241, - /* 540 */ 305, 306, 307, 64, 45, 147, 349, 34, 79, 208, - /* 550 */ 353, 82, 317, 296, 178, 179, 241, 181, 182, 183, + /* 520 */ 192, 193, 194, 195, 145, 290, 47, 12, 13, 14, + /* 530 */ 15, 16, 156, 336, 245, 290, 269, 248, 286, 241, + /* 540 */ 305, 306, 307, 64, 292, 241, 349, 280, 119, 120, + /* 550 */ 353, 290, 317, 296, 178, 179, 241, 181, 182, 183, /* 560 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - /* 570 */ 194, 195, 12, 13, 18, 80, 20, 47, 229, 261, - /* 580 */ 20, 102, 22, 27, 269, 90, 30, 269, 290, 268, - /* 590 */ 20, 276, 113, 336, 64, 277, 33, 241, 241, 241, - /* 600 */ 279, 286, 224, 241, 48, 290, 349, 47, 45, 248, - /* 610 */ 353, 296, 241, 253, 51, 52, 53, 54, 55, 35, - /* 620 */ 308, 260, 307, 4, 64, 310, 311, 312, 313, 314, - /* 630 */ 315, 248, 317, 269, 274, 156, 251, 276, 248, 308, - /* 640 */ 80, 277, 79, 260, 332, 82, 290, 290, 290, 255, - /* 650 */ 256, 336, 290, 241, 285, 286, 271, 178, 179, 276, - /* 660 */ 145, 290, 102, 332, 349, 241, 276, 83, 353, 85, - /* 670 */ 86, 22, 88, 113, 118, 241, 92, 121, 122, 123, + /* 570 */ 194, 195, 12, 13, 18, 248, 20, 145, 229, 253, + /* 580 */ 20, 102, 22, 27, 269, 248, 30, 260, 290, 255, + /* 590 */ 256, 276, 113, 336, 290, 241, 33, 260, 241, 241, + /* 600 */ 274, 286, 224, 276, 48, 290, 349, 47, 45, 248, + /* 610 */ 353, 296, 241, 276, 51, 52, 53, 54, 55, 35, + /* 620 */ 20, 260, 307, 4, 64, 310, 311, 312, 313, 314, + /* 630 */ 315, 248, 317, 79, 20, 156, 22, 276, 248, 0, + /* 640 */ 80, 87, 79, 260, 290, 82, 241, 290, 290, 255, + /* 650 */ 256, 336, 20, 1, 2, 285, 286, 178, 179, 276, + /* 660 */ 145, 290, 102, 49, 349, 241, 276, 83, 353, 85, + /* 670 */ 86, 37, 88, 113, 118, 241, 92, 121, 122, 123, /* 680 */ 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, - /* 690 */ 134, 135, 136, 137, 138, 286, 47, 248, 248, 115, - /* 700 */ 248, 292, 290, 140, 314, 142, 20, 144, 248, 260, - /* 710 */ 260, 269, 260, 241, 290, 145, 156, 241, 241, 329, - /* 720 */ 330, 331, 280, 333, 290, 276, 276, 164, 276, 241, - /* 730 */ 241, 241, 262, 42, 43, 265, 276, 0, 178, 179, + /* 690 */ 134, 135, 136, 137, 138, 290, 57, 248, 248, 115, + /* 700 */ 248, 147, 41, 140, 314, 142, 0, 144, 248, 260, + /* 710 */ 260, 241, 260, 139, 290, 241, 156, 241, 241, 329, + /* 720 */ 330, 331, 21, 333, 290, 276, 276, 164, 276, 241, + /* 730 */ 241, 241, 241, 81, 160, 34, 276, 241, 178, 179, /* 740 */ 92, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 750 */ 190, 191, 192, 193, 194, 195, 248, 37, 269, 248, - /* 760 */ 248, 20, 290, 115, 248, 276, 290, 290, 260, 167, - /* 770 */ 168, 260, 260, 41, 314, 286, 260, 41, 290, 290, - /* 780 */ 290, 257, 64, 259, 276, 296, 0, 276, 276, 329, - /* 790 */ 330, 331, 276, 333, 206, 207, 307, 145, 146, 310, - /* 800 */ 311, 312, 313, 314, 315, 241, 317, 41, 22, 320, - /* 810 */ 14, 14, 84, 324, 325, 87, 20, 20, 18, 84, - /* 820 */ 1, 2, 87, 23, 0, 336, 58, 84, 209, 92, - /* 830 */ 87, 145, 41, 269, 0, 35, 36, 47, 349, 39, - /* 840 */ 276, 84, 353, 41, 87, 0, 22, 81, 111, 112, - /* 850 */ 286, 114, 115, 116, 290, 21, 56, 44, 24, 25, - /* 860 */ 26, 27, 28, 29, 30, 31, 32, 22, 91, 241, - /* 870 */ 47, 307, 81, 41, 310, 311, 312, 313, 314, 315, - /* 880 */ 41, 317, 47, 81, 320, 193, 194, 270, 324, 325, - /* 890 */ 326, 270, 270, 80, 41, 270, 19, 269, 299, 64, - /* 900 */ 270, 242, 338, 113, 276, 258, 356, 41, 344, 345, - /* 910 */ 33, 303, 41, 81, 286, 41, 241, 117, 290, 347, - /* 920 */ 81, 341, 45, 41, 247, 249, 41, 207, 51, 52, - /* 930 */ 53, 54, 55, 279, 81, 307, 113, 334, 310, 311, - /* 940 */ 312, 313, 314, 315, 269, 317, 178, 81, 148, 149, - /* 950 */ 150, 276, 81, 153, 41, 81, 79, 41, 158, 82, - /* 960 */ 228, 286, 226, 81, 41, 290, 81, 309, 269, 337, - /* 970 */ 170, 41, 41, 173, 350, 175, 176, 177, 20, 248, - /* 980 */ 45, 241, 307, 355, 304, 310, 311, 312, 313, 314, - /* 990 */ 315, 47, 317, 116, 81, 320, 154, 81, 297, 324, - /* 1000 */ 325, 326, 255, 248, 81, 40, 248, 284, 208, 269, - /* 1010 */ 248, 81, 81, 20, 139, 282, 276, 243, 141, 243, - /* 1020 */ 345, 144, 20, 301, 282, 253, 286, 20, 253, 276, - /* 1030 */ 290, 286, 294, 253, 20, 287, 253, 253, 253, 162, - /* 1040 */ 248, 164, 241, 0, 4, 64, 243, 307, 269, 243, - /* 1050 */ 310, 311, 312, 313, 314, 315, 269, 317, 269, 19, - /* 1060 */ 320, 290, 269, 269, 324, 325, 326, 269, 269, 248, - /* 1070 */ 269, 301, 294, 33, 269, 335, 269, 276, 269, 269, - /* 1080 */ 251, 163, 300, 20, 251, 45, 216, 286, 251, 251, - /* 1090 */ 50, 290, 276, 215, 286, 55, 241, 309, 287, 223, - /* 1100 */ 346, 222, 346, 291, 290, 342, 291, 290, 307, 211, - /* 1110 */ 290, 310, 311, 312, 313, 314, 315, 210, 317, 79, - /* 1120 */ 339, 320, 82, 276, 269, 324, 325, 326, 207, 20, - /* 1130 */ 308, 276, 40, 230, 80, 92, 335, 227, 343, 225, - /* 1140 */ 291, 286, 290, 142, 340, 290, 290, 287, 291, 290, - /* 1150 */ 241, 323, 327, 288, 111, 112, 276, 114, 115, 116, - /* 1160 */ 265, 80, 307, 251, 276, 310, 311, 312, 313, 314, - /* 1170 */ 315, 251, 317, 272, 259, 320, 248, 251, 269, 324, - /* 1180 */ 325, 326, 241, 357, 298, 276, 243, 295, 352, 302, - /* 1190 */ 335, 351, 263, 239, 252, 286, 263, 263, 0, 290, - /* 1200 */ 241, 0, 72, 0, 47, 174, 47, 47, 47, 174, - /* 1210 */ 269, 0, 47, 47, 0, 174, 307, 276, 47, 310, - /* 1220 */ 311, 312, 313, 314, 315, 0, 317, 286, 269, 320, - /* 1230 */ 47, 290, 241, 324, 325, 276, 0, 47, 0, 160, - /* 1240 */ 113, 159, 80, 156, 0, 286, 0, 152, 307, 290, - /* 1250 */ 241, 310, 311, 312, 313, 314, 315, 151, 317, 0, - /* 1260 */ 269, 0, 44, 0, 0, 0, 307, 276, 0, 310, - /* 1270 */ 311, 312, 313, 314, 315, 0, 317, 286, 269, 320, - /* 1280 */ 0, 290, 0, 324, 325, 276, 0, 0, 0, 0, - /* 1290 */ 0, 0, 0, 0, 0, 286, 0, 241, 307, 290, - /* 1300 */ 0, 310, 311, 312, 313, 314, 315, 316, 317, 318, - /* 1310 */ 319, 40, 241, 0, 0, 0, 307, 0, 0, 310, - /* 1320 */ 311, 312, 313, 314, 315, 269, 317, 0, 22, 320, - /* 1330 */ 0, 0, 276, 0, 325, 0, 0, 0, 40, 37, - /* 1340 */ 269, 44, 286, 14, 41, 44, 290, 276, 14, 0, - /* 1350 */ 38, 0, 0, 37, 0, 0, 37, 286, 0, 0, - /* 1360 */ 0, 290, 0, 307, 293, 37, 310, 311, 312, 313, - /* 1370 */ 314, 315, 47, 317, 0, 37, 37, 45, 307, 59, - /* 1380 */ 0, 310, 311, 312, 313, 314, 315, 241, 317, 47, - /* 1390 */ 45, 47, 45, 47, 37, 45, 0, 0, 0, 0, - /* 1400 */ 89, 22, 0, 47, 348, 0, 22, 0, 87, 41, - /* 1410 */ 47, 47, 47, 47, 41, 269, 47, 241, 48, 47, - /* 1420 */ 0, 0, 276, 47, 47, 22, 22, 0, 22, 20, - /* 1430 */ 22, 0, 286, 47, 0, 161, 290, 22, 0, 0, - /* 1440 */ 145, 0, 41, 212, 145, 269, 142, 80, 37, 212, - /* 1450 */ 140, 41, 276, 307, 41, 80, 310, 311, 312, 313, - /* 1460 */ 314, 315, 286, 317, 241, 319, 290, 41, 81, 293, - /* 1470 */ 81, 80, 80, 241, 81, 80, 44, 41, 206, 44, - /* 1480 */ 2, 81, 212, 307, 41, 41, 310, 311, 312, 313, - /* 1490 */ 314, 315, 269, 317, 44, 81, 47, 81, 47, 276, - /* 1500 */ 47, 269, 47, 47, 47, 41, 81, 44, 276, 286, - /* 1510 */ 80, 44, 22, 290, 0, 22, 293, 80, 286, 37, - /* 1520 */ 241, 178, 290, 81, 143, 80, 80, 80, 180, 81, - /* 1530 */ 307, 44, 80, 310, 311, 312, 313, 314, 315, 307, - /* 1540 */ 317, 81, 310, 311, 312, 313, 314, 315, 269, 317, - /* 1550 */ 241, 80, 140, 80, 44, 276, 90, 80, 47, 91, - /* 1560 */ 81, 80, 47, 80, 80, 286, 81, 81, 47, 290, - /* 1570 */ 81, 80, 47, 81, 80, 47, 80, 47, 269, 81, - /* 1580 */ 80, 22, 241, 80, 104, 276, 307, 92, 80, 310, - /* 1590 */ 311, 312, 313, 314, 315, 286, 317, 47, 80, 290, - /* 1600 */ 47, 104, 22, 104, 59, 104, 58, 64, 41, 78, - /* 1610 */ 269, 47, 241, 47, 22, 113, 307, 276, 47, 310, - /* 1620 */ 311, 312, 313, 314, 315, 47, 317, 286, 47, 47, - /* 1630 */ 47, 290, 64, 47, 47, 47, 47, 47, 47, 0, - /* 1640 */ 269, 37, 47, 47, 45, 0, 45, 276, 307, 47, - /* 1650 */ 37, 310, 311, 312, 313, 314, 315, 286, 317, 241, - /* 1660 */ 0, 290, 47, 45, 37, 0, 47, 45, 37, 0, - /* 1670 */ 47, 46, 0, 0, 241, 22, 21, 21, 307, 22, - /* 1680 */ 22, 310, 311, 312, 313, 314, 315, 269, 317, 358, - /* 1690 */ 20, 358, 358, 358, 276, 358, 358, 358, 358, 358, - /* 1700 */ 358, 358, 269, 358, 286, 358, 241, 358, 290, 276, - /* 1710 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 286, - /* 1720 */ 358, 358, 358, 290, 358, 307, 358, 358, 310, 311, - /* 1730 */ 312, 313, 314, 315, 269, 317, 358, 358, 358, 358, - /* 1740 */ 307, 276, 358, 310, 311, 312, 313, 314, 315, 358, - /* 1750 */ 317, 286, 358, 358, 358, 290, 358, 358, 241, 358, - /* 1760 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 1770 */ 358, 358, 307, 358, 358, 310, 311, 312, 313, 314, - /* 1780 */ 315, 358, 317, 358, 358, 358, 269, 358, 241, 358, - /* 1790 */ 358, 358, 358, 276, 358, 358, 358, 358, 358, 358, - /* 1800 */ 358, 358, 358, 286, 358, 358, 358, 290, 358, 358, - /* 1810 */ 358, 358, 358, 358, 358, 358, 269, 358, 358, 358, - /* 1820 */ 241, 358, 358, 276, 307, 358, 358, 310, 311, 312, - /* 1830 */ 313, 314, 315, 286, 317, 241, 358, 290, 358, 358, - /* 1840 */ 358, 358, 358, 358, 358, 358, 358, 358, 269, 358, - /* 1850 */ 358, 358, 358, 358, 307, 276, 358, 310, 311, 312, - /* 1860 */ 313, 314, 315, 269, 317, 286, 358, 358, 358, 290, - /* 1870 */ 276, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 1880 */ 286, 358, 241, 358, 290, 358, 307, 358, 358, 310, - /* 1890 */ 311, 312, 313, 314, 315, 241, 317, 358, 358, 358, - /* 1900 */ 358, 307, 358, 358, 310, 311, 312, 313, 314, 315, - /* 1910 */ 269, 317, 358, 358, 358, 358, 358, 276, 358, 358, - /* 1920 */ 358, 358, 358, 269, 12, 13, 358, 286, 358, 358, - /* 1930 */ 276, 290, 358, 358, 22, 358, 358, 358, 358, 358, - /* 1940 */ 286, 358, 358, 358, 290, 358, 241, 358, 307, 358, - /* 1950 */ 358, 310, 311, 312, 313, 314, 315, 358, 317, 47, - /* 1960 */ 358, 307, 358, 358, 310, 311, 312, 313, 314, 315, - /* 1970 */ 358, 317, 358, 358, 269, 358, 64, 358, 358, 358, - /* 1980 */ 358, 276, 358, 358, 358, 358, 358, 358, 358, 358, - /* 1990 */ 358, 286, 358, 358, 358, 290, 358, 358, 358, 358, - /* 2000 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2010 */ 358, 358, 307, 358, 102, 310, 311, 312, 313, 314, - /* 2020 */ 315, 358, 317, 358, 358, 113, 358, 358, 358, 358, - /* 2030 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2040 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2050 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2060 */ 358, 358, 358, 358, 358, 358, 358, 358, 156, 358, - /* 2070 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2080 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2090 */ 178, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2100 */ 358, 189, 190, 191, 358, 358, 358, 358, 358, 358, - /* 2110 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2120 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2130 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, + /* 750 */ 190, 191, 192, 193, 194, 195, 248, 41, 269, 248, + /* 760 */ 290, 241, 276, 115, 290, 276, 290, 290, 260, 283, + /* 770 */ 196, 260, 42, 43, 314, 286, 206, 207, 290, 290, + /* 780 */ 290, 290, 14, 308, 276, 296, 290, 276, 20, 329, + /* 790 */ 330, 331, 262, 333, 0, 265, 307, 81, 92, 310, + /* 800 */ 311, 312, 313, 314, 315, 241, 317, 332, 84, 320, + /* 810 */ 290, 87, 84, 324, 325, 87, 84, 111, 112, 87, + /* 820 */ 114, 115, 116, 0, 257, 336, 259, 84, 209, 14, + /* 830 */ 87, 0, 41, 269, 0, 20, 167, 168, 349, 45, + /* 840 */ 276, 207, 353, 41, 18, 22, 41, 145, 146, 23, + /* 850 */ 286, 1, 2, 22, 290, 44, 22, 193, 194, 41, + /* 860 */ 296, 35, 36, 41, 64, 39, 241, 270, 41, 270, + /* 870 */ 41, 307, 81, 270, 310, 311, 312, 313, 314, 315, + /* 880 */ 4, 317, 56, 81, 320, 41, 81, 226, 324, 325, + /* 890 */ 41, 80, 80, 41, 269, 19, 270, 270, 242, 81, + /* 900 */ 336, 276, 90, 299, 41, 47, 80, 47, 81, 33, + /* 910 */ 81, 286, 41, 349, 356, 290, 0, 353, 258, 241, + /* 920 */ 41, 45, 41, 347, 303, 81, 50, 41, 341, 247, + /* 930 */ 81, 55, 307, 81, 249, 310, 311, 312, 313, 314, + /* 940 */ 315, 41, 317, 117, 81, 320, 41, 269, 269, 324, + /* 950 */ 325, 326, 81, 279, 276, 79, 309, 334, 82, 350, + /* 960 */ 81, 47, 81, 338, 286, 350, 350, 81, 290, 344, + /* 970 */ 345, 113, 241, 113, 148, 149, 150, 337, 64, 153, + /* 980 */ 20, 81, 248, 45, 158, 307, 81, 304, 310, 311, + /* 990 */ 312, 313, 314, 315, 47, 317, 170, 255, 248, 173, + /* 1000 */ 269, 175, 176, 177, 297, 154, 248, 276, 92, 40, + /* 1010 */ 284, 139, 248, 20, 243, 282, 243, 286, 282, 20, + /* 1020 */ 301, 290, 253, 286, 20, 253, 348, 111, 112, 294, + /* 1030 */ 114, 115, 116, 253, 208, 276, 20, 253, 307, 0, + /* 1040 */ 287, 310, 311, 312, 313, 314, 315, 241, 317, 253, + /* 1050 */ 228, 320, 248, 243, 253, 324, 325, 326, 269, 269, + /* 1060 */ 21, 269, 269, 24, 25, 26, 27, 28, 29, 30, + /* 1070 */ 31, 32, 269, 248, 64, 269, 345, 269, 269, 243, + /* 1080 */ 241, 251, 276, 269, 301, 269, 269, 290, 269, 163, + /* 1090 */ 20, 251, 286, 286, 251, 300, 290, 276, 251, 216, + /* 1100 */ 346, 346, 215, 223, 343, 342, 222, 294, 269, 211, + /* 1110 */ 290, 287, 210, 307, 340, 276, 310, 311, 312, 313, + /* 1120 */ 314, 315, 290, 317, 291, 286, 320, 339, 291, 290, + /* 1130 */ 324, 325, 326, 276, 207, 290, 20, 309, 40, 227, + /* 1140 */ 225, 335, 241, 308, 357, 80, 307, 230, 142, 310, + /* 1150 */ 311, 312, 313, 314, 315, 291, 317, 290, 290, 320, + /* 1160 */ 291, 288, 287, 324, 325, 326, 276, 290, 327, 251, + /* 1170 */ 269, 265, 251, 276, 335, 80, 272, 276, 323, 259, + /* 1180 */ 248, 251, 243, 298, 302, 263, 295, 286, 263, 252, + /* 1190 */ 352, 290, 263, 351, 0, 239, 241, 0, 352, 351, + /* 1200 */ 72, 0, 47, 174, 351, 241, 352, 47, 307, 47, + /* 1210 */ 47, 310, 311, 312, 313, 314, 315, 174, 317, 0, + /* 1220 */ 47, 320, 47, 174, 269, 324, 325, 326, 0, 47, + /* 1230 */ 0, 276, 47, 269, 0, 47, 335, 0, 80, 160, + /* 1240 */ 276, 286, 159, 113, 156, 290, 152, 151, 0, 0, + /* 1250 */ 286, 0, 0, 44, 290, 0, 0, 0, 0, 241, + /* 1260 */ 296, 0, 307, 0, 0, 310, 311, 312, 313, 314, + /* 1270 */ 315, 307, 317, 0, 310, 311, 312, 313, 314, 315, + /* 1280 */ 0, 317, 0, 0, 0, 0, 0, 269, 0, 0, + /* 1290 */ 0, 0, 0, 40, 276, 0, 0, 0, 0, 241, + /* 1300 */ 336, 0, 0, 22, 286, 0, 0, 0, 290, 40, + /* 1310 */ 355, 0, 0, 349, 37, 41, 44, 353, 241, 44, + /* 1320 */ 0, 0, 14, 0, 14, 307, 0, 269, 310, 311, + /* 1330 */ 312, 313, 314, 315, 276, 317, 0, 38, 320, 45, + /* 1340 */ 37, 37, 324, 325, 286, 0, 269, 0, 290, 0, + /* 1350 */ 0, 37, 59, 276, 37, 0, 0, 0, 0, 37, + /* 1360 */ 47, 45, 47, 286, 37, 307, 47, 290, 310, 311, + /* 1370 */ 312, 313, 314, 315, 241, 317, 45, 47, 320, 45, + /* 1380 */ 0, 0, 324, 325, 307, 47, 0, 310, 311, 312, + /* 1390 */ 313, 314, 315, 316, 317, 318, 319, 22, 47, 47, + /* 1400 */ 47, 47, 269, 19, 41, 0, 41, 89, 47, 276, + /* 1410 */ 22, 87, 0, 47, 47, 22, 48, 33, 0, 286, + /* 1420 */ 22, 47, 0, 290, 22, 0, 22, 20, 0, 45, + /* 1430 */ 47, 0, 145, 241, 0, 51, 52, 53, 54, 55, + /* 1440 */ 307, 22, 142, 310, 311, 312, 313, 314, 315, 0, + /* 1450 */ 317, 0, 80, 320, 37, 41, 41, 212, 325, 81, + /* 1460 */ 81, 269, 41, 79, 41, 80, 82, 80, 276, 140, + /* 1470 */ 80, 44, 81, 145, 80, 2, 41, 212, 286, 41, + /* 1480 */ 81, 81, 290, 44, 206, 293, 161, 241, 44, 81, + /* 1490 */ 81, 12, 13, 41, 47, 47, 41, 47, 44, 307, + /* 1500 */ 116, 22, 310, 311, 312, 313, 314, 315, 47, 317, + /* 1510 */ 47, 47, 80, 178, 81, 269, 44, 241, 81, 22, + /* 1520 */ 80, 80, 276, 180, 81, 141, 47, 81, 144, 0, + /* 1530 */ 37, 44, 286, 80, 44, 143, 290, 80, 80, 80, + /* 1540 */ 80, 140, 80, 64, 212, 269, 162, 81, 164, 80, + /* 1550 */ 241, 80, 276, 307, 22, 90, 310, 311, 312, 313, + /* 1560 */ 314, 315, 286, 317, 47, 319, 290, 91, 47, 293, + /* 1570 */ 81, 80, 47, 81, 80, 47, 81, 81, 269, 80, + /* 1580 */ 241, 102, 47, 307, 80, 276, 310, 311, 312, 313, + /* 1590 */ 314, 315, 113, 317, 81, 286, 47, 22, 80, 290, + /* 1600 */ 92, 47, 293, 80, 104, 47, 104, 104, 269, 104, + /* 1610 */ 241, 80, 248, 80, 22, 276, 307, 64, 59, 310, + /* 1620 */ 311, 312, 313, 314, 315, 286, 317, 113, 58, 290, + /* 1630 */ 78, 41, 47, 47, 47, 156, 47, 22, 269, 0, + /* 1640 */ 276, 47, 47, 47, 64, 276, 307, 47, 47, 310, + /* 1650 */ 311, 312, 313, 314, 315, 286, 317, 178, 47, 290, + /* 1660 */ 296, 241, 37, 47, 47, 47, 47, 47, 189, 190, + /* 1670 */ 191, 0, 45, 37, 45, 47, 307, 241, 314, 310, + /* 1680 */ 311, 312, 313, 314, 315, 0, 317, 47, 45, 269, + /* 1690 */ 37, 0, 47, 329, 330, 331, 276, 333, 37, 45, + /* 1700 */ 336, 0, 47, 46, 0, 269, 286, 0, 22, 21, + /* 1710 */ 290, 22, 276, 349, 22, 21, 20, 353, 358, 358, + /* 1720 */ 358, 358, 286, 358, 241, 358, 290, 307, 358, 358, + /* 1730 */ 310, 311, 312, 313, 314, 315, 358, 317, 358, 358, + /* 1740 */ 358, 358, 241, 307, 358, 358, 310, 311, 312, 313, + /* 1750 */ 314, 315, 269, 317, 358, 358, 358, 241, 358, 276, + /* 1760 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 286, + /* 1770 */ 269, 358, 358, 290, 358, 358, 358, 276, 358, 358, + /* 1780 */ 358, 358, 358, 358, 358, 269, 358, 286, 358, 358, + /* 1790 */ 307, 290, 276, 310, 311, 312, 313, 314, 315, 358, + /* 1800 */ 317, 358, 286, 358, 241, 358, 290, 358, 307, 358, + /* 1810 */ 358, 310, 311, 312, 313, 314, 315, 358, 317, 358, + /* 1820 */ 241, 358, 358, 307, 358, 358, 310, 311, 312, 313, + /* 1830 */ 314, 315, 269, 317, 358, 358, 358, 358, 358, 276, + /* 1840 */ 358, 358, 358, 358, 358, 358, 358, 358, 269, 286, + /* 1850 */ 358, 358, 358, 290, 358, 276, 358, 358, 358, 358, + /* 1860 */ 358, 358, 358, 358, 358, 286, 358, 241, 358, 290, + /* 1870 */ 307, 358, 358, 310, 311, 312, 313, 314, 315, 358, + /* 1880 */ 317, 358, 358, 358, 358, 358, 307, 358, 358, 310, + /* 1890 */ 311, 312, 313, 314, 315, 269, 317, 241, 358, 358, + /* 1900 */ 358, 358, 276, 358, 358, 358, 358, 358, 358, 358, + /* 1910 */ 358, 358, 286, 358, 358, 358, 290, 358, 358, 358, + /* 1920 */ 358, 358, 358, 358, 358, 269, 358, 358, 358, 358, + /* 1930 */ 358, 358, 276, 307, 358, 358, 310, 311, 312, 313, + /* 1940 */ 314, 315, 286, 317, 358, 358, 290, 241, 358, 358, + /* 1950 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, + /* 1960 */ 358, 358, 358, 307, 358, 358, 310, 311, 312, 313, + /* 1970 */ 314, 315, 358, 317, 358, 269, 358, 241, 358, 358, + /* 1980 */ 358, 358, 276, 358, 358, 358, 358, 358, 358, 358, + /* 1990 */ 358, 358, 286, 358, 358, 358, 290, 358, 358, 358, + /* 2000 */ 358, 358, 358, 358, 358, 269, 358, 358, 358, 358, + /* 2010 */ 358, 358, 276, 307, 358, 358, 310, 311, 312, 313, + /* 2020 */ 314, 315, 286, 317, 358, 358, 290, 241, 358, 358, + /* 2030 */ 358, 358, 358, 358, 358, 358, 241, 358, 358, 358, + /* 2040 */ 358, 358, 358, 307, 358, 358, 310, 311, 312, 313, + /* 2050 */ 314, 315, 358, 317, 358, 269, 358, 358, 358, 358, + /* 2060 */ 358, 358, 276, 358, 269, 358, 358, 358, 358, 358, + /* 2070 */ 358, 276, 286, 358, 358, 358, 290, 358, 358, 358, + /* 2080 */ 358, 286, 358, 358, 358, 290, 358, 358, 358, 358, + /* 2090 */ 241, 358, 358, 307, 358, 358, 310, 311, 312, 313, + /* 2100 */ 314, 315, 307, 317, 358, 310, 311, 312, 313, 314, + /* 2110 */ 315, 358, 317, 358, 358, 358, 358, 358, 269, 358, + /* 2120 */ 358, 358, 358, 358, 358, 276, 358, 358, 358, 358, + /* 2130 */ 358, 358, 358, 358, 358, 286, 358, 358, 358, 290, /* 2140 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2150 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2160 */ 358, 358, 358, + /* 2150 */ 358, 358, 358, 358, 358, 358, 307, 358, 358, 310, + /* 2160 */ 311, 312, 313, 314, 315, 358, 317, }; -#define YY_SHIFT_COUNT (590) +#define YY_SHIFT_COUNT (602) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1912) +#define YY_SHIFT_MAX (1707) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 800, 0, 48, 96, 96, 96, 96, 280, 96, 96, - /* 10 */ 328, 376, 560, 376, 376, 376, 376, 376, 376, 376, + /* 0 */ 826, 0, 0, 48, 96, 96, 96, 96, 280, 280, + /* 10 */ 96, 96, 328, 376, 560, 376, 376, 376, 376, 376, /* 20 */ 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, - /* 30 */ 376, 376, 376, 376, 376, 376, 95, 45, 45, 45, - /* 40 */ 1912, 1912, 1912, 131, 50, 54, 54, 130, 304, 304, - /* 50 */ 341, 54, 54, 54, 54, 54, 54, 395, 54, 365, - /* 60 */ 379, 130, 423, 365, 54, 54, 365, 54, 365, 365, - /* 70 */ 423, 365, 54, 446, 556, 63, 14, 14, 13, 479, - /* 80 */ 422, 479, 479, 479, 479, 479, 479, 479, 479, 479, + /* 30 */ 376, 376, 376, 376, 376, 376, 376, 376, 95, 95, + /* 40 */ 375, 375, 375, 1479, 1479, 1479, 100, 50, 171, 45, + /* 50 */ 45, 342, 342, 246, 171, 171, 45, 45, 45, 45, + /* 60 */ 45, 45, 17, 45, 201, 323, 600, 201, 45, 45, + /* 70 */ 201, 45, 201, 201, 600, 201, 45, 215, 556, 63, + /* 80 */ 14, 14, 13, 479, 422, 479, 479, 479, 479, 479, /* 90 */ 479, 479, 479, 479, 479, 479, 479, 479, 479, 479, - /* 100 */ 584, 306, 484, 484, 272, 530, 570, 570, 570, 301, - /* 110 */ 530, 741, 423, 365, 365, 423, 777, 718, 319, 319, - /* 120 */ 319, 319, 319, 319, 319, 877, 834, 377, 349, 28, - /* 130 */ 288, 37, 691, 649, 648, 686, 588, 720, 588, 796, - /* 140 */ 378, 619, 797, 958, 935, 944, 842, 958, 958, 965, - /* 150 */ 875, 875, 958, 993, 993, 1002, 395, 423, 395, 1007, - /* 160 */ 395, 741, 1014, 395, 395, 958, 395, 993, 365, 365, - /* 170 */ 365, 365, 365, 365, 365, 365, 365, 365, 365, 958, - /* 180 */ 993, 981, 1002, 446, 918, 423, 446, 1007, 446, 741, - /* 190 */ 1014, 446, 1063, 870, 878, 981, 870, 878, 981, 981, - /* 200 */ 876, 879, 898, 907, 921, 741, 1109, 1092, 903, 910, - /* 210 */ 914, 1054, 365, 878, 981, 981, 878, 981, 1001, 741, - /* 220 */ 1014, 446, 777, 446, 741, 1081, 718, 958, 446, 993, - /* 230 */ 2104, 2104, 2104, 2104, 2104, 2104, 2104, 2104, 244, 563, - /* 240 */ 141, 1040, 737, 1043, 241, 84, 355, 515, 419, 85, - /* 250 */ 282, 282, 282, 282, 282, 282, 282, 282, 239, 469, - /* 260 */ 417, 398, 268, 342, 36, 36, 36, 36, 499, 122, - /* 270 */ 728, 735, 743, 757, 786, 824, 845, 513, 602, 652, - /* 280 */ 766, 791, 802, 819, 692, 736, 732, 832, 768, 839, - /* 290 */ 813, 853, 866, 874, 882, 885, 790, 823, 871, 913, - /* 300 */ 916, 923, 930, 931, 495, 835, 1198, 1201, 1130, 1203, - /* 310 */ 1157, 1031, 1159, 1160, 1161, 1035, 1211, 1165, 1166, 1041, - /* 320 */ 1214, 1171, 1225, 1183, 1236, 1190, 1238, 1162, 1079, 1082, - /* 330 */ 1127, 1087, 1244, 1246, 1095, 1106, 1259, 1261, 1218, 1263, - /* 340 */ 1264, 1265, 1268, 1275, 1280, 1282, 1286, 1287, 1288, 1289, - /* 350 */ 1290, 1291, 1292, 1293, 1294, 1296, 1300, 1271, 1313, 1314, - /* 360 */ 1315, 1317, 1318, 1327, 1306, 1330, 1331, 1333, 1335, 1336, - /* 370 */ 1337, 1298, 1302, 1303, 1329, 1297, 1334, 1301, 1349, 1312, - /* 380 */ 1316, 1351, 1352, 1354, 1355, 1319, 1358, 1320, 1359, 1360, - /* 390 */ 1325, 1332, 1328, 1362, 1342, 1345, 1338, 1374, 1344, 1347, - /* 400 */ 1339, 1380, 1346, 1350, 1357, 1396, 1397, 1398, 1399, 1311, - /* 410 */ 1321, 1356, 1379, 1402, 1363, 1364, 1365, 1366, 1368, 1373, - /* 420 */ 1369, 1372, 1376, 1405, 1384, 1407, 1403, 1370, 1420, 1404, - /* 430 */ 1377, 1421, 1406, 1427, 1408, 1409, 1431, 1295, 1386, 1434, - /* 440 */ 1274, 1415, 1299, 1304, 1438, 1439, 1441, 1367, 1411, 1310, - /* 450 */ 1401, 1410, 1231, 1387, 1413, 1389, 1375, 1391, 1392, 1393, - /* 460 */ 1426, 1432, 1395, 1436, 1237, 1400, 1414, 1435, 1272, 1443, - /* 470 */ 1450, 1416, 1444, 1270, 1449, 1451, 1453, 1455, 1456, 1457, - /* 480 */ 1478, 1343, 1464, 1425, 1430, 1442, 1463, 1437, 1445, 1467, - /* 490 */ 1490, 1348, 1446, 1448, 1460, 1447, 1452, 1381, 1471, 1514, - /* 500 */ 1482, 1412, 1473, 1466, 1487, 1510, 1477, 1479, 1481, 1493, - /* 510 */ 1483, 1468, 1485, 1511, 1515, 1484, 1486, 1521, 1491, 1489, - /* 520 */ 1525, 1494, 1492, 1528, 1496, 1498, 1530, 1500, 1480, 1497, - /* 530 */ 1499, 1501, 1559, 1495, 1503, 1508, 1550, 1518, 1502, 1553, - /* 540 */ 1580, 1545, 1548, 1543, 1531, 1567, 1564, 1566, 1571, 1578, - /* 550 */ 1581, 1592, 1582, 1583, 1568, 1368, 1586, 1373, 1587, 1588, - /* 560 */ 1589, 1590, 1591, 1595, 1639, 1596, 1599, 1604, 1645, 1602, - /* 570 */ 1601, 1613, 1660, 1615, 1618, 1627, 1665, 1619, 1622, 1631, - /* 580 */ 1669, 1623, 1625, 1672, 1673, 1653, 1655, 1657, 1658, 1656, - /* 590 */ 1670, + /* 100 */ 479, 479, 479, 479, 584, 614, 465, 465, 301, 281, + /* 110 */ 379, 379, 379, 639, 281, 632, 600, 201, 201, 600, + /* 120 */ 408, 800, 319, 319, 319, 319, 319, 319, 319, 1384, + /* 130 */ 1039, 377, 349, 28, 104, 230, 730, 102, 648, 432, + /* 140 */ 570, 634, 570, 768, 378, 378, 378, 619, 815, 960, + /* 150 */ 938, 947, 851, 960, 960, 969, 872, 872, 960, 993, + /* 160 */ 993, 999, 17, 600, 17, 1004, 17, 632, 1016, 17, + /* 170 */ 17, 960, 17, 993, 201, 201, 201, 201, 201, 201, + /* 180 */ 201, 201, 201, 201, 201, 960, 993, 1010, 999, 215, + /* 190 */ 926, 600, 215, 1004, 215, 632, 1016, 215, 1070, 883, + /* 200 */ 887, 1010, 883, 887, 1010, 1010, 880, 884, 898, 902, + /* 210 */ 927, 632, 1116, 1098, 912, 915, 917, 912, 915, 912, + /* 220 */ 915, 1065, 201, 887, 1010, 1010, 887, 1010, 1006, 632, + /* 230 */ 1016, 215, 408, 215, 632, 1095, 800, 960, 215, 993, + /* 240 */ 2167, 2167, 2167, 2167, 2167, 2167, 2167, 2167, 244, 563, + /* 250 */ 141, 876, 706, 916, 241, 84, 355, 515, 419, 85, + /* 260 */ 282, 282, 282, 282, 282, 282, 282, 282, 239, 398, + /* 270 */ 429, 554, 652, 574, 36, 36, 36, 36, 794, 716, + /* 280 */ 724, 728, 732, 743, 823, 831, 834, 701, 669, 702, + /* 290 */ 791, 802, 805, 850, 664, 661, 822, 818, 145, 827, + /* 300 */ 811, 829, 844, 849, 852, 863, 858, 860, 871, 879, + /* 310 */ 881, 886, 900, 905, 812, 914, 1194, 1197, 1128, 1201, + /* 320 */ 1155, 1029, 1160, 1162, 1163, 1043, 1219, 1173, 1175, 1049, + /* 330 */ 1228, 1182, 1230, 1185, 1234, 1188, 1237, 1158, 1079, 1083, + /* 340 */ 1130, 1088, 1251, 1252, 1094, 1096, 1248, 1249, 1209, 1255, + /* 350 */ 1256, 1257, 1258, 1261, 1263, 1264, 1273, 1280, 1282, 1283, + /* 360 */ 1284, 1285, 1286, 1288, 1289, 1290, 1291, 1253, 1292, 1295, + /* 370 */ 1296, 1297, 1298, 1301, 1281, 1302, 1305, 1306, 1307, 1311, + /* 380 */ 1312, 1269, 1277, 1274, 1308, 1272, 1310, 1275, 1320, 1299, + /* 390 */ 1303, 1321, 1323, 1326, 1336, 1304, 1345, 1293, 1347, 1349, + /* 400 */ 1313, 1294, 1314, 1350, 1315, 1316, 1317, 1355, 1319, 1331, + /* 410 */ 1322, 1356, 1330, 1334, 1327, 1357, 1358, 1380, 1381, 1318, + /* 420 */ 1324, 1338, 1375, 1386, 1351, 1352, 1353, 1354, 1363, 1365, + /* 430 */ 1361, 1366, 1367, 1405, 1388, 1412, 1393, 1368, 1418, 1398, + /* 440 */ 1374, 1422, 1402, 1425, 1404, 1407, 1428, 1287, 1383, 1431, + /* 450 */ 1325, 1419, 1328, 1300, 1434, 1449, 1451, 1372, 1417, 1329, + /* 460 */ 1414, 1415, 1245, 1378, 1421, 1379, 1385, 1387, 1390, 1391, + /* 470 */ 1423, 1427, 1394, 1435, 1265, 1399, 1400, 1439, 1278, 1438, + /* 480 */ 1444, 1408, 1452, 1332, 1409, 1447, 1448, 1450, 1461, 1463, + /* 490 */ 1464, 1409, 1473, 1335, 1455, 1433, 1432, 1437, 1454, 1440, + /* 500 */ 1441, 1472, 1497, 1343, 1453, 1443, 1446, 1457, 1458, 1392, + /* 510 */ 1459, 1529, 1493, 1401, 1460, 1465, 1487, 1490, 1462, 1466, + /* 520 */ 1469, 1532, 1471, 1476, 1489, 1517, 1521, 1491, 1492, 1525, + /* 530 */ 1494, 1495, 1528, 1499, 1496, 1535, 1504, 1513, 1549, 1518, + /* 540 */ 1500, 1502, 1503, 1505, 1575, 1508, 1523, 1531, 1554, 1533, + /* 550 */ 1514, 1558, 1592, 1559, 1570, 1553, 1552, 1590, 1585, 1586, + /* 560 */ 1587, 1589, 1594, 1615, 1595, 1596, 1580, 1363, 1600, 1365, + /* 570 */ 1601, 1611, 1616, 1617, 1618, 1619, 1639, 1620, 1627, 1625, + /* 580 */ 1671, 1628, 1629, 1636, 1685, 1640, 1643, 1653, 1691, 1645, + /* 590 */ 1654, 1661, 1701, 1655, 1657, 1704, 1707, 1686, 1688, 1689, + /* 600 */ 1692, 1694, 1696, }; -#define YY_REDUCE_COUNT (237) +#define YY_REDUCE_COUNT (247) #define YY_REDUCE_MIN (-317) -#define YY_REDUCE_MAX (1705) +#define YY_REDUCE_MAX (1849) static const short yy_reduce_ofst[] = { - /* 0 */ 38, 489, 564, 675, 740, 801, 855, 315, 909, 959, - /* 10 */ 991, -223, 1009, 90, 1056, 628, 1071, 1146, 1176, 1223, - /* 20 */ 941, 1232, 1279, 1309, 1341, 1371, 1418, 1433, 1465, 1517, - /* 30 */ 1547, 1579, 1594, 1641, 1654, 1705, -191, 23, 390, 460, - /* 40 */ -224, 235, -282, 257, -280, -141, -112, 197, -244, -240, - /* 50 */ -317, -243, 200, 361, 383, 449, 450, -152, 452, -221, - /* 60 */ -165, -219, -64, 117, 508, 511, 55, 512, 178, 104, - /* 70 */ 47, 318, 516, -250, -124, -312, -312, -312, -113, -170, - /* 80 */ -121, -131, -17, 82, 154, 245, 298, 356, 357, 358, - /* 90 */ 362, 371, 412, 424, 434, 472, 476, 477, 488, 490, - /* 100 */ 321, -29, -99, 161, 360, 224, -277, 312, 331, 385, - /* 110 */ 394, 60, 409, 364, 442, 369, 470, 524, -259, -255, - /* 120 */ 617, 621, 622, 625, 630, 599, 659, 647, 550, 572, - /* 130 */ 608, 580, 677, 676, 654, 658, 603, 603, 603, 699, - /* 140 */ 624, 632, 699, 731, 680, 747, 701, 755, 758, 723, - /* 150 */ 733, 742, 762, 774, 776, 722, 772, 745, 775, 738, - /* 160 */ 780, 753, 748, 783, 784, 792, 785, 803, 779, 787, - /* 170 */ 789, 793, 794, 798, 799, 805, 807, 809, 810, 821, - /* 180 */ 806, 771, 770, 829, 782, 808, 833, 778, 837, 816, - /* 190 */ 811, 838, 788, 754, 812, 814, 756, 815, 817, 820, - /* 200 */ 795, 763, 804, 781, 603, 847, 822, 825, 826, 836, - /* 210 */ 840, 828, 699, 849, 852, 856, 857, 859, 865, 880, - /* 220 */ 860, 912, 895, 920, 888, 901, 915, 928, 926, 943, - /* 230 */ 886, 887, 892, 929, 933, 934, 942, 954, + /* 0 */ 38, 489, 564, 625, 731, 806, 839, 901, 315, 964, + /* 10 */ 1018, 1058, 1077, -223, 1133, 90, 678, 955, 1192, 1246, + /* 20 */ 1276, 1309, 1339, 1369, 1420, 1436, 1483, 1501, 1516, 1563, + /* 30 */ 1579, 1626, 1656, 1706, 1736, 1786, 1795, 1849, -191, 1364, + /* 40 */ 23, 390, 460, -224, 235, -282, 257, -280, 197, -141, + /* 50 */ -112, -244, -240, -317, -202, -190, -243, 200, 327, 361, + /* 60 */ 383, 449, -152, 450, -221, 60, -64, -144, 337, 452, + /* 70 */ 55, 508, 78, 223, 47, 117, 511, -250, -177, -312, + /* 80 */ -312, -312, -113, 245, -34, 261, 298, 304, 354, 357, + /* 90 */ 358, 371, 405, 424, 434, 470, 474, 476, 477, 488, + /* 100 */ 490, 491, 496, 520, 70, -139, 161, 289, 326, 334, + /* 110 */ -277, 149, 475, -114, 394, 486, 252, 170, 267, 370, + /* 120 */ 530, 567, -259, -255, 597, 599, 603, 626, 627, 604, + /* 130 */ 656, 660, 558, 576, 621, 587, 682, 685, 674, 647, + /* 140 */ 623, 623, 623, 679, 609, 615, 616, 640, 679, 734, + /* 150 */ 683, 742, 707, 750, 758, 726, 733, 736, 764, 771, + /* 160 */ 773, 719, 769, 737, 772, 735, 780, 759, 753, 784, + /* 170 */ 796, 804, 801, 810, 789, 790, 792, 793, 803, 808, + /* 180 */ 809, 814, 816, 817, 819, 825, 836, 797, 783, 830, + /* 190 */ 795, 807, 840, 813, 843, 821, 824, 847, 828, 754, + /* 200 */ 833, 820, 755, 837, 832, 845, 761, 763, 774, 788, + /* 210 */ 623, 857, 835, 841, 838, 842, 787, 846, 848, 854, + /* 220 */ 853, 855, 679, 864, 867, 868, 869, 877, 873, 890, + /* 230 */ 875, 918, 906, 921, 897, 904, 920, 932, 930, 939, + /* 240 */ 885, 882, 891, 922, 925, 929, 937, 956, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 10 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 20 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 30 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 40 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 50 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1396, 1327, 1327, - /* 60 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 70 */ 1327, 1327, 1327, 1394, 1534, 1327, 1698, 1327, 1327, 1327, - /* 80 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 90 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 100 */ 1327, 1327, 1327, 1327, 1396, 1327, 1709, 1709, 1709, 1394, - /* 110 */ 1327, 1327, 1327, 1327, 1327, 1327, 1489, 1327, 1327, 1327, - /* 120 */ 1327, 1327, 1327, 1327, 1327, 1572, 1327, 1327, 1774, 1327, - /* 130 */ 1578, 1733, 1327, 1327, 1442, 1725, 1701, 1715, 1702, 1327, - /* 140 */ 1759, 1718, 1327, 1327, 1327, 1327, 1564, 1327, 1327, 1539, - /* 150 */ 1536, 1536, 1327, 1327, 1327, 1327, 1396, 1327, 1396, 1327, - /* 160 */ 1396, 1327, 1327, 1396, 1396, 1327, 1396, 1327, 1327, 1327, - /* 170 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 180 */ 1327, 1327, 1327, 1394, 1574, 1327, 1394, 1327, 1394, 1327, - /* 190 */ 1327, 1394, 1327, 1740, 1738, 1327, 1740, 1738, 1327, 1327, - /* 200 */ 1752, 1748, 1731, 1729, 1715, 1327, 1327, 1327, 1777, 1765, - /* 210 */ 1761, 1327, 1327, 1738, 1327, 1327, 1738, 1327, 1547, 1327, - /* 220 */ 1327, 1394, 1327, 1394, 1327, 1458, 1327, 1327, 1394, 1327, - /* 230 */ 1566, 1580, 1556, 1492, 1492, 1492, 1397, 1332, 1327, 1327, - /* 240 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1454, - /* 250 */ 1643, 1751, 1750, 1674, 1673, 1672, 1670, 1642, 1327, 1327, - /* 260 */ 1327, 1327, 1327, 1327, 1636, 1637, 1635, 1634, 1327, 1327, - /* 270 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 280 */ 1327, 1327, 1327, 1699, 1327, 1762, 1766, 1327, 1327, 1327, - /* 290 */ 1620, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 300 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 310 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 320 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 330 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 340 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 350 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 360 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 370 */ 1327, 1327, 1327, 1361, 1327, 1327, 1327, 1327, 1327, 1327, - /* 380 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 390 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 400 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 410 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1423, 1422, - /* 420 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 430 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 440 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 450 */ 1722, 1732, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 460 */ 1327, 1620, 1327, 1749, 1327, 1708, 1704, 1327, 1327, 1700, - /* 470 */ 1327, 1327, 1760, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 480 */ 1694, 1327, 1667, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 490 */ 1327, 1630, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 500 */ 1327, 1327, 1327, 1327, 1619, 1327, 1658, 1327, 1327, 1327, - /* 510 */ 1327, 1327, 1327, 1327, 1327, 1486, 1327, 1327, 1327, 1327, - /* 520 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1471, 1469, - /* 530 */ 1468, 1467, 1327, 1464, 1327, 1327, 1327, 1327, 1327, 1327, - /* 540 */ 1327, 1327, 1327, 1327, 1327, 1416, 1327, 1327, 1327, 1327, - /* 550 */ 1327, 1327, 1327, 1327, 1327, 1407, 1327, 1406, 1327, 1327, - /* 560 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 570 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 580 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 590 */ 1327, + /* 0 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 10 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 20 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 30 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 40 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 50 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 60 */ 1341, 1341, 1410, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 70 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1408, 1548, 1341, + /* 80 */ 1712, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 90 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 100 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1410, 1341, + /* 110 */ 1723, 1723, 1723, 1408, 1341, 1341, 1341, 1341, 1341, 1341, + /* 120 */ 1503, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1586, + /* 130 */ 1341, 1341, 1789, 1341, 1592, 1747, 1341, 1341, 1456, 1739, + /* 140 */ 1715, 1729, 1716, 1341, 1774, 1774, 1774, 1732, 1341, 1341, + /* 150 */ 1341, 1341, 1578, 1341, 1341, 1553, 1550, 1550, 1341, 1341, + /* 160 */ 1341, 1341, 1410, 1341, 1410, 1341, 1410, 1341, 1341, 1410, + /* 170 */ 1410, 1341, 1410, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 180 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1408, + /* 190 */ 1588, 1341, 1408, 1341, 1408, 1341, 1341, 1408, 1341, 1754, + /* 200 */ 1752, 1341, 1754, 1752, 1341, 1341, 1766, 1762, 1745, 1743, + /* 210 */ 1729, 1341, 1341, 1341, 1780, 1776, 1792, 1780, 1776, 1780, + /* 220 */ 1776, 1341, 1341, 1752, 1341, 1341, 1752, 1341, 1561, 1341, + /* 230 */ 1341, 1408, 1341, 1408, 1341, 1472, 1341, 1341, 1408, 1341, + /* 240 */ 1580, 1594, 1570, 1506, 1506, 1506, 1411, 1346, 1341, 1341, + /* 250 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1468, + /* 260 */ 1657, 1765, 1764, 1688, 1687, 1686, 1684, 1656, 1341, 1341, + /* 270 */ 1341, 1341, 1341, 1341, 1650, 1651, 1649, 1648, 1341, 1341, + /* 280 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 290 */ 1341, 1341, 1341, 1713, 1341, 1777, 1781, 1341, 1341, 1341, + /* 300 */ 1634, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 310 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 320 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 330 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 340 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 350 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 360 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 370 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 380 */ 1341, 1341, 1341, 1375, 1341, 1341, 1341, 1341, 1341, 1341, + /* 390 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 400 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 410 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 420 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1437, 1436, + /* 430 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 440 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 450 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 460 */ 1736, 1746, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 470 */ 1341, 1634, 1341, 1763, 1341, 1722, 1718, 1341, 1341, 1714, + /* 480 */ 1341, 1341, 1775, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 490 */ 1341, 1341, 1708, 1341, 1681, 1341, 1341, 1341, 1341, 1341, + /* 500 */ 1341, 1341, 1341, 1644, 1341, 1341, 1341, 1341, 1341, 1341, + /* 510 */ 1341, 1341, 1341, 1341, 1341, 1341, 1633, 1341, 1672, 1341, + /* 520 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1500, 1341, 1341, + /* 530 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 540 */ 1485, 1483, 1482, 1481, 1341, 1478, 1341, 1341, 1341, 1341, + /* 550 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1430, 1341, 1341, + /* 560 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1421, 1341, 1420, + /* 570 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 580 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 590 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, + /* 600 */ 1341, 1341, 1341, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1937,27 +1946,28 @@ static const char *const yyRuleName[] = { /* 426 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", /* 427 */ "query_expression_body ::= query_expression_body UNION query_expression_body", /* 428 */ "query_primary ::= query_specification", - /* 429 */ "order_by_clause_opt ::=", - /* 430 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 431 */ "slimit_clause_opt ::=", - /* 432 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 433 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 434 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 435 */ "limit_clause_opt ::=", - /* 436 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 437 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 438 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 439 */ "subquery ::= NK_LP query_expression NK_RP", - /* 440 */ "search_condition ::= common_expression", - /* 441 */ "sort_specification_list ::= sort_specification", - /* 442 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 443 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 444 */ "ordering_specification_opt ::=", - /* 445 */ "ordering_specification_opt ::= ASC", - /* 446 */ "ordering_specification_opt ::= DESC", - /* 447 */ "null_ordering_opt ::=", - /* 448 */ "null_ordering_opt ::= NULLS FIRST", - /* 449 */ "null_ordering_opt ::= NULLS LAST", + /* 429 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP", + /* 430 */ "order_by_clause_opt ::=", + /* 431 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 432 */ "slimit_clause_opt ::=", + /* 433 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 434 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 435 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 436 */ "limit_clause_opt ::=", + /* 437 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 438 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 439 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 440 */ "subquery ::= NK_LP query_expression NK_RP", + /* 441 */ "search_condition ::= common_expression", + /* 442 */ "sort_specification_list ::= sort_specification", + /* 443 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 444 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 445 */ "ordering_specification_opt ::=", + /* 446 */ "ordering_specification_opt ::= ASC", + /* 447 */ "ordering_specification_opt ::= DESC", + /* 448 */ "null_ordering_opt ::=", + /* 449 */ "null_ordering_opt ::= NULLS FIRST", + /* 450 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2979,27 +2989,28 @@ static const struct { { 349, -4 }, /* (426) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ { 349, -3 }, /* (427) query_expression_body ::= query_expression_body UNION query_expression_body */ { 353, -1 }, /* (428) query_primary ::= query_specification */ - { 350, 0 }, /* (429) order_by_clause_opt ::= */ - { 350, -3 }, /* (430) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 351, 0 }, /* (431) slimit_clause_opt ::= */ - { 351, -2 }, /* (432) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 351, -4 }, /* (433) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 351, -4 }, /* (434) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 352, 0 }, /* (435) limit_clause_opt ::= */ - { 352, -2 }, /* (436) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 352, -4 }, /* (437) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 352, -4 }, /* (438) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 314, -3 }, /* (439) subquery ::= NK_LP query_expression NK_RP */ - { 335, -1 }, /* (440) search_condition ::= common_expression */ - { 354, -1 }, /* (441) sort_specification_list ::= sort_specification */ - { 354, -3 }, /* (442) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 355, -3 }, /* (443) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 356, 0 }, /* (444) ordering_specification_opt ::= */ - { 356, -1 }, /* (445) ordering_specification_opt ::= ASC */ - { 356, -1 }, /* (446) ordering_specification_opt ::= DESC */ - { 357, 0 }, /* (447) null_ordering_opt ::= */ - { 357, -2 }, /* (448) null_ordering_opt ::= NULLS FIRST */ - { 357, -2 }, /* (449) null_ordering_opt ::= NULLS LAST */ + { 353, -6 }, /* (429) query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ + { 350, 0 }, /* (430) order_by_clause_opt ::= */ + { 350, -3 }, /* (431) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 351, 0 }, /* (432) slimit_clause_opt ::= */ + { 351, -2 }, /* (433) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 351, -4 }, /* (434) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 351, -4 }, /* (435) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 352, 0 }, /* (436) limit_clause_opt ::= */ + { 352, -2 }, /* (437) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 352, -4 }, /* (438) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 352, -4 }, /* (439) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 314, -3 }, /* (440) subquery ::= NK_LP query_expression NK_RP */ + { 335, -1 }, /* (441) search_condition ::= common_expression */ + { 354, -1 }, /* (442) sort_specification_list ::= sort_specification */ + { 354, -3 }, /* (443) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 355, -3 }, /* (444) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 356, 0 }, /* (445) ordering_specification_opt ::= */ + { 356, -1 }, /* (446) ordering_specification_opt ::= ASC */ + { 356, -1 }, /* (447) ordering_specification_opt ::= DESC */ + { 357, 0 }, /* (448) null_ordering_opt ::= */ + { 357, -2 }, /* (449) null_ordering_opt ::= NULLS FIRST */ + { 357, -2 }, /* (450) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3414,7 +3425,7 @@ static YYACTIONTYPE yy_reduce( case 286: /* literal_list ::= signed_literal */ yytestcase(yyruleno==286); case 338: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==338); case 393: /* select_sublist ::= select_item */ yytestcase(yyruleno==393); - case 441: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==441); + case 442: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==442); { yylhsminor.yy60 = createNodeList(pCxt, yymsp[0].minor.yy172); } yymsp[0].minor.yy60 = yylhsminor.yy60; break; @@ -3426,7 +3437,7 @@ static YYACTIONTYPE yy_reduce( case 287: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==287); case 339: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==339); case 394: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==394); - case 442: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==442); + case 443: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==443); { yylhsminor.yy60 = addNodeToList(pCxt, yymsp[-2].minor.yy60, yymsp[0].minor.yy172); } yymsp[-2].minor.yy60 = yylhsminor.yy60; break; @@ -3509,7 +3520,7 @@ static YYACTIONTYPE yy_reduce( case 159: /* tags_def_opt ::= */ yytestcase(yyruleno==159); case 401: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==401); case 418: /* group_by_clause_opt ::= */ yytestcase(yyruleno==418); - case 429: /* order_by_clause_opt ::= */ yytestcase(yyruleno==429); + case 430: /* order_by_clause_opt ::= */ yytestcase(yyruleno==430); { yymsp[1].minor.yy60 = NULL; } break; case 129: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ @@ -3750,8 +3761,8 @@ static YYACTIONTYPE yy_reduce( case 408: /* sliding_opt ::= */ yytestcase(yyruleno==408); case 410: /* fill_opt ::= */ yytestcase(yyruleno==410); case 422: /* having_clause_opt ::= */ yytestcase(yyruleno==422); - case 431: /* slimit_clause_opt ::= */ yytestcase(yyruleno==431); - case 435: /* limit_clause_opt ::= */ yytestcase(yyruleno==435); + case 432: /* slimit_clause_opt ::= */ yytestcase(yyruleno==432); + case 436: /* limit_clause_opt ::= */ yytestcase(yyruleno==436); { yymsp[1].minor.yy172 = NULL; } break; case 207: /* like_pattern_opt ::= LIKE NK_STRING */ @@ -4001,7 +4012,7 @@ static YYACTIONTYPE yy_reduce( case 285: /* signed_literal ::= literal_func */ yytestcase(yyruleno==285); case 340: /* star_func_para ::= expression */ yytestcase(yyruleno==340); case 395: /* select_item ::= common_expression */ yytestcase(yyruleno==395); - case 440: /* search_condition ::= common_expression */ yytestcase(yyruleno==440); + case 441: /* search_condition ::= common_expression */ yytestcase(yyruleno==441); { yylhsminor.yy172 = releaseRawExprNode(pCxt, yymsp[0].minor.yy172); } yymsp[0].minor.yy172 = yylhsminor.yy172; break; @@ -4295,7 +4306,7 @@ static YYACTIONTYPE yy_reduce( break; case 402: /* partition_by_clause_opt ::= PARTITION BY expression_list */ case 419: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==419); - case 430: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==430); + case 431: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==431); { yymsp[-2].minor.yy60 = yymsp[0].minor.yy60; } break; case 404: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ @@ -4358,42 +4369,48 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy172 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy172, yymsp[0].minor.yy172); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 432: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 436: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==436); + case 429: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ +{ yymsp[-5].minor.yy172 = yymsp[-4].minor.yy172; } + yy_destructor(yypParser,350,&yymsp[-3].minor); + yy_destructor(yypParser,351,&yymsp[-2].minor); + yy_destructor(yypParser,352,&yymsp[-1].minor); + break; + case 433: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 437: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==437); { yymsp[-1].minor.yy172 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 433: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 437: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==437); + case 434: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 438: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==438); { yymsp[-3].minor.yy172 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 434: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 438: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==438); + case 435: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 439: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==439); { yymsp[-3].minor.yy172 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 439: /* subquery ::= NK_LP query_expression NK_RP */ + case 440: /* subquery ::= NK_LP query_expression NK_RP */ { yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy172); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 443: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + case 444: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ { yylhsminor.yy172 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), yymsp[-1].minor.yy14, yymsp[0].minor.yy17); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 444: /* ordering_specification_opt ::= */ + case 445: /* ordering_specification_opt ::= */ { yymsp[1].minor.yy14 = ORDER_ASC; } break; - case 445: /* ordering_specification_opt ::= ASC */ + case 446: /* ordering_specification_opt ::= ASC */ { yymsp[0].minor.yy14 = ORDER_ASC; } break; - case 446: /* ordering_specification_opt ::= DESC */ + case 447: /* ordering_specification_opt ::= DESC */ { yymsp[0].minor.yy14 = ORDER_DESC; } break; - case 447: /* null_ordering_opt ::= */ + case 448: /* null_ordering_opt ::= */ { yymsp[1].minor.yy17 = NULL_ORDER_DEFAULT; } break; - case 448: /* null_ordering_opt ::= NULLS FIRST */ + case 449: /* null_ordering_opt ::= NULLS FIRST */ { yymsp[-1].minor.yy17 = NULL_ORDER_FIRST; } break; - case 449: /* null_ordering_opt ::= NULLS LAST */ + case 450: /* null_ordering_opt ::= NULLS LAST */ { yymsp[-1].minor.yy17 = NULL_ORDER_LAST; } break; default: diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 5a385ba25e..0ba062ebe4 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -232,4 +232,12 @@ TEST_F(ParserSelectTest, semanticError) { PARSER_STAGE_TRANSLATE); } +TEST_F(ParserSelectTest, setOperator) { + useDb("root", "test"); + + run("SELECT * FROM t1 UNION ALL SELECT * FROM t1"); + + run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)"); +} + } // namespace ParserTest diff --git a/source/libs/planner/test/planSubqueryTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp index 185f55db10..ab563fd7a0 100644 --- a/source/libs/planner/test/planSubqueryTest.cpp +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -23,12 +23,15 @@ class PlanSubqeuryTest : public PlannerTestBase {}; TEST_F(PlanSubqeuryTest, basic) { useDb("root", "test"); - run("select * from (select * from t1)"); + run("SELECT * FROM (SELECT * FROM t1)"); + + run("SELECT LAST(c1) FROM ( SELECT * FROM t1)"); } TEST_F(PlanSubqeuryTest, doubleGroupBy) { useDb("root", "test"); - run("select count(*) from (select c1 + c3 a, c1 + count(*) b from t1 where c2 = 'abc' group by c1, c3) where a > 100 " - "group by b"); + run("SELECT COUNT(*) FROM (" + "SELECT c1 + c3 a, c1 + COUNT(*) b FROM t1 WHERE c2 = 'abc' GROUP BY c1, c3) " + "WHERE a > 100 GROUP BY b"); } From e9042e709d7c287f713f0745bfcbf317ec1ba3d9 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 14 May 2022 10:36:19 +0800 Subject: [PATCH 12/71] fix: 'create table using' problem --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c86c1ac2e9..cbfbd9c2aa 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3842,7 +3842,7 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS if (pVal->node.resType.type == TSDB_DATA_TYPE_NULL) { // todo } else { - tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), + tdAddColToKVRow(pBuilder, pSchema->colId, nodesGetValueFromNode(pVal), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); } From 61729d6805386b82f1baced563a1c36185fcb6c8 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 14 May 2022 10:43:00 +0800 Subject: [PATCH 13/71] fix(os): update windows compile error --- include/os/osString.h | 3 +++ source/client/src/clientSml.c | 4 ++-- source/os/src/osString.c | 23 +++++++++++++++++++++++ tools/shell/src/shellEngine.c | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/include/os/osString.h b/include/os/osString.h index 026cb33ad9..5f65f97bec 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -37,6 +37,7 @@ typedef int32_t TdUcs4; #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID #define wchar_t WCHAR_T_TYPE_TAOS_FORBID + #define strcasestr STR_CASE_STR_FORBID #endif #ifdef WINDOWS @@ -69,6 +70,8 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size); int32_t taosWcharToMb(char *pStr, TdWchar wchar); int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size); +char *taosStrCaseStr(const char *str, const char *pattern); + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 605d71ffed..47ac15a141 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -439,7 +439,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_INVALID_STB) { - SSchemaAction schemaAction = { .action = SCHEMA_ACTION_CREATE_STABLE, .createSTable = {0}}; + SSchemaAction schemaAction = { SCHEMA_ACTION_CREATE_STABLE, {0}}; memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); schemaAction.createSTable.tags = sTableData->tags; schemaAction.createSTable.fields = sTableData->cols; @@ -455,7 +455,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); } - SSchemaAction schemaAction = {.alterSTable = {0}}; + SSchemaAction schemaAction = { 0 }; memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &schemaAction, true); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 375c5001f4..357de4417b 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -236,3 +236,26 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { ret int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); } int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); } + +char *taosStrCaseStr(const char *str, const char *pattern) { +#ifdef WINDOWS + size_t i; + + if (!*pattern) + return (char*)str; + + for (; *str; str++) { + if (toupper(*str) == toupper(*pattern)) { + for (i = 1;; i++) { + if (!pattern[i]) + return (char*)str; + if (toupper(str[i]) != toupper(pattern[i])) + break; + } + } + } + return NULL; +#else + return strcasestr(str, pattern); +#endif +} \ No newline at end of file diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1e832c0c46..39b97004ff 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -509,7 +509,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t bool shellIsLimitQuery(const char *sql) { //todo refactor - if (strcasestr(sql, " limit ") != NULL) { + if (taosStrCaseStr(sql, " limit ") != NULL) { return true; } From 2e3a64a569ba640653cd3bc9f68c10aab47c6acd Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 14 May 2022 11:15:21 +0800 Subject: [PATCH 14/71] fix(os): update windows compile error --- source/os/src/osString.c | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 357de4417b..7dbd301913 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -238,24 +238,20 @@ int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); } int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); } char *taosStrCaseStr(const char *str, const char *pattern) { -#ifdef WINDOWS - size_t i; + size_t i; - if (!*pattern) - return (char*)str; + if (!*pattern) + return (char*)str; - for (; *str; str++) { - if (toupper(*str) == toupper(*pattern)) { - for (i = 1;; i++) { - if (!pattern[i]) - return (char*)str; - if (toupper(str[i]) != toupper(pattern[i])) - break; - } - } + for (; *str; str++) { + if (toupper(*str) == toupper(*pattern)) { + for (i = 1;; i++) { + if (!pattern[i]) + return (char*)str; + if (toupper(str[i]) != toupper(pattern[i])) + break; } - return NULL; -#else - return strcasestr(str, pattern); -#endif + } + } + return NULL; } \ No newline at end of file From 38dfa18750815e5690b8c5d703b4a5725ce53739 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 14 May 2022 11:18:50 +0800 Subject: [PATCH 15/71] fix(os): update windows compile error --- source/client/src/clientSml.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 47ac15a141..fc909cb0a3 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -439,7 +439,9 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_INVALID_STB) { - SSchemaAction schemaAction = { SCHEMA_ACTION_CREATE_STABLE, {0}}; + SSchemaAction schemaAction; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); schemaAction.createSTable.tags = sTableData->tags; schemaAction.createSTable.fields = sTableData->cols; @@ -455,7 +457,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); } - SSchemaAction schemaAction = { 0 }; + SSchemaAction schemaAction; + memset(&schemaAction, 0, sizeof(SSchemaAction)); memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &schemaAction, true); if (code != TSDB_CODE_SUCCESS) { From 7ad96ac276424b3dc49be61f563b22756f5ef96d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 14 May 2022 11:53:46 +0800 Subject: [PATCH 16/71] enh: control the memory of the rpc queue --- include/common/tglobal.h | 1 - include/util/tprocess.h | 3 +- include/util/tqueue.h | 12 ++- source/client/src/tmq.c | 10 +-- source/common/src/tglobal.c | 2 - source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 2 +- source/dnode/mgmt/mgmt_qnode/src/qmWorker.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/sync/src/syncIO.c | 6 +- source/libs/transport/test/pushServer.c | 2 +- source/libs/transport/test/rserver.c | 2 +- source/util/src/tprocess.c | 4 +- source/util/src/tqueue.c | 74 +++++++++++-------- 15 files changed, 70 insertions(+), 56 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 94b6ca551b..83283cfdad 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -69,7 +69,6 @@ extern int32_t tsNumOfQnodeFetchThreads; extern int32_t tsNumOfSnodeSharedThreads; extern int32_t tsNumOfSnodeUniqueThreads; extern int64_t tsRpcQueueMemoryAllowed; -extern int64_t tsRpcQueueMemoryUsed; // monitor extern bool tsEnableMonitor; diff --git a/include/util/tprocess.h b/include/util/tprocess.h index 7e1767441e..5e5a982ec4 100644 --- a/include/util/tprocess.h +++ b/include/util/tprocess.h @@ -17,6 +17,7 @@ #define _TD_UTIL_PROCESS_H_ #include "os.h" +#include "tqueue.h" #ifdef __cplusplus extern "C" { @@ -25,7 +26,7 @@ extern "C" { typedef enum { PROC_FUNC_REQ = 1, PROC_FUNC_RSP, PROC_FUNC_REGIST, PROC_FUNC_RELEASE } EProcFuncType; typedef struct SProcObj SProcObj; -typedef void *(*ProcMallocFp)(int32_t contLen); +typedef void *(*ProcMallocFp)(int32_t contLen, EQItype itype); typedef void *(*ProcFreeFp)(void *pCont); typedef void (*ProcConsumeFp)(void *parent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen, EProcFuncType ftype); diff --git a/include/util/tqueue.h b/include/util/tqueue.h index fc02ea2491..dbc4d03177 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -48,18 +48,24 @@ typedef struct { int32_t threadNum; } SQueueInfo; +typedef enum { + DEF_QITEM = 0, + RPC_QITEM = 1, +} EQItype; + typedef void (*FItem)(SQueueInfo *pInfo, void *pItem); typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems); STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); -void *taosAllocateQitem(int32_t size); +void *taosAllocateQitem(int32_t size, EQItype itype); void taosFreeQitem(void *pItem); void taosWriteQitem(STaosQueue *queue, void *pItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem); bool taosQueueEmpty(STaosQueue *queue); int32_t taosQueueItemSize(STaosQueue *queue); +int64_t taosQueueMemorySize(STaosQueue *queue); STaosQall *taosAllocateQall(); void taosFreeQall(STaosQall *qall); @@ -77,8 +83,8 @@ int32_t taosGetQueueNumber(STaosQset *qset); int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp); void taosResetQsetThread(STaosQset *qset, void *pItem); -int32_t taosGetQueueItemsNumber(STaosQueue *queue); -int32_t taosGetQsetItemsNumber(STaosQset *qset); + +extern int64_t tsRpcQueueMemoryAllowed; #ifdef __cplusplus } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 639f00ab86..2c3f64c5c1 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -316,7 +316,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { void tmqAssignDelayedHbTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t)); + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); *pTaskType = TMQ_DELAYED_TASK__HB; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); @@ -324,7 +324,7 @@ void tmqAssignDelayedHbTask(void* param, void* tmrId) { void tmqAssignDelayedCommitTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t)); + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); *pTaskType = TMQ_DELAYED_TASK__COMMIT; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); @@ -332,7 +332,7 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { void tmqAssignDelayedReportTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t)); + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); *pTaskType = TMQ_DELAYED_TASK__REPORT; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); @@ -848,7 +848,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); } - SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper)); + SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch); goto CREATE_MSG_FAIL; @@ -987,7 +987,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tmqUpdateEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { - SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper)); + SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM); if (pWrapper == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 338a406eec..025ad0ca0d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -61,8 +61,6 @@ int32_t tsNumOfQnodeQueryThreads = 2; int32_t tsNumOfQnodeFetchThreads = 2; int32_t tsNumOfSnodeSharedThreads = 2; int32_t tsNumOfSnodeUniqueThreads = 2; -int64_t tsRpcQueueMemoryAllowed = 0; -int64_t tsRpcQueueMemoryUsed = 0; // monitor bool tsEnableMonitor = true; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index f55e02e50a..622b8332fc 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -104,7 +104,7 @@ int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { } static inline int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) { - SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg), RPC_QITEM); if (pMsg == NULL) return -1; dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 40b91cb93e..daac7f80bb 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -102,7 +102,7 @@ int32_t qmPutNodeMsgToMonitorQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { } static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { - SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg), RPC_QITEM); if (pMsg == NULL) { return -1; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 1f671179b6..d6e99c0899 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -326,7 +326,7 @@ static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType q SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) return -1; - SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg), RPC_QITEM); int32_t code = 0; if (pMsg != NULL) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 30ec04d9ef..9f350409c8 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -79,7 +79,7 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe needRelease = true; if ((msgFp = dmGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; - if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; + if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg), RPC_QITEM)) == NULL) goto _OVER; if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER; if (pWrapper->procType != DND_PROC_PARENT) { diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 1d371d43f6..fa9e27a5f8 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -124,7 +124,7 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf)); + SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM); if (NULL == pBuf || !allocBuf(pDispatcher, pInput, pBuf)) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index deb158cbae..a3d1717c3b 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -92,7 +92,7 @@ int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) { syncRpcMsgLog2((char *)"==syncIOEqMsg==", pMsg); SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); STaosQueue *pMsgQ = queue; @@ -360,7 +360,7 @@ static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { syncRpcMsgLog2((char *)"==syncIOProcessRequest==", pMsg); SSyncIO *io = pParent; SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); taosWriteQitem(io->pMsgQ, pTemp); } @@ -420,7 +420,7 @@ static void syncIOTickQ(void *param, void *tmrId) { SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); syncRpcMsgLog2((char *)"==syncIOTickQ==", &rpcMsg); taosWriteQitem(io->pMsgQ, pTemp); diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 3099998f57..d8d78c2842 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -114,7 +114,7 @@ int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 14d109dc5a..80785340d1 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -103,7 +103,7 @@ int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 8963a4f94e..8b4fd235fd 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -258,8 +258,8 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea int16_t headLen = CEIL8(rawHeadLen); int32_t bodyLen = CEIL8(rawBodyLen); - void *pHead = (*mallocHeadFp)(headLen); - void *pBody = (*mallocBodyFp)(bodyLen); + void *pHead = (*mallocHeadFp)(headLen, RPC_QITEM); + void *pBody = (*mallocBodyFp)(bodyLen, RPC_QITEM); if (pHead == NULL || pBody == NULL) { taosThreadMutexUnlock(&pQueue->mutex); tsem_post(&pQueue->sem); diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index eca529c632..17a87b1a5f 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -18,20 +18,21 @@ #include "taoserror.h" #include "tlog.h" +int64_t tsRpcQueueMemoryAllowed = 0; +int64_t tsRpcQueueMemoryUsed = 0; + typedef struct STaosQnode STaosQnode; typedef struct STaosQnode { STaosQnode *next; STaosQueue *queue; int32_t size; - int32_t reserved; + int8_t itype; + int8_t reserved[3]; char item[]; } STaosQnode; typedef struct STaosQueue { - int64_t memOfItems; - int32_t numOfItems; - int32_t threadId; STaosQnode *head; STaosQnode *tail; STaosQueue *next; // for queue set @@ -40,15 +41,17 @@ typedef struct STaosQueue { FItem itemFp; FItems itemsFp; TdThreadMutex mutex; + int64_t memOfItems; + int32_t numOfItems; } STaosQueue; typedef struct STaosQset { STaosQueue *head; STaosQueue *current; TdThreadMutex mutex; + tsem_t sem; int32_t numOfQueues; int32_t numOfItems; - tsem_t sem; } STaosQset; typedef struct STaosQall { @@ -120,6 +123,8 @@ bool taosQueueEmpty(STaosQueue *queue) { } int32_t taosQueueItemSize(STaosQueue *queue) { + if (queue == NULL) return 0; + taosThreadMutexLock(&queue->mutex); int32_t numOfItems = queue->numOfItems; taosThreadMutexUnlock(&queue->mutex); @@ -127,32 +132,51 @@ int32_t taosQueueItemSize(STaosQueue *queue) { } int64_t taosQueueMemorySize(STaosQueue *queue) { + if (queue == NULL) return 0; + taosThreadMutexLock(&queue->mutex); int64_t memOfItems = queue->memOfItems; taosThreadMutexUnlock(&queue->mutex); return memOfItems; } -void *taosAllocateQitem(int32_t size) { +void *taosAllocateQitem(int32_t size, EQItype itype) { STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); pNode->size = size; + pNode->itype = itype; if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - uTrace("item:%p, node:%p is allocated", pNode->item, pNode); + if (itype == RPC_QITEM) { + int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size); + if (alloced > tsRpcQueueMemoryUsed) { + taosMemoryFree(pNode); + terrno = TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE; + return NULL; + } + uTrace("item:%p, node:%p is allocated, alloc:%" PRId64, pNode->item, pNode, alloced); + } else { + uTrace("item:%p, node:%p is allocated", pNode->item, pNode); + } + return (void *)pNode->item; } void taosFreeQitem(void *pItem) { if (pItem == NULL) return; - char *temp = pItem; - temp -= sizeof(STaosQnode); - uTrace("item:%p, node:%p is freed", pItem, temp); - taosMemoryFree(temp); + STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); + if (pNode->itype > 0) { + int64_t alloced = atomic_sub_fetch_64(&tsRpcQueueMemoryUsed, pNode->size); + uTrace("item:%p, node:%p is freed, alloc:%" PRId64, pItem, pNode, alloced); + } else { + uTrace("item:%p, node:%p is freed", pItem, pNode); + } + + taosMemoryFree(pNode); } void taosWriteQitem(STaosQueue *queue, void *pItem) { @@ -203,7 +227,13 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { return code; } -STaosQall *taosAllocateQall() { return taosMemoryCalloc(1, sizeof(STaosQall)); } +STaosQall *taosAllocateQall() { + STaosQall *qall = taosMemoryCalloc(1, sizeof(STaosQall)); + if (qall != NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return qall; +} void taosFreeQall(STaosQall *qall) { taosMemoryFree(qall); } @@ -458,23 +488,3 @@ void taosResetQsetThread(STaosQset *qset, void *pItem) { } taosThreadMutexUnlock(&qset->mutex); } - -int32_t taosGetQueueItemsNumber(STaosQueue *queue) { - if (!queue) return 0; - - int32_t num; - taosThreadMutexLock(&queue->mutex); - num = queue->numOfItems; - taosThreadMutexUnlock(&queue->mutex); - return num; -} - -int32_t taosGetQsetItemsNumber(STaosQset *qset) { - if (!qset) return 0; - - int32_t num = 0; - taosThreadMutexLock(&qset->mutex); - num = qset->numOfItems; - taosThreadMutexUnlock(&qset->mutex); - return num; -} From f0df0da98fa750fbb4b789fce941c3fbaa4919b8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 14 May 2022 11:59:23 +0800 Subject: [PATCH 17/71] fix: some problems of parser and planner --- source/libs/parser/src/parTranslater.c | 63 ++++++++++++++----- source/libs/planner/src/planOptimizer.c | 6 +- source/libs/planner/test/planSubqueryTest.cpp | 2 +- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 425696bb28..bbefaa468d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -480,8 +480,8 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { return res; } -static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { - uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : pVal->node.resType.precision); +static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SDataType targetDt) { + uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : targetDt.precision); pVal->node.resType.precision = precision; if (pVal->placeholderNo > 0) { return DEAL_RES_CONTINUE; @@ -493,7 +493,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { } *(int64_t*)&pVal->typeData = pVal->datum.i; } else { - switch (pVal->node.resType.type) { + switch (targetDt.type) { case TSDB_DATA_TYPE_NULL: break; case TSDB_DATA_TYPE_BOOL: @@ -562,27 +562,41 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { } case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pVal->datum.p) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); } - varDataSetLen(pVal->datum.p, pVal->node.resType.bytes); - strncpy(varDataVal(pVal->datum.p), pVal->literal, pVal->node.resType.bytes); + varDataSetLen(pVal->datum.p, targetDt.bytes); + strncpy(varDataVal(pVal->datum.p), pVal->literal, targetDt.bytes); break; } case TSDB_DATA_TYPE_TIMESTAMP: { - if (taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes, precision, tsDaylight) != - TSDB_CODE_SUCCESS) { + if (taosParseTime(pVal->literal, &pVal->datum.i, targetDt.bytes, precision, tsDaylight) != TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(int64_t*)&pVal->typeData = pVal->datum.i; break; } - case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_NCHAR: { + targetDt.bytes *= TSDB_NCHAR_SIZE; + pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + VARSTR_HEADER_SIZE + 1); + if (NULL == pVal->datum.p) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); + ; + } + + int32_t output = 0; + if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), targetDt.bytes, + &output)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } + varDataSetLen(pVal->datum.p, output); + break; + } case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: - // todo + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); default: break; } @@ -591,6 +605,10 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { return DEAL_RES_CONTINUE; } +static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { + return translateValueImpl(pCxt, pVal, pVal->node.resType); +} + static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (nodesIsUnaryOp(pOp)) { if (OP_TYPE_MINUS == pOp->opType) { @@ -636,6 +654,9 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType; } if (nodesIsRegularOp(pOp)) { + if (!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + } if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || !IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } @@ -3847,7 +3868,7 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS if (pVal->node.resType.type == TSDB_DATA_TYPE_NULL) { // todo } else { - tdAddColToKVRow(pBuilder, pSchema->colId, nodesGetValueFromNode(pVal->datum.p), + tdAddColToKVRow(pBuilder, pSchema->colId, nodesGetValueFromNode(pVal), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); } @@ -3861,11 +3882,23 @@ static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* p return scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal); } -static int32_t translateTagVal(STranslateContext* pCxt, SNode* pNode, SValueNode** pVal) { +static SDataType schemaToDataType(SSchema* pSchema) { + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = 0, .scale = 0}; + if (TSDB_DATA_TYPE_VARCHAR == dt.type || TSDB_DATA_TYPE_BINARY == dt.type || TSDB_DATA_TYPE_VARBINARY == dt.type) { + dt.bytes -= VARSTR_HEADER_SIZE; + } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { + dt.bytes = (dt.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } + return dt; +} + +static int32_t translateTagVal(STranslateContext* pCxt, SSchema* pSchema, SNode* pNode, SValueNode** pVal) { if (QUERY_NODE_FUNCTION == nodeType(pNode)) { return createValueFromFunction(pCxt, (SFunctionNode*)pNode, pVal); } else if (QUERY_NODE_VALUE == nodeType(pNode)) { - return (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pNode) ? pCxt->errCode : TSDB_CODE_SUCCESS); + return (DEAL_RES_ERROR == translateValueImpl(pCxt, (SValueNode*)pNode, schemaToDataType(pSchema)) + ? pCxt->errCode + : TSDB_CODE_SUCCESS); } else { return TSDB_CODE_FAILED; } @@ -3894,7 +3927,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName); } SValueNode* pVal = NULL; - int32_t code = translateTagVal(pCxt, pNode, &pVal); + int32_t code = translateTagVal(pCxt, pSchema, pNode, &pVal); if (TSDB_CODE_SUCCESS == code) { if (NULL == pVal) { pVal = (SValueNode*)pNode; @@ -3924,7 +3957,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau int32_t index = 0; FOREACH(pNode, pStmt->pValsOfTags) { SValueNode* pVal = NULL; - int32_t code = translateTagVal(pCxt, pNode, &pVal); + int32_t code = translateTagVal(pCxt, pTagSchema + index, pNode, &pVal); if (TSDB_CODE_SUCCESS == code) { if (NULL == pVal) { pVal = (SValueNode*)pNode; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 6ae1b23b97..9968f63c5d 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -723,7 +723,10 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - return nodesListMakeAppend(pScanNodes, pNode); + if (TSDB_SUPER_TABLE != ((SScanLogicNode*)pNode)->pMeta->tableType) { + return nodesListMakeAppend(pScanNodes, pNode); + } + break; case QUERY_NODE_LOGIC_PLAN_JOIN: code = opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); if (TSDB_CODE_SUCCESS == code) { @@ -739,6 +742,7 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL if (1 != LIST_LENGTH(pNode->pChildren)) { *pNotOptimize = true; + return TSDB_CODE_SUCCESS; } return opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); diff --git a/source/libs/planner/test/planSubqueryTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp index ab563fd7a0..f45cbc6f8f 100644 --- a/source/libs/planner/test/planSubqueryTest.cpp +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -25,7 +25,7 @@ TEST_F(PlanSubqeuryTest, basic) { run("SELECT * FROM (SELECT * FROM t1)"); - run("SELECT LAST(c1) FROM ( SELECT * FROM t1)"); + // run("SELECT LAST(c1) FROM ( SELECT * FROM t1)"); } TEST_F(PlanSubqeuryTest, doubleGroupBy) { From 84018cdbd245e1e2f873357af3cfd29a75867713 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 14 May 2022 12:21:57 +0800 Subject: [PATCH 18/71] fix: some problems of parser and planner --- source/libs/parser/src/parTranslater.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bbefaa468d..c7f0c13de0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -578,15 +578,15 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD break; } case TSDB_DATA_TYPE_NCHAR: { - targetDt.bytes *= TSDB_NCHAR_SIZE; - pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + VARSTR_HEADER_SIZE + 1); + int32_t bytes = targetDt.bytes * TSDB_NCHAR_SIZE; + pVal->datum.p = taosMemoryCalloc(1, bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pVal->datum.p) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); ; } int32_t output = 0; - if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), targetDt.bytes, + if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), bytes, &output)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } @@ -601,6 +601,7 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD break; } } + pVal->node.resType = targetDt; pVal->translate = true; return DEAL_RES_CONTINUE; } From 81e48a9e2307cd2953ac1d742c8736eefcc581e1 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 14 May 2022 12:25:29 +0800 Subject: [PATCH 19/71] fix(os): 3.0 TDengine package error --- packaging/release.sh | 8 ++++---- source/libs/tdb/CMakeLists.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index a56d991bf8..1077eb569a 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -63,12 +63,12 @@ cp ${install_files} ${install_dir} header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h" cp ${header_files} ${install_dir}/inc -bin_files="${compile_dir}/source/dnode/mgmt/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table ${compile_dir}/tests/test/c/tmq_sim ${script_dir}/remove.sh ${compile_dir}/build/bin/taosBenchmark ${compile_dir}/build/bin/taosdump" +bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos ${compile_dir}/build/bin/create_table ${compile_dir}/build/bin/tmq_sim ${script_dir}/remove.sh ${compile_dir}/build/bin/taosBenchmark ${compile_dir}/build/bin/taosdump" cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : -cp -rf ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/ -cp -rf ${compile_dir}/source/libs/tdb/libtdb.so ${install_dir}/lib/ -cp -rf ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries" +cp ${compile_dir}/build/lib/libtaos.so ${install_dir}/lib/ +cp ${compile_dir}/build/lib/libtdb.so ${install_dir}/lib/ +cp ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries" cp -rf ${compile_dir}/build/lib/pkgconfig ${install_dir}/lib/ > /dev/null || echo -e "failed to copy pkgconfig directory" diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 01490030f2..722f6bddef 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ # tdb -add_library(tdb STATIC "") +add_library(tdb SHARED "") target_sources(tdb PRIVATE "src/db/tdbPCache.c" From ec6efecf9f4292b096cd90778eda7b043b234861 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 14 May 2022 14:05:45 +0800 Subject: [PATCH 20/71] test: add error case for rerun fail --- tests/system-test/99-TDcase/TD-15563.py | 186 ++++++++++++------------ 1 file changed, 89 insertions(+), 97 deletions(-) diff --git a/tests/system-test/99-TDcase/TD-15563.py b/tests/system-test/99-TDcase/TD-15563.py index b8d3abca5c..5931360b90 100644 --- a/tests/system-test/99-TDcase/TD-15563.py +++ b/tests/system-test/99-TDcase/TD-15563.py @@ -49,7 +49,38 @@ class TDTestCase: print(cur) return cur - def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg,showRow,cdbName,valgrind=0): + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' @@ -58,7 +89,7 @@ class TDTestCase: shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) @@ -87,6 +118,8 @@ class TDTestCase: pre_insert = "insert into " sql = pre_insert + t = time.time() + startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): sql += " %s_%d values "%(stbName,i) @@ -127,7 +160,7 @@ class TDTestCase: return def tmqCase1(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db") + tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db, inclue 1 stb") tdLog.info("step 1: create database, stb, ctb and insert data") # create and start thread parameterDict = {'cfg': '', \ @@ -135,11 +168,13 @@ class TDTestCase: 'vgroups': 4, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 100000, \ - 'batchNum': 200, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath + self.initConsumerTable() + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) @@ -149,23 +184,16 @@ class TDTestCase: topicName1 = 'topic_db1' tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) - - tdLog.info("create consume info table and consume result table") - cdbName = parameterDict["dbName"] - tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) - tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) - consumerId = 0 expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] topicList = topicName1 ifcheckdata = 0 + ifManualCommit = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ auto.commit.interval.ms:6000,\ auto.offset.reset:earliest' - sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - tdSql.query(sql) + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) event.wait() @@ -173,32 +201,28 @@ class TDTestCase: pollDelay = 5 showMsg = 1 showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName) + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) # wait for data ready prepareEnvThread.join() tdLog.info("insert process end, and start to check consume result") - while 1: - tdSql.query("select * from %s.consumeresult"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 1: - break - else: - time.sleep(5) - - tdLog.info("consumer result: %d, %d"%(tdSql.getData(0 , 2), tdSql.getData(0 , 3))) - tdSql.checkData(0 , 1, consumerId) - # mulit rows and mulit tables in one sql, this num of msg is not sure - #tdSql.checkData(0 , 2, expectmsgcnt) - tdSql.checkData(0 , 3, expectrowcnt+1) + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicName1) tdLog.printNoPrefix("======== test case 1 end ...... ") def tmqCase2(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db") + tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db, inclue 1 stb") tdLog.info("step 1: create database, stb, ctb and insert data") # create and start thread parameterDict = {'cfg': '', \ @@ -206,11 +230,13 @@ class TDTestCase: 'vgroups': 4, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 100000, \ + 'rowsPerTbl': 10000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath + self.initConsumerTable() + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) @@ -221,27 +247,19 @@ class TDTestCase: tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) - tdLog.info("create consume info table and consume result table") - cdbName = parameterDict["dbName"] - tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) - tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) - consumerId = 0 expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] topicList = topicName1 ifcheckdata = 0 + ifManualCommit = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ auto.commit.interval.ms:6000,\ auto.offset.reset:earliest' - sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - tdSql.query(sql) - + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + consumerId = 1 - sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - tdSql.query(sql) + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) event.wait() @@ -249,30 +267,20 @@ class TDTestCase: pollDelay = 5 showMsg = 1 showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName) + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) # wait for data ready prepareEnvThread.join() tdLog.info("insert process end, and start to check consume result") - while 1: - tdSql.query("select * from %s.consumeresult"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 2: - break - else: - time.sleep(5) - - consumerId0 = tdSql.getData(0 , 1) - consumerId1 = tdSql.getData(1 , 1) - actConsumeRows0 = tdSql.getData(0 , 3) - actConsumeRows1 = tdSql.getData(1 , 3) - - tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0)) - tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1)) - - totalConsumeRows = actConsumeRows0 + actConsumeRows1 - if totalConsumeRows != expectrowcnt + 2: + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicName1) @@ -288,11 +296,13 @@ class TDTestCase: 'vgroups': 4, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 100000, \ + 'rowsPerTbl': 10000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath + self.initConsumerTable() + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) @@ -303,7 +313,7 @@ class TDTestCase: 'vgroups': 4, \ 'stbName': 'stb2', \ 'ctbNum': 10, \ - 'rowsPerTbl': 100000, \ + 'rowsPerTbl': 10000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -316,59 +326,41 @@ class TDTestCase: tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) - tdLog.info("create consume info table and consume result table") - cdbName = parameterDict["dbName"] - tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) - tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) - consumerId = 0 expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] topicList = topicName1 ifcheckdata = 0 + ifManualCommit = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ auto.commit.interval.ms:6000,\ auto.offset.reset:earliest' - sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - tdSql.query(sql) + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) # consumerId = 1 - # sql = "insert into %s.consumeinfo values "%cdbName - # sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - # tdSql.query(sql) + # self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) event.wait() tdLog.info("start consume processor") pollDelay = 5 showMsg = 1 - showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName) + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) # wait for data ready prepareEnvThread.join() prepareEnvThread2.join() tdLog.info("insert process end, and start to check consume result") - while 1: - tdSql.query("select * from %s.consumeresult"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 1: - break - else: - time.sleep(5) - - consumerId0 = tdSql.getData(0 , 1) - #consumerId1 = tdSql.getData(1 , 1) - actConsumeRows0 = tdSql.getData(0 , 3) - #actConsumeRows1 = tdSql.getData(1 , 3) - - tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0)) - #tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1)) - - #totalConsumeRows = actConsumeRows0 + actConsumeRows1 - if actConsumeRows0 != expectrowcnt + 1: + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicName1) @@ -386,9 +378,9 @@ class TDTestCase: cfgPath = buildPath + "/../sim/psim/cfg" tdLog.info("cfgPath: %s" % cfgPath) - #self.tmqCase1(cfgPath, buildPath) - self.tmqCase2(cfgPath, buildPath) - #self.tmqCase3(cfgPath, buildPath) + self.tmqCase1(cfgPath, buildPath) + #self.tmqCase2(cfgPath, buildPath) + self.tmqCase3(cfgPath, buildPath) def stop(self): tdSql.close() From c47bbb12dd9bdffb3685f4d7915e3646300d92b9 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 14 May 2022 14:44:08 +0800 Subject: [PATCH 21/71] fix(os): add udfd to make install --- packaging/make_install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packaging/make_install.sh b/packaging/make_install.sh index f70f176a40..e3e22a655f 100755 --- a/packaging/make_install.sh +++ b/packaging/make_install.sh @@ -169,6 +169,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdump || : @@ -183,6 +184,7 @@ function install_bin() { [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : + [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || : ${csudo}cp -r ${binary_dir}/build/bin/tarbitrator ${install_main_dir}/bin || : @@ -197,6 +199,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || : + [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : @@ -213,6 +216,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${clientName} ] || [ -x ${install_main_2_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || ${csudo}ln -s ${install_main_2_dir}/bin/${clientName} || : [ -x ${install_main_dir}/bin/${serverName} ] || [ -x ${install_main_2_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || ${csudo}ln -s ${install_main_2_dir}/bin/${serverName} || : [ -x ${install_main_dir}/bin/taosadapter ] || [ -x ${install_main_2_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || ${csudo}ln -s ${install_main_2_dir}/bin/taosadapter || : + [ -x ${install_main_dir}/bin/udfd ] || [ -x ${install_main_2_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || ${csudo}ln -s ${install_main_2_dir}/bin/udfd || : [ -x ${install_main_dir}/bin/taosdump ] || [ -x ${install_main_2_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || ln -s ${install_main_2_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] || [ -x ${install_main_2_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || ln -s ${install_main_2_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : fi From 5d8c8677914e4336c17320d0ca76473a856d150d Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 14 May 2022 15:30:11 +0800 Subject: [PATCH 22/71] fix(os): make install copy file error --- cmake/cmake.install | 2 +- packaging/cfg/nginxd.service | 22 + packaging/cfg/taos.cfg | 312 +++++++++ packaging/cfg/taosd.service | 21 + packaging/cfg/tarbitratord.service | 20 + packaging/check_package.sh | 252 +++++++ packaging/deb/DEBIAN/control | 13 + packaging/deb/DEBIAN/postinst | 13 + packaging/deb/DEBIAN/postrm | 2 + packaging/deb/DEBIAN/preinst | 40 ++ packaging/deb/DEBIAN/prerm | 42 ++ packaging/deb/makedeb.sh | 147 ++++ packaging/deb/taosd | 95 +++ packaging/deb/tarbitratord | 88 +++ packaging/docker/Dockerfile | 32 + packaging/docker/README.md | 664 +++++++++++++++++++ packaging/docker/bin/entrypoint.sh | 83 +++ packaging/docker/bin/env-to-cfg | 13 + packaging/docker/docker-compose.yml | 77 +++ packaging/docker/dockerManifest.sh | 82 +++ packaging/docker/dockerbuild.sh | 174 +++++ packaging/docker/dockerbuildi.sh | 56 ++ packaging/release.bat | 62 ++ packaging/release.sh | 18 +- packaging/rpm/makerpm.sh | 87 +++ packaging/rpm/taosd | 145 ++++ packaging/rpm/tarbitratord | 141 ++++ packaging/rpm/tdengine.spec | 236 +++++++ packaging/tools/check_os.sh | 52 ++ packaging/tools/get_client.sh | 21 + packaging/tools/get_os.sh | 14 + packaging/tools/get_version.sh | 15 + packaging/{ => tools}/install.sh | 0 packaging/tools/install_arbi.sh | 339 ++++++++++ packaging/tools/install_client.sh | 320 +++++++++ packaging/{ => tools}/make_install.sh | 12 +- packaging/tools/makearbi.sh | 71 ++ packaging/tools/makeclient.sh | 246 +++++++ packaging/tools/makepkg.sh | 378 +++++++++++ packaging/tools/post.sh | 539 +++++++++++++++ packaging/tools/preun.sh | 142 ++++ packaging/tools/release_note | 136 ++++ packaging/{ => tools}/remove.sh | 0 packaging/tools/remove_arbi.sh | 130 ++++ packaging/tools/remove_client.sh | 85 +++ packaging/tools/repair_link.sh | 39 ++ packaging/tools/run_taosd_and_taosadapter.sh | 3 + packaging/tools/set_core.sh | 40 ++ packaging/tools/startPre.sh | 51 ++ packaging/tools/taosd-dump-cfg.gdb | 144 ++++ 50 files changed, 5700 insertions(+), 16 deletions(-) create mode 100644 packaging/cfg/nginxd.service create mode 100644 packaging/cfg/taos.cfg create mode 100644 packaging/cfg/taosd.service create mode 100644 packaging/cfg/tarbitratord.service create mode 100644 packaging/check_package.sh create mode 100644 packaging/deb/DEBIAN/control create mode 100644 packaging/deb/DEBIAN/postinst create mode 100644 packaging/deb/DEBIAN/postrm create mode 100644 packaging/deb/DEBIAN/preinst create mode 100644 packaging/deb/DEBIAN/prerm create mode 100644 packaging/deb/makedeb.sh create mode 100644 packaging/deb/taosd create mode 100644 packaging/deb/tarbitratord create mode 100644 packaging/docker/Dockerfile create mode 100644 packaging/docker/README.md create mode 100644 packaging/docker/bin/entrypoint.sh create mode 100644 packaging/docker/bin/env-to-cfg create mode 100644 packaging/docker/docker-compose.yml create mode 100644 packaging/docker/dockerManifest.sh create mode 100644 packaging/docker/dockerbuild.sh create mode 100644 packaging/docker/dockerbuildi.sh create mode 100644 packaging/release.bat create mode 100644 packaging/rpm/makerpm.sh create mode 100644 packaging/rpm/taosd create mode 100644 packaging/rpm/tarbitratord create mode 100644 packaging/rpm/tdengine.spec create mode 100644 packaging/tools/check_os.sh create mode 100644 packaging/tools/get_client.sh create mode 100644 packaging/tools/get_os.sh create mode 100644 packaging/tools/get_version.sh rename packaging/{ => tools}/install.sh (100%) create mode 100644 packaging/tools/install_arbi.sh create mode 100644 packaging/tools/install_client.sh rename packaging/{ => tools}/make_install.sh (99%) create mode 100644 packaging/tools/makearbi.sh create mode 100644 packaging/tools/makeclient.sh create mode 100644 packaging/tools/makepkg.sh create mode 100644 packaging/tools/post.sh create mode 100644 packaging/tools/preun.sh create mode 100644 packaging/tools/release_note rename packaging/{ => tools}/remove.sh (100%) create mode 100644 packaging/tools/remove_arbi.sh create mode 100644 packaging/tools/remove_client.sh create mode 100644 packaging/tools/repair_link.sh create mode 100644 packaging/tools/run_taosd_and_taosadapter.sh create mode 100644 packaging/tools/set_core.sh create mode 100644 packaging/tools/startPre.sh create mode 100644 packaging/tools/taosd-dump-cfg.gdb diff --git a/cmake/cmake.install b/cmake/cmake.install index 5a05c0c7dc..f51c6566fa 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -1,5 +1,5 @@ IF (TD_LINUX) - SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/make_install.sh") + SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})") ELSEIF (TD_WINDOWS) diff --git a/packaging/cfg/nginxd.service b/packaging/cfg/nginxd.service new file mode 100644 index 0000000000..97c0e2f829 --- /dev/null +++ b/packaging/cfg/nginxd.service @@ -0,0 +1,22 @@ +[Unit] +Description=Nginx For TDengine Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=forking +PIDFile=/usr/local/nginxd/logs/nginx.pid +ExecStart=/usr/local/nginxd/sbin/nginx +ExecStop=/usr/local/nginxd/sbin/nginx -s stop +TimeoutStopSec=1000000s +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TimeoutStartSec=0 +StandardOutput=null +Restart=always +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg new file mode 100644 index 0000000000..4570f2d124 --- /dev/null +++ b/packaging/cfg/taos.cfg @@ -0,0 +1,312 @@ +######################################################## +# # +# TDengine Configuration # +# Any questions, please email support@taosdata.com # +# # +######################################################## + +# first fully qualified domain name (FQDN) for TDengine system +# firstEp hostname:6030 + +# local fully qualified domain name (FQDN) +# fqdn hostname + +# first port number for the connection (12 continuous UDP/TCP port number are used) +# serverPort 6030 + +# log file's directory +# logDir /var/log/taos + +# data file's directory +# dataDir /var/lib/taos + +# temporary file's directory +# tempDir /tmp/ + +# the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only +# arbitrator arbitrator_hostname:6042 + +# number of threads per CPU core +# numOfThreadsPerCore 1.0 + +# number of threads to commit cache data +# numOfCommitThreads 4 + +# the proportion of total CPU cores available for query processing +# 2.0: the query threads will be set to double of the CPU cores. +# 1.0: all CPU cores are available for query processing [default]. +# 0.5: only half of the CPU cores are available for query. +# 0.0: only one core available. +# ratioOfQueryCores 1.0 + +# the last_row/first/last aggregator will not change the original column name in the result fields +keepColumnName 1 + +# number of management nodes in the system +# numOfMnodes 1 + +# enable/disable backuping vnode directory when removing vnode +# vnodeBak 1 + +# enable/disable installation / usage report +# telemetryReporting 1 + +# enable/disable load balancing +# balance 1 + +# role for dnode. 0 - any, 1 - mnode, 2 - dnode +# role 0 + +# max timer control blocks +# maxTmrCtrl 512 + +# time interval of system monitor, seconds +# monitorInterval 30 + +# number of seconds allowed for a dnode to be offline, for cluster only +# offlineThreshold 864000 + +# RPC re-try timer, millisecond +# rpcTimer 300 + +# RPC maximum time for ack, seconds. +# rpcMaxTime 600 + +# time interval of dnode status reporting to mnode, seconds, for cluster only +# statusInterval 1 + +# time interval of heart beat from shell to dnode, seconds +# shellActivityTimer 3 + +# minimum sliding window time, milli-second +# minSlidingTime 10 + +# minimum time window, milli-second +# minIntervalTime 10 + +# maximum delay before launching a stream computation, milli-second +# maxStreamCompDelay 20000 + +# maximum delay before launching a stream computation for the first time, milli-second +# maxFirstStreamCompDelay 10000 + +# retry delay when a stream computation fails, milli-second +# retryStreamCompDelay 10 + +# the delayed time for launching a stream computation, from 0.1(default, 10% of whole computing time window) to 0.9 +# streamCompDelayRatio 0.1 + +# max number of vgroups per db, 0 means configured automatically +# maxVgroupsPerDb 0 + +# max number of tables per vnode +# maxTablesPerVnode 1000000 + +# cache block size (Mbyte) +# cache 16 + +# number of cache blocks per vnode +# blocks 6 + +# number of days per DB file +# days 10 + +# number of days to keep DB file +# keep 3650 + +# minimum rows of records in file block +# minRows 100 + +# maximum rows of records in file block +# maxRows 4096 + +# the number of acknowledgments required for successful data writing +# quorum 1 + +# enable/disable compression +# comp 2 + +# write ahead log (WAL) level, 0: no wal; 1: write wal, but no fysnc; 2: write wal, and call fsync +# walLevel 1 + +# if walLevel is set to 2, the cycle of fsync being executed, if set to 0, fsync is called right away +# fsync 3000 + +# number of replications, for cluster only +# replica 1 + +# the compressed rpc message, option: +# -1 (no compression) +# 0 (all message compressed), +# > 0 (rpc message body which larger than this value will be compressed) +# compressMsgSize -1 + +# query retrieved column data compression option: +# -1 (no compression) +# 0 (all retrieved column data compressed), +# > 0 (any retrieved column size greater than this value all data will be compressed.) +# compressColData -1 + +# max length of an SQL +# maxSQLLength 65480 + +# max length of WildCards +# maxWildCardsLength 100 + +# the maximum number of records allowed for super table time sorting +# maxNumOfOrderedRes 100000 + +# system time zone +# timezone Asia/Shanghai (CST, +0800) +# system time zone (for windows 10) +# timezone UTC-8 + +# system locale +# locale en_US.UTF-8 + +# default system charset +# charset UTF-8 + +# max number of connections allowed in dnode +# maxShellConns 5000 + +# max number of connections allowed in client +# maxConnections 5000 + +# stop writing logs when the disk size of the log folder is less than this value +# minimalLogDirGB 1.0 + +# stop writing temporary files when the disk size of the tmp folder is less than this value +# minimalTmpDirGB 1.0 + +# if disk free space is less than this value, taosd service exit directly within startup process +# minimalDataDirGB 2.0 + +# One mnode is equal to the number of vnode consumed +# mnodeEqualVnodeNum 4 + +# enbale/disable http service +# http 1 + +# enable/disable system monitor +# monitor 1 + +# enable/disable recording the SQL statements via restful interface +# httpEnableRecordSql 0 + +# number of threads used to process http requests +# httpMaxThreads 2 + +# maximum number of rows returned by the restful interface +# restfulRowLimit 10240 + +# database name must be specified in restful interface if the following parameter is set, off by default +# httpDbNameMandatory 1 + +# http keep alive, default is 30 seconds +# httpKeepAlive 30000 + +# The following parameter is used to limit the maximum number of lines in log files. +# max number of lines per log filters +# numOfLogLines 10000000 + +# enable/disable async log +# asyncLog 1 + +# time of keeping log files, days +# logKeepDays 0 + + +# The following parameters are used for debug purpose only. +# debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR +# 131: output warning and error +# 135: output debug, warning and error +# 143: output trace, debug, warning and error to log +# 199: output debug, warning and error to both screen and file +# 207: output trace, debug, warning and error to both screen and file + +# debug flag for all log type, take effect when non-zero value +# debugFlag 0 + +# debug flag for meta management messages +# mDebugFlag 135 + +# debug flag for dnode messages +# dDebugFlag 135 + +# debug flag for sync module +# sDebugFlag 135 + +# debug flag for WAL +# wDebugFlag 135 + +# debug flag for SDB +# sdbDebugFlag 135 + +# debug flag for RPC +# rpcDebugFlag 131 + +# debug flag for TAOS TIMER +# tmrDebugFlag 131 + +# debug flag for TDengine client +# cDebugFlag 131 + +# debug flag for JNI +# jniDebugFlag 131 + +# debug flag for storage +# uDebugFlag 131 + +# debug flag for http server +# httpDebugFlag 131 + +# debug flag for monitor +# monDebugFlag 131 + +# debug flag for query +# qDebugFlag 131 + +# debug flag for vnode +# vDebugFlag 131 + +# debug flag for TSDB +# tsdbDebugFlag 131 + +# debug flag for continue query +# cqDebugFlag 131 + +# enable/disable recording the SQL in taos client +# enableRecordSql 0 + +# generate core file when service crash +# enableCoreFile 1 + +# maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden +# maxBinaryDisplayWidth 30 + +# enable/disable stream (continuous query) +# stream 1 + +# in retrieve blocking model, only in 50% query threads will be used in query processing in dnode +# retrieveBlockingModel 0 + +# the maximum allowed query buffer size in MB during query processing for each data node +# -1 no limit (default) +# 0 no query allowed, queries are disabled +# queryBufferSize -1 + +# percent of redundant data in tsdb meta will compact meta data,0 means donot compact +# tsdbMetaCompactRatio 0 + +# default string type used for storing JSON String, options can be binary/nchar, default is nchar +# defaultJSONStrType nchar + +# force TCP transmission +# rpcForceTcp 0 + +# unit MB. Flush vnode wal file if walSize > walFlushSize and walSize > cache*0.5*blocks +# walFlushSize 1024 + +# unit Hour. Latency of data migration +# keepTimeOffset 0 diff --git a/packaging/cfg/taosd.service b/packaging/cfg/taosd.service new file mode 100644 index 0000000000..db08001df9 --- /dev/null +++ b/packaging/cfg/taosd.service @@ -0,0 +1,21 @@ +[Unit] +Description=TDengine server service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/taosd +ExecStartPre=/usr/local/taos/bin/startPre.sh +TimeoutStopSec=1000000s +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TimeoutStartSec=0 +StandardOutput=null +Restart=always +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target diff --git a/packaging/cfg/tarbitratord.service b/packaging/cfg/tarbitratord.service new file mode 100644 index 0000000000..051b1ae44f --- /dev/null +++ b/packaging/cfg/tarbitratord.service @@ -0,0 +1,20 @@ +[Unit] +Description=TDengine arbitrator service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/tarbitrator +TimeoutStopSec=1000000s +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TimeoutStartSec=0 +StandardOutput=null +Restart=always +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target diff --git a/packaging/check_package.sh b/packaging/check_package.sh new file mode 100644 index 0000000000..7b61650162 --- /dev/null +++ b/packaging/check_package.sh @@ -0,0 +1,252 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +verMode=edge +pagMode=full + +iplist="" +serverFqdn="" + +# -----------------------Variables definition--------------------- +script_dir="../release" +# Dynamic directory +data_dir="/var/lib/taos" +log_dir="/var/log/taos" + +data_link_dir="/usr/local/taos/data" +log_link_dir="/usr/local/taos/log" + +cfg_install_dir="/etc/taos" + +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/taos" + +# old bin dir +sbin_dir="/usr/local/taos/bin" + +temp_version="" +fin_result="" + +service_config_dir="/etc/systemd/system" +nginx_port=6060 +nginx_dir="/usr/local/nginxd" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +# ============================= get input parameters ================================================= + +# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] + +# set parameters by default value +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] + +while getopts "hv:d:" arg +do + case $arg in + d) + #echo "interactiveFqdn=$OPTARG" + script_dir=$( echo $OPTARG ) + ;; + h) + echo "Usage: `basename $0` -d scripy_path" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" + +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function check_file() { + #check file whether exists + if [ ! -e $1/$2 ];then + echo -e "$1/$2 \033[31mnot exists\033[0m!quit" + fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n" + echo -e $fin_result + exit 8 + fi +} + +function get_package_name() { + var=$1 + if [[ $1 =~ 'aarch' ]];then + echo ${var::-21} + else + echo ${var::-17} + fi +} + +function check_link() { + #check Link whether exists or broken + if [ -L $1 ] ; then + if [ ! -e $1 ] ; then + echo -e "$1 \033[31Broken link\033[0m" + fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n" + echo -e $fin_result + exit 8 + fi + else + echo -e "$1 \033[31mnot exists\033[0m!quit" + fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n" + echo -e $fin_result + exit 8 + fi +} + +function check_main_path() { + #check install main dir and all sub dir + main_dir=("" "cfg" "bin" "connector" "driver" "examples" "include" "init.d") + for i in "${main_dir[@]}";do + check_file ${install_main_dir} $i + done + if [ "$verMode" == "cluster" ]; then + nginx_main_dir=("admin" "conf" "html" "sbin" "logs") + for i in "${nginx_main_dir[@]}";do + check_file ${nginx_dir} $i + done + fi + echo -e "Check main path:\033[32mOK\033[0m!" +} + +function check_bin_path() { + # check install bin dir and all sub dir + bin_dir=("taos" "taosd" "taosadapter" "taosdemo" "remove.sh" "tarbitrator" "set_core.sh") + for i in "${bin_dir[@]}";do + check_file ${sbin_dir} $i + done + lbin_dir=("taos" "taosd" "taosadapter" "taosdemo" "rmtaos" "tarbitrator" "set_core") + for i in "${lbin_dir[@]}";do + check_link ${bin_link_dir}/$i + done + if [ "$verMode" == "cluster" ]; then + check_file ${nginx_dir}/sbin nginx + fi + echo -e "Check bin path:\033[32mOK\033[0m!" +} + +function check_lib_path() { + # check all links + check_link ${lib_link_dir}/libtaos.so + check_link ${lib_link_dir}/libtaos.so.1 + + if [[ -d ${lib64_link_dir} ]]; then + check_link ${lib64_link_dir}/libtaos.so + check_link ${lib64_link_dir}/libtaos.so.1 + fi + echo -e "Check lib path:\033[32mOK\033[0m!" +} + +function check_header_path() { + # check all header + header_dir=("taos.h" "taosdef.h" "taoserror.h") + for i in "${header_dir[@]}";do + check_link ${inc_link_dir}/$i + done + echo -e "Check bin path:\033[32mOK\033[0m!" +} + +function check_taosadapter_config_dir() { + # check all config + check_file ${cfg_install_dir} taosadapter.toml + check_file ${cfg_install_dir} taosadapter.service + check_file ${install_main_dir}/cfg taosadapter.toml.org + echo -e "Check conf path:\033[32mOK\033[0m!" +} + +function check_config_dir() { + # check all config + check_file ${cfg_install_dir} taos.cfg + check_file ${install_main_dir}/cfg taos.cfg.org + echo -e "Check conf path:\033[32mOK\033[0m!" +} + +function check_log_path() { + # check log path + check_file ${log_dir} + echo -e "Check log path:\033[32mOK\033[0m!" +} + +function check_data_path() { + # check data path + check_file ${data_dir} + echo -e "Check data path:\033[32mOK\033[0m!" +} + +function install_TDengine() { + cd ${script_dir} + tar zxf $1 + temp_version=$(get_package_name $1) + cd $(get_package_name $1) + echo -e "\033[32muninstall TDengine && install TDengine...\033[0m" + rmtaos >/dev/null 2>&1 || echo 'taosd not installed' && echo -e '\n\n' |./install.sh >/dev/null 2>&1 + echo -e "\033[32mTDengine has been installed!\033[0m" + echo -e "\033[32mTDengine is starting...\033[0m" + kill_process taos && systemctl start taosd && sleep 10 +} + +function test_TDengine() { + check_main_path + check_bin_path + check_lib_path + check_header_path + check_config_dir + check_taosadapter_config_dir + check_log_path + check_data_path + result=`taos -s 'create database test ;create table test.tt(ts timestamp ,i int);insert into test.tt values(now,11);select * from test.tt' 2>&1 ||:` + if [[ $result =~ "Unable to establish" ]];then + echo -e "\033[31mTDengine connect failed\033[0m" + fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n" + echo -e $fin_result + exit 8 + fi + echo -e "Check TDengine connect:\033[32mOK\033[0m!" + fin_result=$fin_result"\033[32m$temp_version\033[0m test OK!\n" +} +# ## ==============================Main program starts from here============================ +TD_package_name=`ls ${script_dir}/*server*gz |awk -F '/' '{print $NF}' ` +temp=`pwd` +for i in $TD_package_name;do + if [[ $i =~ 'enterprise' ]];then + verMode="cluster" + else + verMode="" + fi + cd $temp + install_TDengine $i + test_TDengine +done +echo "============================================================" +echo -e $fin_result diff --git a/packaging/deb/DEBIAN/control b/packaging/deb/DEBIAN/control new file mode 100644 index 0000000000..c3136e8f0d --- /dev/null +++ b/packaging/deb/DEBIAN/control @@ -0,0 +1,13 @@ +Package: tdengine +Version: 1.0.0 +Section: utils +Priority: optional +#Essential: no +#Depends: no +#Suggests: no +Architecture: amd64 +Installed-Size: 66666 +Maintainer: support@taosdata.com +Provides: taosdata +Homepage: http://taosdata.com +Description: Big Data Platform Designed and Optimized for IoT. diff --git a/packaging/deb/DEBIAN/postinst b/packaging/deb/DEBIAN/postinst new file mode 100644 index 0000000000..a6c2817082 --- /dev/null +++ b/packaging/deb/DEBIAN/postinst @@ -0,0 +1,13 @@ +#!/bin/bash +#set -x +#path=`pwd` +insmetaPath="/usr/local/taos/script" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +${csudo}chmod -R 744 ${insmetaPath} +cd ${insmetaPath} +${csudo}./post.sh diff --git a/packaging/deb/DEBIAN/postrm b/packaging/deb/DEBIAN/postrm new file mode 100644 index 0000000000..98b0a844a8 --- /dev/null +++ b/packaging/deb/DEBIAN/postrm @@ -0,0 +1,2 @@ +#!/bin/bash + diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst new file mode 100644 index 0000000000..061f35538b --- /dev/null +++ b/packaging/deb/DEBIAN/preinst @@ -0,0 +1,40 @@ +#!/bin/bash + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +# Stop the service if running +if pidof taosd &> /dev/null; then + if pidof systemd &> /dev/null; then + ${csudo}systemctl stop taosd || : + elif $(which service &> /dev/null); then + ${csudo}service taosd stop || : + else + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi + echo "Stop taosd service success!" + sleep 1 +fi + +# if taos.cfg already softlink, remove it +cfg_install_dir="/etc/taos" +install_main_dir="/usr/local/taos" +if [ -f "${install_main_dir}/taos.cfg" ]; then + ${csudo}rm -f ${install_main_dir}/cfg/taos.cfg || : +fi + +if [ -f "${install_main_dir}/taosadapter.toml" ]; then + ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.toml || : +fi + +if [ -f "${install_main_dir}/taosadapter.service" ]; then + ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.service || : +fi + +# there can not libtaos.so*, otherwise ln -s error +${csudo}rm -f ${install_main_dir}/driver/libtaos* || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm new file mode 100644 index 0000000000..4298b33355 --- /dev/null +++ b/packaging/deb/DEBIAN/prerm @@ -0,0 +1,42 @@ +#!/bin/bash + +insmetaPath="/usr/local/taos/script" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +${csudo}chmod -R 744 ${insmetaPath} || : +#cd ${insmetaPath} +#${csudo}./preun.sh +if [ -f ${insmetaPath}/preun.sh ]; then + cd ${insmetaPath} + ${csudo}./preun.sh +else + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + inc_link_dir="/usr/include" + + data_link_dir="/usr/local/taos/data" + log_link_dir="/usr/local/taos/log" + cfg_link_dir="/usr/local/taos/cfg" + + # Remove all links + ${csudo}rm -f ${bin_link_dir}/taos || : + ${csudo}rm -f ${bin_link_dir}/taosd || : + ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${bin_link_dir}/taosdemo || : + ${csudo}rm -f ${cfg_link_dir}/* || : + ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + + ${csudo}rm -f ${log_link_dir} || : + ${csudo}rm -f ${data_link_dir} || : + + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +fi + diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh new file mode 100644 index 0000000000..e43c1a3569 --- /dev/null +++ b/packaging/deb/makedeb.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# Generate deb package for ubuntu +set -e +# set -x + +#curr_dir=$(pwd) +compile_dir=$1 +output_dir=$2 +tdengine_ver=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" +pkg_dir="${top_dir}/debworkroom" + +#echo "curr_dir: ${curr_dir}" +#echo "top_dir: ${top_dir}" +#echo "script_dir: ${script_dir}" +echo "compile_dir: ${compile_dir}" +echo "pkg_dir: ${pkg_dir}" + +if [ -d ${pkg_dir} ]; then + rm -rf ${pkg_dir} +fi +mkdir -p ${pkg_dir} +cd ${pkg_dir} + +libfile="libtaos.so.${tdengine_ver}" + +# create install dir +install_home_path="/usr/local/taos" +mkdir -p ${pkg_dir}${install_home_path} +mkdir -p ${pkg_dir}${install_home_path}/bin +mkdir -p ${pkg_dir}${install_home_path}/cfg +#mkdir -p ${pkg_dir}${install_home_path}/connector +mkdir -p ${pkg_dir}${install_home_path}/driver +mkdir -p ${pkg_dir}${install_home_path}/examples +mkdir -p ${pkg_dir}${install_home_path}/include +#mkdir -p ${pkg_dir}${install_home_path}/init.d +mkdir -p ${pkg_dir}${install_home_path}/script + +cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg +if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then + cp ${compile_dir}/test/cfg/taosadapter.toml ${pkg_dir}${install_home_path}/cfg || : +fi +if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then + cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || : +fi + +#cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d +cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script +cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script +cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin + +cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin +#cp ${compile_dir}/build/bin/taosBenchmark ${pkg_dir}${install_home_path}/bin + +if [ -f "${compile_dir}/build/bin/taosadapter" ]; then + cp ${compile_dir}/build/bin/taosadapter ${pkg_dir}${install_home_path}/bin ||: +fi + +cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver +cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include +cp ${compile_dir}/../src/inc/taosdef.h ${pkg_dir}${install_home_path}/include +cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include +cp -r ${top_dir}/examples/* ${pkg_dir}${install_home_path}/examples +#cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector +#cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector +#cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector +#cp ${compile_dir}/build/lib/taos-jdbcdriver*.* ${pkg_dir}${install_home_path}/connector ||: + +install_user_local_path="/usr/local" + +if [ -f ${compile_dir}/build/bin/jemalloc-config ]; then + mkdir -p ${pkg_dir}${install_user_local_path}/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${compile_dir}/build/bin/jemalloc-config ${pkg_dir}${install_user_local_path}/bin/ + if [ -f ${compile_dir}/build/bin/jemalloc.sh ]; then + cp ${compile_dir}/build/bin/jemalloc.sh ${pkg_dir}${install_user_local_path}/bin/ + fi + if [ -f ${compile_dir}/build/bin/jeprof ]; then + cp ${compile_dir}/build/bin/jeprof ${pkg_dir}${install_user_local_path}/bin/ + fi + if [ -f ${compile_dir}/build/include/jemalloc/jemalloc.h ]; then + cp ${compile_dir}/build/include/jemalloc/jemalloc.h ${pkg_dir}${install_user_local_path}/include/jemalloc/ + fi + if [ -f ${compile_dir}/build/lib/libjemalloc.so.2 ]; then + cp ${compile_dir}/build/lib/libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/ + ln -sf libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/libjemalloc.so + fi + if [ -f ${compile_dir}/build/lib/libjemalloc.a ]; then + cp ${compile_dir}/build/lib/libjemalloc.a ${pkg_dir}${install_user_local_path}/lib/ + fi + if [ -f ${compile_dir}/build/lib/libjemalloc_pic.a ]; then + cp ${compile_dir}/build/lib/libjemalloc_pic.a ${pkg_dir}${install_user_local_path}/lib/ + fi + if [ -f ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ]; then + cp ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ${pkg_dir}${install_user_local_path}/lib/pkgconfig/ + fi + if [ -f ${compile_dir}/build/share/doc/jemalloc/jemalloc.html ]; then + cp ${compile_dir}/build/share/doc/jemalloc/jemalloc.html ${pkg_dir}${install_user_local_path}/share/doc/jemalloc/ + fi + if [ -f ${compile_dir}/build/share/man/man3/jemalloc.3 ]; then + cp ${compile_dir}/build/share/man/man3/jemalloc.3 ${pkg_dir}${install_user_local_path}/share/man/man3/ + fi +fi + +cp -r ${compile_dir}/../packaging/deb/DEBIAN ${pkg_dir}/ +chmod 755 ${pkg_dir}/DEBIAN/* + +# modify version of control +debver="Version: "$tdengine_ver +sed -i "2c$debver" ${pkg_dir}/DEBIAN/control + +#get taos version, then set deb name +if [ "$verMode" == "cluster" ]; then + debname="TDengine-server-"${tdengine_ver}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + debname="TDengine-server"-${tdengine_ver}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + debname="TDengine-server-"${tdengine_ver}-${verType}-${osType}-${cpuType}".deb" +elif [ "$verType" == "stable" ]; then + debname=${debname}".deb" +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +# make deb package +dpkg -b ${pkg_dir} $debname +echo "make deb package success!" + +cp ${pkg_dir}/*.deb ${output_dir} + +# clean temp dir +rm -rf ${pkg_dir} diff --git a/packaging/deb/taosd b/packaging/deb/taosd new file mode 100644 index 0000000000..4dd176d097 --- /dev/null +++ b/packaging/deb/taosd @@ -0,0 +1,95 @@ +#!/bin/bash +# +# Modified from original source: Elastic Search +# https://github.com/elasticsearch/elasticsearch +# Thank you to the Elastic Search authors +# +# chkconfig: 2345 99 01 +# +### BEGIN INIT INFO +# Provides: TDengine +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts TDengine taosd +# Description: Starts TDengine taosd, a time-series database engine +### END INIT INFO + +set -e + +PATH="/bin:/usr/bin:/sbin:/usr/sbin" +NAME="TDengine" +USER="root" +GROUP="root" +DAEMON="/usr/local/taos/bin/taosd" +DAEMON_OPTS="" + +HTTPD_NAME="taosadapter" +DAEMON_HTTPD_NAME=$HTTPD_NAME +DAEMON_HTTPD="/usr/local/taos/bin/$HTTPD_NAME" + +PID_FILE="/var/run/$NAME.pid" +APPARGS="" + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +. /lib/lsb/init-functions + +case "$1" in + start) + + log_action_begin_msg "Starting TDengine..." + $DAEMON_HTTPD & + if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then + + touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE" + + if [ -n "$MAX_OPEN_FILES" ]; then + ulimit -n $MAX_OPEN_FILES + fi + + start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS + + log_end_msg $? + fi + ;; + + stop) + log_action_begin_msg "Stopping TDengine..." + pkill -9 $DAEMON_HTTPD_NAME + set +e + if [ -f "$PID_FILE" ]; then + start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null + if [ $? -eq 1 ]; then + log_action_cont_msg "TSD is not running but pid file exists, cleaning up" + elif [ $? -eq 3 ]; then + PID="`cat $PID_FILE`" + log_failure_msg "Failed to stop TDengine (pid $PID)" + exit 1 + fi + rm -f "$PID_FILE" + else + log_action_cont_msg "TDengine was not running" + fi + log_action_end_msg 0 + set -e + ;; + + restart|force-reload) + if [ -f "$PID_FILE" ]; then + $0 stop + sleep 1 + fi + $0 start + ;; + status) + status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME" + ;; + *) + exit 1 + ;; +esac + +exit 0 diff --git a/packaging/deb/tarbitratord b/packaging/deb/tarbitratord new file mode 100644 index 0000000000..8ece6cef06 --- /dev/null +++ b/packaging/deb/tarbitratord @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Modified from original source: Elastic Search +# https://github.com/elasticsearch/elasticsearch +# Thank you to the Elastic Search authors +# +# chkconfig: 2345 99 01 +# +### BEGIN INIT INFO +# Provides: taoscluster +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts taoscluster tarbitrator +# Description: Starts taoscluster tarbitrator, a arbitrator +### END INIT INFO + +set -e + +PATH="/bin:/usr/bin:/sbin:/usr/sbin" +NAME="taoscluster" +USER="root" +GROUP="root" +DAEMON="/usr/local/taos/bin/tarbitrator" +DAEMON_OPTS="" +PID_FILE="/var/run/$NAME.pid" +APPARGS="" + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +. /lib/lsb/init-functions + +case "$1" in + start) + + log_action_begin_msg "Starting tarbitrator..." + if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then + + touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE" + + if [ -n "$MAX_OPEN_FILES" ]; then + ulimit -n $MAX_OPEN_FILES + fi + + start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS + + log_end_msg $? + fi + ;; + + stop) + log_action_begin_msg "Stopping tarbitrator..." + set +e + if [ -f "$PID_FILE" ]; then + start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null + if [ $? -eq 1 ]; then + log_action_cont_msg "TSD is not running but pid file exists, cleaning up" + elif [ $? -eq 3 ]; then + PID="`cat $PID_FILE`" + log_failure_msg "Failed to stop tarbitrator (pid $PID)" + exit 1 + fi + rm -f "$PID_FILE" + else + log_action_cont_msg "tarbitrator was not running" + fi + log_action_end_msg 0 + set -e + ;; + + restart|force-reload) + if [ -f "$PID_FILE" ]; then + $0 stop + sleep 1 + fi + $0 start + ;; + status) + status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME" + ;; + *) + exit 1 + ;; +esac + +exit 0 diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile new file mode 100644 index 0000000000..58ba6e3f42 --- /dev/null +++ b/packaging/docker/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:18.04 + +WORKDIR /root + +ARG pkgFile +ARG dirName +ARG cpuType +RUN echo ${pkgFile} && echo ${dirName} + +COPY ${pkgFile} /root/ +RUN tar -zxf ${pkgFile} +WORKDIR /root/ +RUN cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root +RUN rm /root/${pkgFile} +RUN rm -rf /root/${dirName} + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get clean && apt-get update && apt-get install -y locales tzdata netcat && locale-gen en_US.UTF-8 +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" \ + LC_CTYPE=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 + +COPY ./bin/* /usr/bin/ + +ENV TINI_VERSION v0.19.0 +RUN bash -c 'echo -e "Downloading tini-${cpuType} ..."' +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--", "/usr/bin/entrypoint.sh"] +CMD ["taosd"] +VOLUME [ "/var/lib/taos", "/var/log/taos", "/corefile" ] diff --git a/packaging/docker/README.md b/packaging/docker/README.md new file mode 100644 index 0000000000..2722dbfef5 --- /dev/null +++ b/packaging/docker/README.md @@ -0,0 +1,664 @@ +# TDengine Docker Image Quick Reference + +## What is TDengine? + +TDengine is an open-sourced big data platform under [GNU AGPL v3.0](http://www.gnu.org/licenses/agpl-3.0.html), designed and optimized for the Internet of Things (IoT), Connected Cars, Industrial IoT, and IT Infrastructure and Application Monitoring. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and cost of development and operation. + +- **10x Faster on Insert/Query Speeds**: Through the innovative design on storage, on a single-core machine, over 20K requests can be processed, millions of data points can be ingested, and over 10 million data points can be retrieved in a second. It is 10 times faster than other databases. + +- **1/5 Hardware/Cloud Service Costs**: Compared with typical big data solutions, less than 1/5 of computing resources are required. Via column-based storage and tuned compression algorithms for different data types, less than 1/10 of storage space is needed. + +- **Full Stack for Time-Series Data**: By integrating a database with message queuing, caching, and stream computing features together, it is no longer necessary to integrate Kafka/Redis/HBase/Spark or other software. It makes the system architecture much simpler and more robust. + +- **Powerful Data Analysis**: Whether it is 10 years or one minute ago, data can be queried just by specifying the time range. Data can be aggregated over time, multiple time streams or both. Ad Hoc queries or analyses can be executed via TDengine shell, Python, R or Matlab. + +- **Seamless Integration with Other Tools**: Telegraf, Grafana, Matlab, R, and other tools can be integrated with TDengine without a line of code. MQTT, OPC, Hadoop, Spark, and many others will be integrated soon. + +- **Zero Management, No Learning Curve**: It takes only seconds to download, install, and run it successfully; there are no other dependencies. Automatic partitioning on tables or DBs. Standard SQL is used, with C/C++, Python, JDBC, Go and RESTful connectors. + +## How to use this image + +### Start a TDengine instance with RESTful API exposed + +Simply, you can use `docker run` to start a TDengine instance and connect it with restful connectors(eg. [JDBC-RESTful](https://www.taosdata.com/cn/documentation/connector/java)). + +```bash +docker run -d --name tdengine -p 6041:6041 tdengine/tdengine +``` + +This command starts a docker container by name `tdengine` with TDengine server running, and maps the container's HTTP port 6041 to the host's port 6041. If you have `curl` in your host, you can list the databases by the command: + +```bash +curl -u root:taosdata -d "show databases" localhost:6041/rest/sql +``` + +You can execute the `taos` shell command in the container: + +```bash +$ docker exec -it tdengine taos + +Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show databases; + name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | +==================================================================================================================================================================================================================================================================================== + log | 2022-01-17 13:57:22.270 | 10 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | +Query OK, 1 row(s) in set (0.002843s) +``` + +Since TDengine use container hostname to establish connections, it's a bit more complex to use taos shell and native connectors(such as JDBC-JNI) with TDengine container instance. This is the recommended way to expose ports and use TDengine with docker in simple cases. If you want to use taos shell or taosc/connectors smoothly outside the `tdengine` container, see next use cases that match you need. + +### Start with host network + +```bash +docker run -d --name tdengine --network host tdengine/tdengine +``` + +Starts container with `host` network will use host's hostname as fqdn instead of container id. It's much like starting natively with `systemd` in host. After installing the client, you can use `taos` shell as normal in host path. + +```bash +$ taos + +Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes; + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | host:6030 | 1 | 8 | ready | any | 2022-01-17 22:10:32.619 | | +Query OK, 1 row(s) in set (0.003233s) +``` + +### Start with exposed ports and specified hostname + +Set the fqdn explicitly will help you to use in other environment or applications. We provide environment variable `TAOS_FQDN` or `fqdn` config option to explicitly set the hostname used by TDengine container instance(s). + +Use `TAOS_FQDN` variable within `docker run` command: + +```bash +docker run -d \ + --name tdengine \ + -e TAOS_FQDN=tdengine \ + -p 6030-6049:6030-6049 \ + -p 6030-6049:6030-6049/udp \ + tdengine/tdengine +``` + +This command starts a docker container with TDengine server running and maps the container's TCP ports from 6030 to 6049 to the host's ports from 6030 to 6049 with TCP protocol and UDP ports range 6030-6039 to the host's UDP ports 6030-6039. If the host is already running TDengine server and occupying the same port(s), you need to map the container's port to a different unused port segment. (Please see TDengine 2.0 Port Description for details). In order to support TDengine clients accessing TDengine server services, both TCP and UDP ports need to be exposed by default(unless `rpcForceTcp` is set to `1`). + +If you want to use taos shell or native connectors([JDBC-JNI](https://www.taosdata.com/cn/documentation/connector/java), or [driver-go](https://github.com/taosdata/driver-go)), you need to make sure the `TAOS_FQDN` is resolvable at `/etc/hosts` or with custom DNS service. + +If you set the `TAOS_FQDN` to host's hostname, it will works as using `hosts` network like previous use case. Otherwise, like in `-e TAOS_FQDN=tdengine`, you can add the hostname record `tdengine` into `/etc/hosts` (use `127.0.0.1` here in host path, if use TDengine client/application in other hosts, you should set the right ip to the host eg. `192.168.10.1`(check the real ip in host with `hostname -i` or `ip route list default`) to make the TDengine endpoint resolvable): + +```bash +echo 127.0.0.1 tdengine |sudo tee -a /etc/hosts +``` + +Then you can use `taos` with the host `tdengine`: + +```bash +taos -h tdengine +``` + +Or develop/test applications with native connectors. As in python: + +```python +import taos; +conn = taos.connect(host = "tdengine") +res = conn.query("show databases") +for row in res.fetch_all_into_dict(): + print(row) +``` + +See the results: + +```bash +Python 3.8.10 (default, Nov 26 2021, 20:14:08) +[GCC 9.3.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import taos; +>>> conn = taos.connect(host = "tdengine") +>>> res = conn.query("show databases") +>>> for row in res.fetch_all_into_dict(): +... print(row) +... +{'name': 'log', 'created_time': datetime.datetime(2022, 1, 17, 22, 56, 2, 490000), 'ntables': 11, 'vgroups': 1, 'replica': 1, 'quorum': 1, 'days': 10, 'keep': '30', 'cache(MB)': 1, 'blocks': 3, 'minrows': 100, 'maxrows': 4096, 'wallevel': 1, 'fsync': 3000, 'comp': 2, 'cachelast': 0, 'precision': 'us', 'update': 0, 'status': 'ready'} +``` + +### Start with specific network + +Alternatively, you can use TDengine natively by using specific network. + +First, create network for TDengine server and client/application. + +```bash +docker network create td-net +``` + +Start TDengine instance with service name as fqdn (explicitly set with `TAOS_FQDN`): + +```bash +docker run -d --name tdengine --network td-net \ + -e TAOS_FQDN=tdengine \ + tdengine/tdengine +``` + +Start TDengine client in another container with the specific network: + +```bash +docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine tdengine/tdengine taos +# or +docker run --rm -it --network td-net -e tdengine/tdengine taos -h tdengine +``` + +When you build your application with docker, you should add the TDengine client in the dockerfile, as based on `ubuntu:20.04` image, install the client like this: + +```dockerfile +FROM ubuntu:20.04 +RUN apt-get update && apt-get install -y wget +ENV TDENGINE_VERSION=2.4.0.0 +RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && cd TDengine-client-${TDENGINE_VERSION} \ + && ./install_client.sh \ + && cd ../ \ + && rm -rf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz TDengine-client-${TDENGINE_VERSION} +## add your application next, eg. go, build it in builder stage, copy the binary to the runtime +#COPY --from=builder /path/to/build/app /usr/bin/ +#CMD ["app"] +``` + +Here is an Go example app: + + + + +```go +/* + * In this test program, we'll create a database and insert 4 records then select out. + */ +package main + +import ( + "database/sql" + "flag" + "fmt" + "time" + + _ "github.com/taosdata/driver-go/v2/taosSql" +) + +type config struct { + hostName string + serverPort string + user string + password string +} + +var configPara config +var taosDriverName = "taosSql" +var url string + +func init() { + flag.StringVar(&configPara.hostName, "h", "", "The host to connect to TDengine server.") + flag.StringVar(&configPara.serverPort, "p", "", "The TCP/IP port number to use for the connection to TDengine server.") + flag.StringVar(&configPara.user, "u", "root", "The TDengine user name to use when connecting to the server.") + flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.") + flag.Parse() +} + +func printAllArgs() { + fmt.Printf("============= args parse result: =============\n") + fmt.Printf("hostName: %v\n", configPara.hostName) + fmt.Printf("serverPort: %v\n", configPara.serverPort) + fmt.Printf("usr: %v\n", configPara.user) + fmt.Printf("password: %v\n", configPara.password) + fmt.Printf("================================================\n") +} + +func main() { + printAllArgs() + + url = "root:taosdata@/tcp(" + configPara.hostName + ":" + configPara.serverPort + ")/" + + taos, err := sql.Open(taosDriverName, url) + checkErr(err, "open database error") + defer taos.Close() + + taos.Exec("create database if not exists test") + taos.Exec("use test") + taos.Exec("create table if not exists tb1 (ts timestamp, a int)") + _, err = taos.Exec("insert into tb1 values(now, 0)(now+1s,1)(now+2s,2)(now+3s,3)") + checkErr(err, "failed to insert") + rows, err := taos.Query("select * from tb1") + checkErr(err, "failed to select") + + defer rows.Close() + for rows.Next() { + var r struct { + ts time.Time + a int + } + err := rows.Scan(&r.ts, &r.a) + if err != nil { + fmt.Println("scan error:\n", err) + return + } + fmt.Println(r.ts, r.a) + } +} + +func checkErr(err error, prompt string) { + if err != nil { + fmt.Println("ERROR: %s\n", prompt) + panic(err) + } +} +``` + + + + +Full version of dockerfile could be: + +```dockerfile +FROM golang:1.17.6-buster as builder +ENV TDENGINE_VERSION=2.4.0.0 +RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && cd TDengine-client-${TDENGINE_VERSION} \ + && ./install_client.sh \ + && cd ../ \ + && rm -rf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz TDengine-client-${TDENGINE_VERSION} +WORKDIR /usr/src/app/ +ENV GOPROXY="https://goproxy.io,direct" +COPY ./main.go ./go.mod ./go.sum /usr/src/app/ +RUN go env && go mod tidy && go build + +FROM ubuntu:20.04 +RUN apt-get update && apt-get install -y wget +ENV TDENGINE_VERSION=2.4.0.0 +RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && cd TDengine-client-${TDENGINE_VERSION} \ + && ./install_client.sh \ + && cd ../ \ + && rm -rf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz TDengine-client-${TDENGINE_VERSION} + +## add your application next, eg. go, build it in builder stage, copy the binary to the runtime +COPY --from=builder /usr/src/app/app /usr/bin/ +CMD ["app"] +``` + +Suppose you have `main.go`, `go.mod` `go.sum`, `app.dockerfile`, build the app and run it with network `td-net`: + +```bash +$ docker build -t app -f app.dockerfile +$ docker run --rm --network td-net app -h tdengine -p 6030 +============= args parse result: ============= +hostName: tdengine +serverPort: 6030 +usr: root +password: taosdata +================================================ +2022-01-17 15:56:55.48 +0000 UTC 0 +2022-01-17 15:56:56.48 +0000 UTC 1 +2022-01-17 15:56:57.48 +0000 UTC 2 +2022-01-17 15:56:58.48 +0000 UTC 3 +2022-01-17 15:58:01.842 +0000 UTC 0 +2022-01-17 15:58:02.842 +0000 UTC 1 +2022-01-17 15:58:03.842 +0000 UTC 2 +2022-01-17 15:58:04.842 +0000 UTC 3 +2022-01-18 01:43:48.029 +0000 UTC 0 +2022-01-18 01:43:49.029 +0000 UTC 1 +2022-01-18 01:43:50.029 +0000 UTC 2 +2022-01-18 01:43:51.029 +0000 UTC 3 +``` + +Now you must be much familiar with developing and testing with TDengine, let's see some more complex cases. + +### Start with docker-compose with multiple nodes(instances) + +Start a 2-replicas-2-mnodes-2-dnodes-1-arbitrator TDengine cluster with `docker-compose` is quite simple. Save the file as `docker-compose.yml`: + +```yaml +version: "3" +services: + arbitrator: + image: tdengine/tdengine:$VERSION + command: tarbitrator + td-1: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: +``` + +You may notice that: + +- We use `VERSION` environment variable to set `tdengine` image tag version once. +- **`TAOS_FIRST_EP`** **MUST** be set to join the newly created instances into an existing TDengine cluster. If you want more instances, use `TAOS_SECOND_EP` in case of HA(High Availability) concerns. +- `TAOS_NUM_OF_MNODES` is for setting number of mnodes for the cluster. +- `TAOS_REPLICA` set the default database replicas, `2` means there're one master and one slave copy of data. The `replica` option should be `1 <= replica <= 3`, and not greater than dnodes number. +- `TAOS_ARBITRATOR` set the arbitrator entrypoint of the cluster for failover/election stuff. It's better to use arbitrator in a two nodes cluster. +- The way to start an arbitrator service is as easy as abc: just add command name `tarbitrator`(which is the binary name of arbitrator daemon) in docker-compose service option: `command: tarbitrator`, and everything is ok now. + +Now run `docker-compose up -d` with version specified: + +```bash +$ VERSION=2.4.0.0 docker-compose up -d +Creating network "test_default" with the default driver +Creating volume "test_taosdata-td1" with default driver +Creating volume "test_taoslog-td1" with default driver +Creating volume "test_taosdata-td2" with default driver +Creating volume "test_taoslog-td2" with default driver +Creating test_td-1_1 ... done +Creating test_arbitrator_1 ... done +Creating test_td-2_1 ... done +``` + +Check the status: + +```bash +$ docker-compose ps + Name Command State Ports +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_arbitrator_1 /usr/bin/entrypoint.sh tar ... Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp +test_td-1_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp +test_td-2_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp +``` + +Check dnodes with taos shell: + +```bash +$ docker-compose exec td-1 taos -s "show dnodes" + +Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | td-1:6030 | 1 | 8 | ready | any | 2022-01-18 02:47:42.871 | | + 2 | td-2:6030 | 0 | 8 | ready | any | 2022-01-18 02:47:43.518 | | + 0 | arbitrator:6042 | 0 | 0 | ready | arb | 2022-01-18 02:47:43.633 | - | +Query OK, 3 row(s) in set (0.000811s) +``` + +### Start a TDengine cluster with scaled taosadapter service + +In previous use case, you could see the way to start other services built with TDengine(`taosd` as the default command). There's another important service you should know: + +> **taosAdapter** is a TDengine’s companion tool and is a bridge/adapter between TDengine cluster and application. It provides an easy-to-use and efficient way to ingest data from data collections agents(like Telegraf, StatsD, CollectD) directly. It also provides InfluxDB/OpenTSDB compatible data ingestion interface to allow InfluxDB/OpenTSDB applications to immigrate to TDengine seamlessly. + +`taosadapter` is running inside `tdengine` image by default, you can disable it by `TAOS_DISABLE_ADAPTER=true`. Running `taosadapter` in a separate container is like how `arbitrator` does: + +```yaml +services: + # ... + adapter: + image: tdengine/tdengine:$VERSION + command: taosadapter +``` + +`taosadapter` could be scaled with docker-compose, so that you can manage the `taosadapter` nodes easily. Here is an example shows 4-`taosadapter` instances in a TDengine cluster(much like previous use cases): + +```yaml +version: "3" + +networks: + inter: + api: + +services: + arbitrator: + image: tdengine/tdengine:$VERSION + command: tarbitrator + networks: + - inter + td-1: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ + adapter: + image: tdengine/tdengine:$VERSION + command: taosadapter + networks: + - inter + environment: + TAOS_FIRST_EP: "td-1" + TAOS_SECOND_EP: "td-2" + deploy: + replicas: 4 + nginx: + image: nginx + depends_on: + - adapter + networks: + - inter + - api + ports: + - 6041:6041 + - 6044:6044/udp + command: [ + "sh", + "-c", + "while true; + do curl -s http://adapter:6041/-/ping >/dev/null && break; + done; + printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}' + > /etc/nginx/conf.d/rest.conf; + printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}' + >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; + nginx -g 'daemon off;'", + ] +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: +``` + +Start the cluster: + +```bash +$ VERSION=2.4.0.0 docker-compose up -d +Creating network "docker_inter" with the default driver +Creating network "docker_api" with the default driver +Creating volume "docker_taosdata-td1" with default driver +Creating volume "docker_taoslog-td1" with default driver +Creating volume "docker_taosdata-td2" with default driver +Creating volume "docker_taoslog-td2" with default driver +Creating docker_td-2_1 ... done +Creating docker_arbitrator_1 ... done +Creating docker_td-1_1 ... done +Creating docker_adapter_1 ... done +Creating docker_adapter_2 ... done +Creating docker_adapter_3 ... done +``` + +It will start a TDengine cluster with two dnodes and four taosadapter instances, expose ports 6041/tcp and 6044/udp to host. + +`6041` is the RESTful API endpoint port, you can verify that the RESTful interface taosAdapter provides working using the `curl` command. + +```bash +$ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' 127.0.0.1:6041/rest/sql +{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[["log","2022-01-18 04:37:42.902",16,1,1,1,10,"30",1,3,100,4096,1,3000,2,0,"us",0,"ready"]],"rows":1} +``` + +If you run curl in batch(here we use [hyperfine](https://github.com/sharkdp/hyperfine) - a command-line benchmarking tool), the requests are balanced into 4 adapter instances. + +```bash +hyperfine -m10 'curl -u root:taosdata localhost:6041/rest/sql -d "describe log.log"' +``` + +View the logs with `docker-compose logs`: + +```bash +$ docker-compose logs adapter +# some logs skipped +adapter_2 | 01/18 04:57:44.616529 00000039 TAOS_ADAPTER info "| 200 | 162.185µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 +adapter_1 | 01/18 04:57:44.627695 00000039 TAOS_ADAPTER info "| 200 | 145.485µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=17 +adapter_3 | 01/18 04:57:44.639165 00000040 TAOS_ADAPTER info "| 200 | 146.913µs | 172.21.0.9 | POST | /rest/sql " sessionID=17 model=web +adapter_4 | 01/18 04:57:44.650829 00000039 TAOS_ADAPTER info "| 200 | 153.201µs | 172.21.0.9 | POST | /rest/sql " sessionID=17 model=web +adapter_2 | 01/18 04:57:44.662422 00000039 TAOS_ADAPTER info "| 200 | 211.393µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=19 +adapter_1 | 01/18 04:57:44.673426 00000039 TAOS_ADAPTER info "| 200 | 154.714µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 +adapter_3 | 01/18 04:57:44.684788 00000040 TAOS_ADAPTER info "| 200 | 131.876µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 +adapter_4 | 01/18 04:57:44.696261 00000039 TAOS_ADAPTER info "| 200 | 162.173µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 +adapter_2 | 01/18 04:57:44.707414 00000039 TAOS_ADAPTER info "| 200 | 164.419µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 +adapter_1 | 01/18 04:57:44.720842 00000039 TAOS_ADAPTER info "| 200 | 179.374µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=19 +adapter_3 | 01/18 04:57:44.732184 00000040 TAOS_ADAPTER info "| 200 | 141.174µs | 172.21.0.9 | POST | /rest/sql " sessionID=19 model=web +adapter_4 | 01/18 04:57:44.744024 00000039 TAOS_ADAPTER info "| 200 | 159.774µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=19 +adapter_2 | 01/18 04:57:44.773732 00000039 TAOS_ADAPTER info "| 200 | 178.993µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=21 +adapter_1 | 01/18 04:57:44.796518 00000039 TAOS_ADAPTER info "| 200 | 238.24µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 +adapter_3 | 01/18 04:57:44.810744 00000040 TAOS_ADAPTER info "| 200 | 176.133µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 +adapter_4 | 01/18 04:57:44.826395 00000039 TAOS_ADAPTER info "| 200 | 149.215µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 +``` + +`6044/udp` is the [StatsD](https://github.com/statsd/statsd)-compatible port, you can verify this feature with `nc` command(usually provided by `netcat` package). + +```bash +echo "foo:1|c" | nc -u -w0 127.0.0.1 6044 +``` + +Check the result in `taos` shell with `docker-compose exec`: + +```bash +$ dc exec td-1 taos + +Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show databases; + name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | +==================================================================================================================================================================================================================================================================================== + log | 2022-01-18 04:37:42.902 | 17 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | + statsd | 2022-01-18 04:45:02.563 | 1 | 1 | 2 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | +Query OK, 2 row(s) in set (0.001838s) + +taos> select * from statsd.foo; + ts | value | metric_type | +======================================================================================= + 2022-01-18 04:45:02.563422822 | 1 | counter | +Query OK, 1 row(s) in set (0.003854s) +``` + +Use `docker-compose up -d adapter=1 to reduce the instances to 1 + +### Deploy TDengine cluster in Docker Swarm with `docker-compose.yml` + +If you use docker swarm mode, it will schedule arbitrator/taosd/taosadapter services into different hosts automatically. If you've no experience with k8s/kubernetes, this is the most convenient way to scale out the TDengine cluster with multiple hosts/servers. + +Use the `docker-compose.yml` file in previous use case, and deploy with `docker stack` or `docker deploy`: + +```bash +$ VERSION=2.4.0 docker stack deploy -c docker-compose.yml taos +Creating network taos_inter +Creating network taos_api +Creating service taos_arbitrator +Creating service taos_td-1 +Creating service taos_td-2 +Creating service taos_adapter +Creating service taos_nginx +``` + +Now you've created a TDengine cluster with multiple host servers. + +Use `docker service` or `docker stack` to manage the cluster: + + + +```bash +$ docker stack ps taos +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +79ni8temw59n taos_nginx.1 nginx:latest TM1701 Running Running about a minute ago +3e94u72msiyg taos_adapter.1 tdengine/tdengine:2.4.0 TM1702 Running Running 56 seconds ago +100amjkwzsc6 taos_td-2.1 tdengine/tdengine:2.4.0 TM1703 Running Running about a minute ago +pkjehr2vvaaa taos_td-1.1 tdengine/tdengine:2.4.0 TM1704 Running Running 2 minutes ago +tpzvgpsr1qkt taos_arbitrator.1 tdengine/tdengine:2.4.0 TM1705 Running Running 2 minutes ago +rvss3g5yg6fa taos_adapter.2 tdengine/tdengine:2.4.0 TM1706 Running Running 56 seconds ago +i2augxamfllf taos_adapter.3 tdengine/tdengine:2.4.0 TM1707 Running Running 56 seconds ago +lmjyhzccpvpg taos_adapter.4 tdengine/tdengine:2.4.0 TM1708 Running Running 56 seconds ago +$ docker service ls +ID NAME MODE REPLICAS IMAGE PORTS +561t4lu6nfw6 taos_adapter replicated 4/4 tdengine/tdengine:2.4.0 +3hk5ct3q90sm taos_arbitrator replicated 1/1 tdengine/tdengine:2.4.0 +d8qr52envqzu taos_nginx replicated 1/1 nginx:latest *:6041->6041/tcp, *:6044->6044/udp +2isssfvjk747 taos_td-1 replicated 1/1 tdengine/tdengine:2.4.0 +9pzw7u02ichv taos_td-2 replicated 1/1 tdengine/tdengine:2.4.0 +``` + + + +It shows that there are two dnodes, one arbitrator, four taosadapter and one nginx reverse-forward service in this cluster. + +You can scale down the taosadapter replicas to `1` by `docker service`: + +```bash +$ docker service scale taos_adapter=1 +taos_adapter scaled to 1 +overall progress: 1 out of 1 tasks +1/1: running [==================================================>] +verify: Service converged + +$ docker service ls -f name=taos_adapter +ID NAME MODE REPLICAS IMAGE PORTS +561t4lu6nfw6 taos_adapter replicated 1/1 tdengine/tdengine:2.4.0 +``` + +Now it remains only 1 taosadapter instance in the cluster. + +When you want to remove the cluster, just type: + +```bash +docker stack rm taos +``` + +### Environment Variables + +When you start `tdengine` image, you can adjust the configuration of TDengine by passing environment variables on the `docker run` command line or in the docker compose file. You can use all of the environment variables that passed to taosd or taosadapter. diff --git a/packaging/docker/bin/entrypoint.sh b/packaging/docker/bin/entrypoint.sh new file mode 100644 index 0000000000..7173a35140 --- /dev/null +++ b/packaging/docker/bin/entrypoint.sh @@ -0,0 +1,83 @@ +#!/bin/sh +set -e +# for TZ awareness +if [ "$TZ" != "" ]; then + ln -sf /usr/share/zoneinfo/$TZ /etc/localtime + echo $TZ >/etc/timezone +fi + +# option to disable taosadapter, default is no +DISABLE_ADAPTER=${TAOS_DISABLE_ADAPTER:-0} +unset TAOS_DISABLE_ADAPTER + +# to get mnodeEpSet from data dir +DATA_DIR=${TAOS_DATA_DIR:-/var/lib/taos} + +# append env to custom taos.cfg +CFG_DIR=/tmp/taos +CFG_FILE=$CFG_DIR/taos.cfg + +mkdir -p $CFG_DIR >/dev/null 2>&1 + +[ -f /etc/taos/taos.cfg ] && cat /etc/taos/taos.cfg | grep -E -v "^#|^\s*$" >$CFG_FILE +env-to-cfg >>$CFG_FILE + +FQDN=$(cat $CFG_FILE | grep -E -v "^#|^$" | grep fqdn | tail -n1 | sed -E 's/.*fqdn\s+//') + +# ensure the fqdn is resolved as localhost +grep "$FQDN" /etc/hosts >/dev/null || echo "127.0.0.1 $FQDN" >>/etc/hosts + +# parse first ep host and port +FIRST_EP_HOST=${TAOS_FIRST_EP%:*} +FIRST_EP_PORT=${TAOS_FIRST_EP#*:} + +# in case of custom server port +SERVER_PORT=$(cat $CFG_FILE | grep -E -v "^#|^$" | grep serverPort | tail -n1 | sed -E 's/.*serverPort\s+//') +SERVER_PORT=${SERVER_PORT:-6030} + +# for other binaries like interpreters +if echo $1 | grep -E "taosd$" - >/dev/null; then + true # will run taosd +else + cp -f $CFG_FILE /etc/taos/taos.cfg || true + $@ + exit $? +fi + +set +e +ulimit -c unlimited +# set core files pattern, maybe failed +sysctl -w kernel.core_pattern=/corefile/core-$FQDN-%e-%p >/dev/null >&1 +set -e + +if [ "$DISABLE_ADAPTER" = "0" ]; then + which taosadapter >/dev/null && taosadapter & + # wait for 6041 port ready + for _ in $(seq 1 20); do + nc -z localhost 6041 && break + sleep 0.5 + done +fi + +# if has mnode ep set or the host is first ep or not for cluster, just start. +if [ -f "$DATA_DIR/dnode/mnodeEpSet.json" ] || + [ "$TAOS_FQDN" = "$FIRST_EP_HOST" ]; then + $@ -c $CFG_DIR +# others will first wait the first ep ready. +else + if [ "$TAOS_FIRST_EP" = "" ]; then + echo "run TDengine with single node." + $@ -c $CFG_DIR + exit $? + fi + while true; do + es=0 + taos -h $FIRST_EP_HOST -P $FIRST_EP_PORT -n startup >/dev/null || es=$? + if [ "$es" -eq 0 ]; then + taos -h $FIRST_EP_HOST -P $FIRST_EP_PORT -s "create dnode \"$FQDN:$SERVER_PORT\";" + break + fi + sleep 1s + done + $@ -c $CFG_DIR +fi diff --git a/packaging/docker/bin/env-to-cfg b/packaging/docker/bin/env-to-cfg new file mode 100644 index 0000000000..b7741d02e1 --- /dev/null +++ b/packaging/docker/bin/env-to-cfg @@ -0,0 +1,13 @@ +#!/bin/sh +set -e +self=$0 + +snake_to_camel_case() { + echo $1 | awk -F _ '{printf "%s", $1; for(i=2; i<=NF; i++) printf "%s", toupper(substr($i,1,1)) substr($i,2); print"";}' +} + +if echo $1 | grep -E "^$" - >/dev/null; then + export |grep -E 'TAOS_.*' -o| sed 's/TAOS_//' |tr A-Z a-z | awk -F"=" '{print "name=$(""'$self' " $1"); echo $name "$2}' |sh +else + snake_to_camel_case $1 +fi diff --git a/packaging/docker/docker-compose.yml b/packaging/docker/docker-compose.yml new file mode 100644 index 0000000000..e15ad0cf4f --- /dev/null +++ b/packaging/docker/docker-compose.yml @@ -0,0 +1,77 @@ +version: "3" + +networks: + inter: + api: + +services: + arbitrator: + image: tdengine/tdengine:$VERSION + command: tarbitrator + networks: + - inter + td-1: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ + adapter: + image: tdengine/tdengine:$VERSION + command: taosadapter + networks: + - inter + environment: + TAOS_FIRST_EP: "td-1" + TOAS_SECOND_EP: "td-2" + deploy: + replicas: 4 + update_config: + parallelism: 4 + nginx: + image: nginx + depends_on: + - adapter + networks: + - inter + - api + ports: + - 6041:6041 + - 6044:6044/udp + command: [ + "sh", + "-c", + "while true; + do curl -s http://adapter:6041/-/ping >/dev/null && break; + done; + printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}' + > /etc/nginx/conf.d/rest.conf; + printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}' + >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; + nginx -g 'daemon off;'", + ] +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh new file mode 100644 index 0000000000..b90e0e9a65 --- /dev/null +++ b/packaging/docker/dockerManifest.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -e +#set -x + +# dockerbuild.sh +# -n [version number] +# -p [xxxx] +# -V [stable | beta] + +# set parameters by default value +version="" +passWord="" +verType="" + +while getopts "hn:p:V:" arg +do + case $arg in + n) + #echo "version=$OPTARG" + version=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + V) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "version=${version}" + +#docker manifest rm tdengine/tdengine +#docker manifest rm tdengine/tdengine:${version} +if [ "$verType" == "beta" ]; then + docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} + docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest + docker manifest rm tdengine/tdengine-beta:${version} + docker manifest rm tdengine/tdengine-beta:latest + docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} + docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest + docker manifest inspect tdengine/tdengine:latest + docker manifest inspect tdengine/tdengine:${version} + docker login -u tdengine -p ${passWord} #replace the docker registry username and password + docker manifest push tdengine/tdengine-beta:${version} + docker manifest push tdengine/tdengine-beta:latest +elif [ "$verType" == "stable" ]; then + docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} + docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + docker manifest rm tdengine/tdengine:latest + docker manifest rm tdengine/tdengine:${version} + docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} + docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + docker manifest inspect tdengine/tdengine:latest + docker manifest inspect tdengine/tdengine:${version} + docker login -u tdengine -p ${passWord} #replace the docker registry username and password + docker manifest push tdengine/tdengine:${version} + docker manifest push tdengine/tdengine:latest +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +# docker manifest create -a tdengine/${dockername}:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} +# docker manifest create -a tdengine/${dockername}:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + +# docker login -u tdengine -p ${passWord} #replace the docker registry username and password + +# docker manifest push tdengine/tdengine:latest + +# # how set latest version ??? diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh new file mode 100644 index 0000000000..c5e3ce15fb --- /dev/null +++ b/packaging/docker/dockerbuild.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# + +set -e +#set -x + +# dockerbuild.sh +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -n [version number] +# -p [password for docker hub] +# -V [stable | beta] +# -f [pkg file] + +# set parameters by default value +cpuType="" +cpuTypeAlias="" +version="" +passWord="" +pkgFile="" +verType="stable" +dockerLatest="n" + +while getopts "hc:n:p:f:V:a:b:" arg +do + case $arg in + c) + #echo "cpuType=$OPTARG" + cpuType=$(echo $OPTARG) + ;; + n) + #echo "version=$OPTARG" + version=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + f) + #echo "pkgFile=$OPTARG" + pkgFile=$(echo $OPTARG) + ;; + b) + #echo "branchName=$OPTARG" + branchName=$(echo $OPTARG) + ;; + V) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + a) + #echo "dockerLatest=$OPTARG" + dockerLatest=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo " -n [version number] " + echo " -p [password for docker hub] " + echo " -V [stable | beta] " + echo " -f [pkg file] " + echo " -a [y | n ] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + + +# Check_verison() +# { +# } + + +if [ "$verType" == "beta" ]; then + dockername=${cpuType}-${verType} + dirName=${pkgFile%-beta*} +elif [ "$verType" == "stable" ]; then + dockername=${cpuType} + dirName=${pkgFile%-Linux*} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + + +echo "cpuType=${cpuType} version=${version} pkgFile=${pkgFile} verType=${verType} " +echo "$(pwd)" +echo "====NOTES: ${pkgFile} must be in the same directory as dockerbuild.sh====" + +scriptDir=$(dirname $(readlink -f $0)) +comunityArchiveDir=/nas/TDengine/v$version/community # community version’package directory +communityDir=${scriptDir}/../../../community +DockerfilePath=${communityDir}/packaging/docker/ +Dockerfile=${communityDir}/packaging/docker/Dockerfile +cd ${scriptDir} +cp -f ${comunityArchiveDir}/${pkgFile} . + +echo "dirName=${dirName}" + +if [[ "${cpuType}" == "x64" ]] || [[ "${cpuType}" == "amd64" ]]; then + cpuTypeAlias="amd64" +elif [[ "${cpuType}" == "aarch64" ]]; then + cpuTypeAlias="arm64" +elif [[ "${cpuType}" == "aarch32" ]]; then + cpuTypeAlias="armhf" +else + echo "Unknown cpuType: ${cpuType}" + exit 1 +fi + +docker build --rm -f "${Dockerfile}" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias} +docker login -u tdengine -p ${passWord} #replace the docker registry username and password +docker push tdengine/tdengine-${dockername}:${version} + +if [ -n "$(docker ps -aq)" ] ;then + echo "delete docker process" + docker stop $(docker ps -aq) + docker rm $(docker ps -aq) +fi + +if [ -n "$(pidof taosd)" ] ;then + echo "kill taosd " + kill -9 $(pidof taosd) +fi + +if [ -n "$(pidof power)" ] ;then + echo "kill power " + kill -9 $(pidof power) +fi + + +echo ">>>>>>>>>>>>> check whether tdengine/tdengine-${dockername}:${version} has been published" +docker run -d --name doctest -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp tdengine/tdengine-${dockername}:${version} +sleep 2 +curl -u root:taosdata -d 'show variables;' 127.0.0.1:6041/rest/sql > temp1.data +data_version=$( cat temp1.data |jq .data| jq '.[]' |grep "version" -A 2 -B 1 | jq ".[1]") +echo "${data_version}" +if [ "${data_version}" == "\"${version}\"" ] ; then + echo "docker version is right " +else + echo "docker version is wrong " + exit 1 +fi +rm -rf temp1.data + +# set this version to latest version +if [ ${dockerLatest} == 'y' ] ;then + docker tag tdengine/tdengine-${dockername}:${version} tdengine/tdengine-${dockername}:latest + docker push tdengine/tdengine-${dockername}:latest + echo ">>>>>>>>>>>>> check whether tdengine/tdengine-${dockername}:latest has been published correctly" + docker run -d --name doctestla -p 7030-7049:6030-6049 -p 7030-7049:6030-6049/udp tdengine/tdengine-${dockername}:latest + sleep 2 + curl -u root:taosdata -d 'show variables;' 127.0.0.1:7041/rest/sql > temp2.data + version_latest=` cat temp2.data |jq .data| jq '.[]' |grep "version" -A 2 -B 1 | jq ".[1]" ` + echo "${version_latest}" + if [ "${version_latest}" == "\"${version}\"" ] ; then + echo "docker version is right " + else + echo "docker version is wrong " + exit 1 + fi +fi +rm -rf temp2.data + +if [ -n "$(docker ps -aq)" ] ;then + echo "delte docker process" + docker stop $(docker ps -aq) + docker rm $(docker ps -aq) +fi + +cd ${scriptDir} +rm -f ${pkgFile} diff --git a/packaging/docker/dockerbuildi.sh b/packaging/docker/dockerbuildi.sh new file mode 100644 index 0000000000..3189479290 --- /dev/null +++ b/packaging/docker/dockerbuildi.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# + +set -e +#set -x + +# dockerbuild.sh +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -n [version number] +# -p [password for docker hub] + +# set parameters by default value +cpuType=aarch64 +verNumber="" +passWord="" + +while getopts "hc:n:p:f:" arg +do + case $arg in + c) + #echo "cpuType=$OPTARG" + cpuType=$(echo $OPTARG) + ;; + n) + #echo "verNumber=$OPTARG" + verNumber=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo " -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +pkgFile=TDengine-server-${verNumber}-Linux-${cpuType}.tar.gz + +echo "cpuType=${cpuType} verNumber=${verNumber} pkgFile=${pkgFile} " + +scriptDir=`pwd` +pkgDir=$scriptDir/../../release/ + +cp -f ${pkgDir}/${pkgFile} . + +./dockerbuild.sh -c ${cpuType} -f ${pkgFile} -n ${verNumber} -p ${passWord} + +rm -f ${pkgFile} diff --git a/packaging/release.bat b/packaging/release.bat new file mode 100644 index 0000000000..a3fa575837 --- /dev/null +++ b/packaging/release.bat @@ -0,0 +1,62 @@ +@echo off + +set internal_dir=%~dp0\..\..\ +set community_dir=%~dp0\.. +cd %community_dir% +git checkout -- . +cd %community_dir%\packaging + +:: %1 name %2 version +if !%1==! GOTO USAGE +if !%2==! GOTO USAGE +if %1 == taos GOTO TAOS +if %1 == power GOTO POWER +if %1 == tq GOTO TQ +if %1 == pro GOTO PRO +if %1 == kh GOTO KH +if %1 == jh GOTO JH +GOTO USAGE + +:TAOS +goto RELEASE + +:POWER +call sed_power.bat %community_dir% +goto RELEASE + +:TQ +call sed_tq.bat %community_dir% +goto RELEASE + +:PRO +call sed_pro.bat %community_dir% +goto RELEASE + +:KH +call sed_kh.bat %community_dir% +goto RELEASE + +:JH +call sed_jh.bat %community_dir% +goto RELEASE + +:RELEASE +echo release windows-client-64 for %1, version: %2 +if not exist %internal_dir%\debug\ver-%2-64bit-%1 ( + md %internal_dir%\debug\ver-%2-64bit-%1 +) else ( + rd /S /Q %internal_dir%\debug\ver-%2-64bit-%1 + md %internal_dir%\debug\ver-%2-64bit-%1 +) +cd %internal_dir%\debug\ver-%2-64bit-%1 +call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 +cmake ../../ -G "NMake Makefiles" -DVERNUMBER=%2 -DCPUTYPE=x64 +set CL=/MP4 +nmake install +goto EXIT0 + +:USAGE +echo Usage: release.bat $productName $version +goto EXIT0 + +:EXIT0 \ No newline at end of file diff --git a/packaging/release.sh b/packaging/release.sh index 1077eb569a..ef3018a913 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -21,17 +21,17 @@ echo "script_dir: ${script_dir}" echo "top_dir: ${top_dir}" cd ${top_dir} -git checkout -- . -git checkout 3.0 -git pull || : +# git checkout -- . +# git checkout 3.0 +# git pull || : echo "curr_dir: ${curr_dir}" # 2. cmake executable file compile_dir="${top_dir}/debug" -if [ -d ${compile_dir} ]; then - rm -rf ${compile_dir} -fi +# if [ -d ${compile_dir} ]; then +# rm -rf ${compile_dir} +# fi mkdir -p ${compile_dir} @@ -56,14 +56,14 @@ mkdir -p ${install_dir}/bin mkdir -p ${install_dir}/lib mkdir -p ${install_dir}/inc -install_files="${script_dir}/install.sh" -chmod a+x ${script_dir}/install.sh || : +install_files="${script_dir}/tools/install.sh" +chmod a+x ${script_dir}/tools/install.sh || : cp ${install_files} ${install_dir} header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h" cp ${header_files} ${install_dir}/inc -bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos ${compile_dir}/build/bin/create_table ${compile_dir}/build/bin/tmq_sim ${script_dir}/remove.sh ${compile_dir}/build/bin/taosBenchmark ${compile_dir}/build/bin/taosdump" +bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos ${compile_dir}/build/bin/create_table ${compile_dir}/build/bin/tmq_sim ${script_dir}/tools/remove.sh ${compile_dir}/build/bin/taosBenchmark ${compile_dir}/build/bin/taosdump" cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : cp ${compile_dir}/build/lib/libtaos.so ${install_dir}/lib/ diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh new file mode 100644 index 0000000000..31673488e6 --- /dev/null +++ b/packaging/rpm/makerpm.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Generate rpm package for centos + +set -e +# set -x + +#curr_dir=$(pwd) +compile_dir=$1 +output_dir=$2 +tdengine_ver=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" +pkg_dir="${top_dir}/rpmworkroom" +spec_file="${script_dir}/tdengine.spec" + +#echo "curr_dir: ${curr_dir}" +#echo "top_dir: ${top_dir}" +#echo "script_dir: ${script_dir}" +echo "compile_dir: ${compile_dir}" +echo "pkg_dir: ${pkg_dir}" +echo "spec_file: ${spec_file}" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +function cp_rpm_package() { + local cur_dir + cd $1 + cur_dir=$(pwd) + + for dirlist in "$(ls ${cur_dir})"; do + if test -d ${dirlist}; then + cd ${dirlist} + cp_rpm_package ${cur_dir}/${dirlist} + cd .. + fi + if test -e ${dirlist}; then + cp ${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm + fi + done +} + +if [ -d ${pkg_dir} ]; then + ${csudo}rm -rf ${pkg_dir} +fi +${csudo}mkdir -p ${pkg_dir} +cd ${pkg_dir} + +${csudo}mkdir -p BUILD BUILDROOT RPMS SOURCES SPECS SRPMS + +${csudo}rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_dir}" --define="_compiledir ${compile_dir}" -bb ${spec_file} + +# copy rpm package to output_dir, and modify package name, then clean temp dir +#${csudo}cp -rf RPMS/* ${output_dir} +cp_rpm_package ${pkg_dir}/RPMS + + +if [ "$verMode" == "cluster" ]; then + rpmname="TDengine-server-"${tdengine_ver}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + rpmname="TDengine-server"-${tdengine_ver}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + rpmname="TDengine-server-"${tdengine_ver}-${verType}-${osType}-${cpuType}".rpm" +elif [ "$verType" == "stable" ]; then + rpmname=${rpmname}".rpm" +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +mv ${output_dir}/TDengine-${tdengine_ver}.rpm ${output_dir}/${rpmname} + +cd .. +${csudo}rm -rf ${pkg_dir} diff --git a/packaging/rpm/taosd b/packaging/rpm/taosd new file mode 100644 index 0000000000..b79361f36e --- /dev/null +++ b/packaging/rpm/taosd @@ -0,0 +1,145 @@ +#!/bin/bash +# +# taosd This shell script takes care of starting and stopping TDengine. +# +# chkconfig: 2345 99 01 +# description: TDengine is a districuted, scalable, high-performance Time Series Database +# (TSDB). More than just a pure database, TDengine also provides the ability +# to do stream computing, aggregation etc. +# +# +### BEGIN INIT INFO +# Provides: taosd +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Short-Description: start and stop taosd +# Description: TDengine is a districuted, scalable, high-performance Time Series Database +# (TSDB). More than just a pure database, TDengine also provides the ability +# to do stream computing, aggregation etc. +### END INIT INFO + +# Source init functions +. /etc/init.d/functions + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +# Default program options +NAME=taosd +PROG=/usr/local/taos/bin/taosd +USER=root +GROUP=root + +# Default directories +LOCK_DIR=/var/lock/subsys +PID_DIR=/var/run/$NAME + +# Set file names +LOCK_FILE=$LOCK_DIR/$NAME +PID_FILE=$PID_DIR/$NAME.pid + +[ -e $PID_DIR ] || mkdir -p $PID_DIR + +PROG_OPTS="" + +start() { + echo -n "Starting ${NAME}: " + # check identity + curid="`id -u -n`" + if [ "$curid" != root ] && [ "$curid" != "$USER" ] ; then + echo "Must be run as root or $USER, but was run as $curid" + return 1 + fi + # Sets the maximum number of open file descriptors allowed. + ulimit -n $MAX_OPEN_FILES + curulimit="`ulimit -n`" + if [ "$curulimit" -lt $MAX_OPEN_FILES ] ; then + echo "'ulimit -n' must be greater than or equal to $MAX_OPEN_FILES, is $curulimit" + return 1 + fi + + if [ "`id -u -n`" == root ] ; then + # Changes the owner of the lock, and the pid files to allow + # non-root OpenTSDB daemons to run /usr/share/opentsdb/bin/opentsdb_restart.py. + touch $LOCK_FILE && chown $USER:$GROUP $LOCK_FILE + touch $PID_FILE && chown $USER:$GROUP $PID_FILE + daemon --user $USER --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + else + # Don't have to change user. + daemon --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + fi + retval=$? + sleep 2 + echo + [ $retval -eq 0 ] && (findproc > $PID_FILE && touch $LOCK_FILE) + return $retval +} + +stop() { + echo -n "Stopping ${NAME}: " + killproc -p $PID_FILE $NAME + retval=$? + echo + # Non-root users don't have enough permission to remove pid and lock files. + # So, the opentsdb_restart.py cannot get rid of the files, and the command + # "service opentsdb status" will complain about the existing pid file. + # Makes the pid file empty. + echo > $PID_FILE + [ $retval -eq 0 ] && (rm -f $PID_FILE && rm -f $LOCK_FILE) + return $retval +} + +restart() { + stop + start +} + +reload() { + restart +} + +force_reload() { + restart +} + +rh_status() { + # run checks to determine if the service is running or use generic status + status -p $PID_FILE -l $LOCK_FILE $NAME +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 2 +esac + +exit $? diff --git a/packaging/rpm/tarbitratord b/packaging/rpm/tarbitratord new file mode 100644 index 0000000000..34fedab3d9 --- /dev/null +++ b/packaging/rpm/tarbitratord @@ -0,0 +1,141 @@ +#!/bin/bash +# +# tarbitratord This shell script takes care of starting and stopping tarbitrator. +# +# chkconfig: 2345 99 01 +# description: tarbitrator is a arbitrator used in TDengine cluster. +# +# +### BEGIN INIT INFO +# Provides: taoscluster +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Short-Description: start and stop tarbitrator +# Description: tarbitrator is a arbitrator used in TDengine cluster. +### END INIT INFO + +# Source init functions +. /etc/init.d/functions + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +# Default program options +NAME=tarbitrator +PROG=/usr/local/taos/bin/tarbitrator +USER=root +GROUP=root + +# Default directories +LOCK_DIR=/var/lock/subsys +PID_DIR=/var/run/$NAME + +# Set file names +LOCK_FILE=$LOCK_DIR/$NAME +PID_FILE=$PID_DIR/$NAME.pid + +[ -e $PID_DIR ] || mkdir -p $PID_DIR + +PROG_OPTS="" + +start() { + echo -n "Starting ${NAME}: " + # check identity + curid="`id -u -n`" + if [ "$curid" != root ] && [ "$curid" != "$USER" ] ; then + echo "Must be run as root or $USER, but was run as $curid" + return 1 + fi + # Sets the maximum number of open file descriptors allowed. + ulimit -n $MAX_OPEN_FILES + curulimit="`ulimit -n`" + if [ "$curulimit" -lt $MAX_OPEN_FILES ] ; then + echo "'ulimit -n' must be greater than or equal to $MAX_OPEN_FILES, is $curulimit" + return 1 + fi + + if [ "`id -u -n`" == root ] ; then + # Changes the owner of the lock, and the pid files to allow + # non-root OpenTSDB daemons to run /usr/share/opentsdb/bin/opentsdb_restart.py. + touch $LOCK_FILE && chown $USER:$GROUP $LOCK_FILE + touch $PID_FILE && chown $USER:$GROUP $PID_FILE + daemon --user $USER --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + else + # Don't have to change user. + daemon --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + fi + retval=$? + sleep 2 + echo + [ $retval -eq 0 ] && (findproc > $PID_FILE && touch $LOCK_FILE) + return $retval +} + +stop() { + echo -n "Stopping ${NAME}: " + killproc -p $PID_FILE $NAME + retval=$? + echo + # Non-root users don't have enough permission to remove pid and lock files. + # So, the opentsdb_restart.py cannot get rid of the files, and the command + # "service opentsdb status" will complain about the existing pid file. + # Makes the pid file empty. + echo > $PID_FILE + [ $retval -eq 0 ] && (rm -f $PID_FILE && rm -f $LOCK_FILE) + return $retval +} + +restart() { + stop + start +} + +reload() { + restart +} + +force_reload() { + restart +} + +rh_status() { + # run checks to determine if the service is running or use generic status + status -p $PID_FILE -l $LOCK_FILE $NAME +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 2 +esac + +exit $? diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec new file mode 100644 index 0000000000..022482c867 --- /dev/null +++ b/packaging/rpm/tdengine.spec @@ -0,0 +1,236 @@ +%define homepath /usr/local/taos +%define userlocalpath /usr/local +%define cfg_install_dir /etc/taos +%define __strip /bin/true + +Name: tdengine +Version: %{_version} +Release: 3%{?dist} +Summary: tdengine from taosdata +Group: Application/Database +License: AGPL +URL: www.taosdata.com +AutoReqProv: no + +#BuildRoot: %_topdir/BUILDROOT +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root + +#Prefix: /usr/local/taos + +#BuildRequires: +#Requires: + +%description +Big Data Platform Designed and Optimized for IoT + +#"prep" Nothing needs to be done +#%prep +#%setup -q +#%setup -T + +#"build" Nothing needs to be done +#%build +#%configure +#make %{?_smp_mflags} + +%install +#make install DESTDIR=%{buildroot} +rm -rf %{buildroot} + +echo topdir: %{_topdir} +echo version: %{_version} +echo buildroot: %{buildroot} + +libfile="libtaos.so.%{_version}" + +# create install path, and cp file +mkdir -p %{buildroot}%{homepath}/bin +mkdir -p %{buildroot}%{homepath}/cfg +#mkdir -p %{buildroot}%{homepath}/connector +mkdir -p %{buildroot}%{homepath}/driver +mkdir -p %{buildroot}%{homepath}/examples +mkdir -p %{buildroot}%{homepath}/include +#mkdir -p %{buildroot}%{homepath}/init.d +mkdir -p %{buildroot}%{homepath}/script + +cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg +if [ -f %{_compiledir}/test/cfg/taosadapter.toml ]; then + cp %{_compiledir}/test/cfg/taosadapter.toml %{buildroot}%{homepath}/cfg +fi +if [ -f %{_compiledir}/test/cfg/taosadapter.service ]; then + cp %{_compiledir}/test/cfg/taosadapter.service %{buildroot}%{homepath}/cfg +fi +#cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d +cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script +cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/script +cp %{_compiledir}/../packaging/tools/startPre.sh %{buildroot}%{homepath}/bin +cp %{_compiledir}/../packaging/tools/set_core.sh %{buildroot}%{homepath}/bin +cp %{_compiledir}/../packaging/tools/taosd-dump-cfg.gdb %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin +#cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin + +if [ -f %{_compiledir}/build/bin/taosadapter ]; then + cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin ||: +fi +cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver +cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include +cp %{_compiledir}/../src/inc/taosdef.h %{buildroot}%{homepath}/include +cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include +#cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector +#cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector +#cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector +#cp %{_compiledir}/build/lib/taos-jdbcdriver*.* %{buildroot}%{homepath}/connector ||: +cp -r %{_compiledir}/../examples/* %{buildroot}%{homepath}/examples + +if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then + mkdir -p %{buildroot}%{userlocalpath}/bin + mkdir -p %{buildroot}%{userlocalpath}/lib + mkdir -p %{buildroot}%{userlocalpath}/lib/pkgconfig + mkdir -p %{buildroot}%{userlocalpath}/include + mkdir -p %{buildroot}%{userlocalpath}/include/jemalloc + mkdir -p %{buildroot}%{userlocalpath}/share + mkdir -p %{buildroot}%{userlocalpath}/share/doc + mkdir -p %{buildroot}%{userlocalpath}/share/doc/jemalloc + mkdir -p %{buildroot}%{userlocalpath}/share/man + mkdir -p %{buildroot}%{userlocalpath}/share/man/man3 + + cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{userlocalpath}/bin/ + if [ -f %{_compiledir}/build/bin/jemalloc.sh ]; then + cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{userlocalpath}/bin/ + fi + if [ -f %{_compiledir}/build/bin/jeprof ]; then + cp %{_compiledir}/build/bin/jeprof %{buildroot}%{userlocalpath}/bin/ + fi + if [ -f %{_compiledir}/build/include/jemalloc/jemalloc.h ]; then + cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{userlocalpath}/include/jemalloc/ + fi + if [ -f %{_compiledir}/build/lib/libjemalloc.so.2 ]; then + cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/ + ln -sf libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/libjemalloc.so + fi + if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then + cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{userlocalpath}/lib/ + fi + if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then + cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{userlocalpath}/lib/ + fi + if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then + cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{userlocalpath}/lib/pkgconfig/ + fi + if [ -f %{_compiledir}/build/share/doc/jemalloc/jemalloc.html ]; then + cp %{_compiledir}/build/share/doc/jemalloc/jemalloc.html %{buildroot}%{userlocalpath}/share/doc/jemalloc/ + fi + if [ -f %{_compiledir}/build/share/man/man3/jemalloc.3 ]; then + cp %{_compiledir}/build/share/man/man3/jemalloc.3 %{buildroot}%{userlocalpath}/share/man/man3/ + fi +fi + +#Scripts executed before installation +%pre +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +# Stop the service if running +if pidof taosd &> /dev/null; then + if pidof systemd &> /dev/null; then + ${csudo}systemctl stop taosd || : + elif $(which service &> /dev/null); then + ${csudo}service taosd stop || : + else + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi + echo "Stop taosd service success!" + sleep 1 +fi +# if taos.cfg already exist, remove it +if [ -f %{cfg_install_dir}/taos.cfg ]; then + ${csudo}rm -f %{cfg_install_dir}/cfg/taos.cfg || : +fi + +# if taosadapter.toml already exist, remove it +if [ -f %{cfg_install_dir}/taosadapter.toml ]; then + ${csudo}rm -f %{cfg_install_dir}/cfg/taosadapter.toml || : +fi + +# there can not libtaos.so*, otherwise ln -s error +${csudo}rm -f %{homepath}/driver/libtaos* || : + +#Scripts executed after installation +%post +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi +cd %{homepath}/script +${csudo}./post.sh + +# Scripts executed before uninstall +%preun +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi +# only remove package to call preun.sh, not but update(2) +if [ $1 -eq 0 ];then + #cd %{homepath}/script + #${csudo}./preun.sh + + if [ -f %{homepath}/script/preun.sh ]; then + cd %{homepath}/script + ${csudo}./preun.sh + else + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + inc_link_dir="/usr/include" + + data_link_dir="/usr/local/taos/data" + log_link_dir="/usr/local/taos/log" + cfg_link_dir="/usr/local/taos/cfg" + + # Remove all links + ${csudo}rm -f ${bin_link_dir}/taos || : + ${csudo}rm -f ${bin_link_dir}/taosd || : + ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${cfg_link_dir}/* || : + ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${inc_link_dir}/taosdef.h || : + ${csudo}rm -f ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + + ${csudo}rm -f ${log_link_dir} || : + ${csudo}rm -f ${data_link_dir} || : + + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi +fi + +# Scripts executed after uninstall +%postun + +# clean build dir +%clean +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi +${csudo}rm -rf %{buildroot} + +#Specify the files to be packaged +%files +/* +#%doc + +#Setting default permissions +%defattr (-,root,root,0755) +#%{prefix} + +#%changelog diff --git a/packaging/tools/check_os.sh b/packaging/tools/check_os.sh new file mode 100644 index 0000000000..d5ec3326b0 --- /dev/null +++ b/packaging/tools/check_os.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +CSI=$(echo -e "\033[") +CRED="${CSI}1;31m" +CFAILURE="$CRED" +CEND="${CSI}0m" +if [ -n "$(grep 'Aliyun Linux release' /etc/issue)" -o -e /etc/redhat-release ]; then + OS=CentOS + [ -n "$(grep ' 7\.' /etc/redhat-release 2> /dev/null)" ] && CentOS_RHEL_version=7 + [ -n "$(grep ' 6\.' /etc/redhat-release 2> /dev/null)" -o -n "$(grep 'Aliyun Linux release6 15' /etc/issue)" ] && CentOS_RHEL_version=6 + [ -n "$(grep ' 5\.' /etc/redhat-release 2> /dev/null)" -o -n "$(grep 'Aliyun Linux release5' /etc/issue)" ] && CentOS_RHEL_version=5 +elif [ -n "$(grep 'Amazon Linux AMI release' /etc/issue)" -o -e /etc/system-release ]; then + OS=CentOS + CentOS_RHEL_version=6 +elif [ -n "$(grep 'bian' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == "Debian" ]; then + OS=Debian + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + Debian_version=$(lsb_release -sr | awk -F. '{print $1}') +elif [ -n "$(grep 'Deepin' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == "Deepin" ]; then + OS=Debian + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + Debian_version=$(lsb_release -sr | awk -F. '{print $1}') +elif [ -n "$(grep 'Kali GNU/Linux Rolling' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == "Kali" ]; then + OS=Debian + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + if [ -n "$(grep 'VERSION="2016.*"' /etc/os-release)" ]; then + Debian_version=8 + else + echo "${CFAILURE}Does not support this OS, Please contact the author! ${CEND}" + kill -9 $$ + fi +elif [ -n "$(grep 'Ubuntu' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == "Ubuntu" -o -n "$(grep 'Linux Mint' /etc/issue)" ]; then + OS=Ubuntu + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + Ubuntu_version=$(lsb_release -sr | awk -F. '{print $1}') + [ -n "$(grep 'Linux Mint 18' /etc/issue)" ] && Ubuntu_version=16 +elif [ -n "$(grep 'elementary' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == 'elementary' ]; then + OS=Ubuntu + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + Ubuntu_version=16 +else + echo "${CFAILURE}Does not support this OS, Please contact the author! ${CEND}" + kill -9 $$ +fi + +echo "${CFAILURE}${OS}${CEND}" +if [ "$OS" == 'CentOS' ]; then + echo ${CentOS_RHEL_version} +else + echo ${Ubuntu_version} +fi + diff --git a/packaging/tools/get_client.sh b/packaging/tools/get_client.sh new file mode 100644 index 0000000000..c29e3e79e8 --- /dev/null +++ b/packaging/tools/get_client.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# + +log_dir=$1 +result_file=$2 + +if [ ! -n "$1" ];then + echo "Pleas input the director of taosdlog." + echo "usage: ./get_client.sh " + exit 1 +else + log_dir=$1 +fi + +if [ ! -n "$2" ];then + result_file=clientInfo.txt +else + result_file=$2 +fi + +grep "new TCP connection" ${log_dir}/taosdlog.* | sed -e "s/0x.* from / /"|sed -e "s/,.*$//"|sed -e "s/:[0-9]*$//"|sort -r|uniq -f 2|sort -k 3 -r|uniq -f 2 > ${result_file} diff --git a/packaging/tools/get_os.sh b/packaging/tools/get_os.sh new file mode 100644 index 0000000000..1216f7a6a6 --- /dev/null +++ b/packaging/tools/get_os.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# This file is used to install TAOS time-series database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +# set -x + +# -----------------------Variables definition--------------------- +OS=$(cat /etc/*-release | grep "^NAME=" | cut -d= -f2) +len=$(echo ${#OS}) +len=$((len-2)) +retval=$(echo -ne ${OS:1:${len}} | cut -d" " -f1) +echo -ne $retval diff --git a/packaging/tools/get_version.sh b/packaging/tools/get_version.sh new file mode 100644 index 0000000000..44c8c6bf21 --- /dev/null +++ b/packaging/tools/get_version.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# This file is used to install TAOS time-series database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +# set -x + +# -----------------------Variables definition--------------------- +verinfo=$(cat $1 | grep " version" | cut -d '"' -f2) +verinfo=$(echo $verinfo | tr "\n" " ") +len=$(echo ${#verinfo}) +len=$((len-1)) +retval=$(echo -ne ${verinfo:0:${len}}) +echo -ne $retval diff --git a/packaging/install.sh b/packaging/tools/install.sh similarity index 100% rename from packaging/install.sh rename to packaging/tools/install.sh diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh new file mode 100644 index 0000000000..22874f058a --- /dev/null +++ b/packaging/tools/install_arbi.sh @@ -0,0 +1,339 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tarbitrator" + +# old bin dir +bin_dir="/usr/local/tarbitrator/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo >/dev/null; then + csudo="sudo " +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &>/dev/null; then + service_mod=0 +elif $(which service &>/dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &>/dev/null); then + initd_mod=1 + elif $(which insserv &>/dev/null); then + initd_mod=2 + elif $(which update-rc.d &>/dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) || : +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu"; then + # echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian"; then + # echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin"; then + # echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos"; then + # echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora"; then + # echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo}rm -rf ${install_main_dir} || : + ${csudo}mkdir -p ${install_main_dir} + ${csudo}mkdir -p ${install_main_dir}/bin + #${csudo}mkdir -p ${install_main_dir}/include + ${csudo}mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo}rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo}rm -f ${bin_link_dir}/tarbitrator || : + ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_arbi.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : + ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo}/usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}/usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &>/dev/null; then + ${csudo}service tarbitratord stop || : + fi + + if ((${initd_mod} == 1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}chkconfig --del tarbitratord || : + fi + elif ((${initd_mod} == 2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}insserv -r tarbitratord || : + fi + elif ((${initd_mod} == 3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo}rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &>/dev/null); then + ${csudo}init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + if ((${os_type} == 1)); then + ${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type} == 2)); then + ${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord + fi + + if ((${initd_mod} == 1)); then + ${csudo}chkconfig --add tarbitratord || : + ${csudo}chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod} == 2)); then + ${csudo}insserv tarbitratord || : + ${csudo}insserv -d tarbitratord || : + elif ((${initd_mod} == 3)); then + ${csudo}update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo}systemctl stop tarbitratord &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null + + ${csudo}rm -f ${tarbitratord_service_config} +} + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo}bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo}systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod} == 0)); then + install_service_on_systemd + elif ((${service_mod} == 1)); then + install_service_on_sysvinit + else + kill_tarbitrator + fi +} + +function update_TDengine() { + # Start to update + echo -e "${GREEN}Start to update TDengine's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &>/dev/null; then + if ((${service_mod} == 0)); then + ${csudo}systemctl stop tarbitratord || : + elif ((${service_mod} == 1)); then + ${csudo}service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + install_jemalloc + + echo + if ((${service_mod} == 0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}systemctl start tarbitratord${NC}" + elif ((${service_mod} == 1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}" +} + +function install_TDengine() { + # Start to install + echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + install_jemalloc + + echo + if ((${service_mod} == 0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}systemctl start tarbitratord${NC}" + elif ((${service_mod} == 1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mTDengine's arbitrator is installed successfully!${NC}" + echo +} + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update_TDengine +else + install_TDengine +fi diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh new file mode 100644 index 0000000000..5508f2b5d0 --- /dev/null +++ b/packaging/tools/install_client.sh @@ -0,0 +1,320 @@ +#!/bin/bash +# +# This file is used to install TDengine client on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- + +dataDir="/var/lib/taos" +logDir="/var/log/taos" +productName="TDengine" +installDir="/usr/local/taos" +configDir="/etc/taos" +serverName="taosd" +clientName="taos" +uninstallScript="rmtaos" +configFile="taos.cfg" +tarName="taos.tar.gz" + +osType=Linux +pagMode=full +verMode=edge + +if [ "$osType" != "Darwin" ]; then + script_dir=$(dirname $(readlink -f "$0")) + # Dynamic directory + data_dir=${dataDir} + log_dir=${logDir} +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + data_dir=${dataDir} + log_dir=~/${productName}/log +fi + +log_link_dir="${installDir}/log" + +cfg_install_dir=${configDir} + +if [ "$osType" != "Darwin" ]; then + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + lib64_link_dir="/usr/lib64" + inc_link_dir="/usr/include" +else + bin_link_dir="/usr/local/bin" + lib_link_dir="/usr/local/lib" + inc_link_dir="/usr/local/include" +fi + +#install main path +install_main_dir="${installDir}" + +# old bin dir +bin_dir="${installDir}/bin" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +update_flag=0 + +function kill_client() { + pid=$(ps -ef | grep "${clientName}" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo}rm -rf ${install_main_dir} || : + ${csudo}mkdir -p ${install_main_dir} + ${csudo}mkdir -p ${install_main_dir}/cfg + ${csudo}mkdir -p ${install_main_dir}/bin + ${csudo}mkdir -p ${install_main_dir}/driver + if [ $productName == "TDengine" ]; then + ${csudo}mkdir -p ${install_main_dir}/examples + fi + ${csudo}mkdir -p ${install_main_dir}/include + if [ "$verMode" == "cluster" ]; then + ${csudo}mkdir -p ${install_main_dir}/connector + fi +} + +function install_bin() { + # Remove links + ${csudo}rm -f ${bin_link_dir}/${clientName} || : + if [ "$osType" != "Darwin" ]; then + ${csudo}rm -f ${bin_link_dir}/taosdemo || : + fi + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/set_core || : + + ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : + if [ "$osType" != "Darwin" ]; then + [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + fi + [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : +} + +function clean_lib() { + sudo rm -f /usr/lib/libtaos.* || : + sudo rm -rf ${lib_dir} || : +} + +function install_lib() { + # Remove links + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo}rm -rf ${v15_java_app_dir} || : + + ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* + + if [ "$osType" != "Darwin" ]; then + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [ -d "${lib64_link_dir}" ]; then + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + else + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo}ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib + fi + + if [ "$osType" != "Darwin" ]; then + ${csudo}ldconfig + else + ${csudo}update_dyld_shared_cache + fi +} + +function install_header() { + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : + ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo}/usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}/usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf > /dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi +} + +function install_config() { + if [ ! -f ${cfg_install_dir}/${configFile} ]; then + ${csudo}mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} + ${csudo}chmod 644 ${cfg_install_dir}/* + fi + + ${csudo}cp -f ${script_dir}/cfg/${configFile} ${install_main_dir}/cfg/${configFile}.org + ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg +} + + +function install_log() { + ${csudo}rm -rf ${log_dir} || : + + if [ "$osType" != "Darwin" ]; then + ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} + else + mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} + fi + ${csudo}ln -s ${log_dir} ${install_main_dir}/log +} + +function install_connector() { + ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function update_TDengine() { + # Start to update + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + tar -zxf ${tarName} + + echo -e "${GREEN}Start to update ${productName} client...${NC}" + # Stop the client shell if running + if pidof ${clientName} &> /dev/null; then + kill_client + sleep 1 + fi + + install_main_path + + install_log + install_header + install_lib + install_jemalloc + if [ "$verMode" == "cluster" ]; then + install_connector + fi + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}" + + rm -rf $(tar -tf ${tarName}) +} + +function install_TDengine() { + # Start to install + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + tar -zxf ${tarName} + + echo -e "${GREEN}Start to install ${productName} client...${NC}" + + install_main_path + install_log + install_header + install_lib + install_jemalloc + if [ "$verMode" == "cluster" ]; then + install_connector + fi + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}" + + rm -rf $(tar -tf ${tarName}) +} + + +## ==============================Main program starts from here============================ +# Install or updata client and client +# if server is already install, don't install client + if [ -e ${bin_dir}/${serverName} ]; then + echo -e "\033[44;32;1mThere are already installed ${productName} server, so don't need install client!${NC}" + exit 0 + fi + + if [ -x ${bin_dir}/${clientName} ]; then + update_flag=1 + update_TDengine + else + install_TDengine + fi diff --git a/packaging/make_install.sh b/packaging/tools/make_install.sh similarity index 99% rename from packaging/make_install.sh rename to packaging/tools/make_install.sh index e3e22a655f..31cb5e87b9 100755 --- a/packaging/make_install.sh +++ b/packaging/tools/make_install.sh @@ -186,7 +186,7 @@ function install_bin() { [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || : - ${csudo}cp -r ${binary_dir}/build/bin/tarbitrator ${install_main_dir}/bin || : + # ${csudo}cp -r ${binary_dir}/build/bin/tarbitrator ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || : @@ -372,15 +372,15 @@ function install_config() { if [ ! -f ${cfg_install_dir}/${configFile} ]; then ${csudo}mkdir -p ${cfg_install_dir} [ -f ${script_dir}/../cfg/${configFile} ] && - ${csudo}cp ${script_dir}/../cfg/${configFile} ${cfg_install_dir} || : + ${csudo}cp ${script_dir}/../cfg/${configFile} ${cfg_install_dir} ${csudo}chmod 644 ${cfg_install_dir}/${configFile} ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ - ${cfg_install_dir}/${configFile}.${verNumber} || : + ${cfg_install_dir}/${configFile}.${verNumber} ${csudo}ln -s ${cfg_install_dir}/${configFile} \ ${install_main_dir}/cfg/${configFile} else ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ - ${cfg_install_dir}/${configFile}.${verNumber} || : + ${cfg_install_dir}/${configFile}.${verNumber} fi } @@ -475,10 +475,10 @@ function install_service_on_sysvinit() { if ((${os_type} == 1)); then # ${csudo}cp -f ${script_dir}/../deb/${serverName} ${install_main_dir}/init.d - ${csudo}cp ${script_dir}/../deb/${serverName} ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/${serverName} || : + ${csudo}cp ${script_dir}/../deb/${serverName} ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/${serverName} elif ((${os_type} == 2)); then # ${csudo}cp -f ${script_dir}/../rpm/${serverName} ${install_main_dir}/init.d - ${csudo}cp ${script_dir}/../rpm/${serverName} ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/${serverName} || : + ${csudo}cp ${script_dir}/../rpm/${serverName} ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/${serverName} fi if ((${initd_mod} == 1)); then diff --git a/packaging/tools/makearbi.sh b/packaging/tools/makearbi.sh new file mode 100644 index 0000000000..f0cc54a355 --- /dev/null +++ b/packaging/tools/makearbi.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +productName="TDengine" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/${productName}-enterprise-arbitrator-${version}" +else + install_dir="${release_dir}/${productName}-arbitrator-${version}" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh" +install_files="${script_dir}/install_arbi.sh" + +#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taosdef.h ${code_dir}/inc/taoserror.h" +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi.sh || : +#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +# install_dir has been distinguishes cluster from edege, so comments this code +pkg_name=${install_dir}-${osType}-${cpuType} + +if [[ "$verType" == "beta" ]] || [[ "$verType" == "preRelease" ]]; then + pkg_name=${install_dir}-${verType}-${osType}-${cpuType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh new file mode 100644 index 0000000000..a86f62a32c --- /dev/null +++ b/packaging/tools/makeclient.sh @@ -0,0 +1,246 @@ +#!/bin/bash +# +# Generate tar.gz package for linux client in all os system +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 +dbName=$9 + +productName="TDengine" +clientName="taos" +configFile="taos.cfg" +tarName="taos.tar.gz" + +if [ "$osType" != "Darwin" ]; then + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/../..)" +else + script_dir=$(dirname $0) + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/../.. +fi + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' + +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/${productName}-enterprise-client-${version}" +else + install_dir="${release_dir}/${productName}-client-${version}" +fi + +# Directories and files. + +if [ "$osType" != "Darwin" ]; then + if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/${clientName} + bin_files="${build_dir}/bin/${clientName} \ + ${script_dir}/remove_client.sh" + else + bin_files="${build_dir}/bin/${clientName} \ + ${script_dir}/remove_client.sh \ + ${script_dir}/set_core.sh \ + ${script_dir}/get_client.sh" + fi + lib_files="${build_dir}/lib/libtaos.so.${version}" +else + bin_files="${build_dir}/bin/${clientName} ${script_dir}/remove_client.sh" + lib_files="${build_dir}/lib/libtaos.${version}.dylib" +fi + +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taosdef.h ${code_dir}/inc/taoserror.h" +if [ "$dbName" != "taos" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + +install_files="${script_dir}/install_client.sh" + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* + +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + +cd ${install_dir} + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f ${tarName} * --remove-files || : +else + tar -zcv -f ${tarName} * || : + mv ${tarName} .. + rm -rf ./* + mv ../${tarName} . +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$osType" == "Darwin" ]; then + sed 's/osType=Linux/osType=Darwin/g' ${install_dir}/install_client.sh >>install_client_temp.sh + mv install_client_temp.sh ${install_dir}/install_client.sh +fi + +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install_client.sh >>install_client_temp.sh + mv install_client_temp.sh ${install_dir}/install_client.sh +fi + +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install_client.sh >>install_client_temp.sh + mv install_client_temp.sh ${install_dir}/install_client.sh +fi +chmod a+x ${install_dir}/install_client.sh + +if [[ $productName == "TDengine" ]]; then + # Copy example code + mkdir -p ${install_dir}/examples + examples_dir="${top_dir}/examples" + cp -r ${examples_dir}/c ${install_dir}/examples + if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp -r ${examples_dir}/JDBC ${install_dir}/examples + cp -r ${examples_dir}/matlab ${install_dir}/examples + cp -r ${examples_dir}/python ${install_dir}/examples + cp -r ${examples_dir}/R ${install_dir}/examples + cp -r ${examples_dir}/go ${install_dir}/examples + cp -r ${examples_dir}/nodejs ${install_dir}/examples + cp -r ${examples_dir}/C# ${install_dir}/examples + mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../src/kit/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json + fi + + if [ "$verMode" == "cluster" ]; then + # Copy connector + connector_dir="${code_dir}/connector" + mkdir -p ${install_dir}/connector + if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ "$osType" != "Darwin" ]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + git clone --depth 1 https://github.com/taosdata/taos-connector-python ${install_dir}/connector/python + rm -rf ${install_dir}/connector/python/.git ||: +# cp -r ${connector_dir}/python ${install_dir}/connector + git clone --depth 1 https://github.com/taosdata/taos-connector-node ${install_dir}/connector/nodejs + rm -rf ${install_dir}/connector/nodejs/.git ||: + + git clone --depth 1 https://github.com/taosdata/taos-connector-dotnet ${install_dir}/connector/dotnet + rm -rf ${install_dir}/connector/dotnet/.git ||: +# cp -r ${connector_dir}/nodejs ${install_dir}/connector + git clone --depth 1 https://github.com/taosdata/libtaos-rs ${install_dir}/connector/rust + rm -rf ${install_dir}/connector/rust/.git ||: + fi + fi +fi +# Copy driver +mkdir -p ${install_dir}/driver +cp ${lib_files} ${install_dir}/driver + +# Copy connector +connector_dir="${code_dir}/connector" +mkdir -p ${install_dir}/connector + +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ "$osType" != "Darwin" ]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector + cp -r ${connector_dir}/nodejs ${install_dir}/connector +fi +# Copy release note +# cp ${script_dir}/release_note ${install_dir} + +# exit 1 + +cd ${release_dir} + +# install_dir has been distinguishes cluster from edege, so comments this code +pkg_name=${install_dir}-${osType}-${cpuType} + +# if [ "$verMode" == "cluster" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# elif [ "$verMode" == "edge" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# else +# echo "unknow verMode, nor cluster or edge" +# exit 1 +# fi + +if [[ "$verType" == "beta" ]] || [[ "$verType" == "preRelease" ]]; then + pkg_name=${install_dir}-${verType}-${osType}-${cpuType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +else + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) || : + mv "$(basename ${pkg_name}).tar.gz" .. + rm -rf ./* + mv ../"$(basename ${pkg_name}).tar.gz" . +fi + +cd ${curr_dir} diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh new file mode 100644 index 0000000000..b4e60c13f5 --- /dev/null +++ b/packaging/tools/makepkg.sh @@ -0,0 +1,378 @@ +#!/bin/bash +# +# Generate tar.gz package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 +versionComp=$9 +dbName=${10} + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +productName="TDengine" +serverName="taosd" +clientName="taos" +configFile="taos.cfg" +tarName="taos.tar.gz" +dumpName="taosdump" +benchmarkName="taosBenchmark" +toolsName="taostools" +adapterName="taosadapter" +defaultPasswd="taosdata" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/${productName}-enterprise-server-${version}" +else + install_dir="${release_dir}/${productName}-server-${version}" +fi + +if [ -d ${top_dir}/src/kit/taos-tools/packaging/deb ]; then + cd ${top_dir}/src/kit/taos-tools/packaging/deb + [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" + + taostools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}') + taostools_install_dir="${release_dir}/${clientName}Tools-${taostools_ver}" + + cd ${curr_dir} +else + taostools_install_dir="${release_dir}/${clientName}Tools-${version}" +fi + +# Directories and files +if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/${serverName} + strip ${build_dir}/bin/${clientName} + # lite version doesn't include taosadapter, which will lead to no restful interface + bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark" + taostools_bin_files="" +else + + wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ + && echo "TDinsight.sh downloaded!" \ + || echo "failed to download TDinsight.sh" + # download TDinsight caches + orig_pwd=$(pwd) + tdinsight_caches="" + cd ${build_dir}/bin/ && \ + chmod +x TDinsight.sh + tdinsight_caches=$(./TDinsight.sh --download-only | xargs -i printf "${build_dir}/bin/{} ") + cd $orig_pwd + echo "TDinsight caches: $tdinsight_caches" + + taostools_bin_files=" ${build_dir}/bin/taosdump \ + ${build_dir}/bin/taosBenchmark \ + ${build_dir}/bin/TDinsight.sh \ + $tdinsight_caches" + + bin_files="${build_dir}/bin/${serverName} \ + ${build_dir}/bin/${clientName} \ + ${taostools_bin_files} \ + ${build_dir}/bin/taosadapter \ + ${build_dir}/bin/tarbitrator\ + ${script_dir}/remove.sh \ + ${script_dir}/set_core.sh \ + ${script_dir}/run_taosd_and_taosadapter.sh \ + ${script_dir}/startPre.sh \ + ${script_dir}/taosd-dump-cfg.gdb" +fi + +lib_files="${build_dir}/lib/libtaos.so.${version}" +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taosdef.h ${code_dir}/inc/taoserror.h" + +if [ "$dbName" != "taos" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + +install_files="${script_dir}/install.sh" +nginx_dir="${code_dir}/../../enterprise/src/plugins/web" + +init_file_deb=${script_dir}/../deb/taosd +init_file_rpm=${script_dir}/../rpm/taosd +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} + +if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then + cp ${compile_dir}/test/cfg/taosadapter.toml ${install_dir}/cfg || : +fi + +if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then + cp ${compile_dir}/test/cfg/taosadapter.service ${install_dir}/cfg || : +fi + +if [ -f "${cfg_dir}/${serverName}.service" ]; then + cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || : +fi + +if [ -f "${top_dir}/packaging/cfg/tarbitratord.service" ]; then + cp ${top_dir}/packaging/cfg/tarbitratord.service ${install_dir}/cfg || : +fi + +if [ -f "${top_dir}/packaging/cfg/nginxd.service" ]; then + cp ${top_dir}/packaging/cfg/nginxd.service ${install_dir}/cfg || : +fi + +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb +mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +if [ $adapterName != "taosadapter" ]; then + mv ${install_dir}/cfg/taosadapter.toml ${install_dir}/cfg/$adapterName.toml + sed -i "s/path = \"\/var\/log\/taos\"/path = \"\/var\/log\/${productName}\"/g" ${install_dir}/cfg/$adapterName.toml + sed -i "s/password = \"taosdata\"/password = \"${defaultPasswd}\"/g" ${install_dir}/cfg/$adapterName.toml + + mv ${install_dir}/cfg/taosadapter.service ${install_dir}/cfg/$adapterName.service + sed -i "s/TDengine/${productName}/g" ${install_dir}/cfg/$adapterName.service + sed -i "s/taosAdapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service + sed -i "s/taosadapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service + + mv ${install_dir}/bin/taosadapter ${install_dir}/bin/${adapterName} + mv ${install_dir}/bin/run_taosd_and_taosadapter.sh ${install_dir}/bin/run_${serverName}_and_${adapterName}.sh + mv ${install_dir}/bin/taosd-dump-cfg.gdb ${install_dir}/bin/${serverName}-dump-cfg.gdb +fi + +if [ -n "${taostools_bin_files}" ]; then + mkdir -p ${taostools_install_dir} || echo -e "failed to create ${taostools_install_dir}" + mkdir -p ${taostools_install_dir}/bin \ + && cp ${taostools_bin_files} ${taostools_install_dir}/bin \ + && chmod a+x ${taostools_install_dir}/bin/* || : + + if [ -f ${top_dir}/src/kit/taos-tools/packaging/tools/install-taostools.sh ]; then + cp ${top_dir}/src/kit/taos-tools/packaging/tools/install-taostools.sh \ + ${taostools_install_dir}/ > /dev/null \ + && chmod a+x ${taostools_install_dir}/install-taostools.sh \ + || echo -e "failed to copy install-taostools.sh" + else + echo -e "install-taostools.sh not found" + fi + + if [ -f ${top_dir}/src/kit/taos-tools/packaging/tools/uninstall-taostools.sh ]; then + cp ${top_dir}/src/kit/taos-tools/packaging/tools/uninstall-taostools.sh \ + ${taostools_install_dir}/ > /dev/null \ + && chmod a+x ${taostools_install_dir}/uninstall-taostools.sh \ + || echo -e "failed to copy uninstall-taostools.sh" + else + echo -e "uninstall-taostools.sh not found" + fi + + if [ -f ${build_dir}/lib/libavro.so.23.0.0 ]; then + mkdir -p ${taostools_install_dir}/avro/{lib,lib/pkgconfig} || echo -e "failed to create ${taostools_install_dir}/avro" + cp ${build_dir}/lib/libavro.* ${taostools_install_dir}/avro/lib + cp ${build_dir}/lib/pkgconfig/avro-c.pc ${taostools_install_dir}/avro/lib/pkgconfig + fi +fi + +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh + mv remove_temp.sh ${install_dir}/bin/remove.sh + + mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd + cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png + rm -rf ${install_dir}/nginxd/png + + if [ "$cpuType" == "aarch64" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/ + elif [ "$cpuType" == "aarch32" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/32bit/nginx ${install_dir}/nginxd/sbin/ + fi + rm -rf ${install_dir}/nginxd/sbin/arm +fi + +cd ${install_dir} +tar -zcv -f ${tarName} * --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${tarName} error !!!" + exit $exitcode +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install.sh >>install_temp.sh + mv install_temp.sh ${install_dir}/install.sh +fi +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install.sh >>install_temp.sh + mv install_temp.sh ${install_dir}/install.sh +fi +chmod a+x ${install_dir}/install.sh + +if [[ $dbName == "taos" ]]; then + # Copy example code + mkdir -p ${install_dir}/examples + examples_dir="${top_dir}/examples" + cp -r ${examples_dir}/c ${install_dir}/examples + if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ -d ${examples_dir}/JDBC/connectionPools/target ]; then + rm -rf ${examples_dir}/JDBC/connectionPools/target + fi + if [ -d ${examples_dir}/JDBC/JDBCDemo/target ]; then + rm -rf ${examples_dir}/JDBC/JDBCDemo/target + fi + if [ -d ${examples_dir}/JDBC/mybatisplus-demo/target ]; then + rm -rf ${examples_dir}/JDBC/mybatisplus-demo/target + fi + if [ -d ${examples_dir}/JDBC/springbootdemo/target ]; then + rm -rf ${examples_dir}/JDBC/springbootdemo/target + fi + if [ -d ${examples_dir}/JDBC/SpringJdbcTemplate/target ]; then + rm -rf ${examples_dir}/JDBC/SpringJdbcTemplate/target + fi + if [ -d ${examples_dir}/JDBC/taosdemo/target ]; then + rm -rf ${examples_dir}/JDBC/taosdemo/target + fi + + cp -r ${examples_dir}/JDBC ${install_dir}/examples + cp -r ${examples_dir}/matlab ${install_dir}/examples + cp -r ${examples_dir}/python ${install_dir}/examples + cp -r ${examples_dir}/R ${install_dir}/examples + cp -r ${examples_dir}/go ${install_dir}/examples + cp -r ${examples_dir}/nodejs ${install_dir}/examples + cp -r ${examples_dir}/C# ${install_dir}/examples + mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../src/kit/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json + fi +fi + +# Copy driver +mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt + +# Copy connector +if [ "$verMode" == "cluster" ]; then + connector_dir="${code_dir}/connector" + mkdir -p ${install_dir}/connector + if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + git clone --depth 1 https://github.com/taosdata/taos-connector-python ${install_dir}/connector/python + rm -rf ${install_dir}/connector/python/.git ||: + + git clone --depth 1 https://github.com/taosdata/taos-connector-node ${install_dir}/connector/nodejs + rm -rf ${install_dir}/connector/nodejs/.git ||: + + git clone --depth 1 https://github.com/taosdata/taos-connector-dotnet ${install_dir}/connector/dotnet + rm -rf ${install_dir}/connector/dotnet/.git ||: + + git clone --depth 1 https://github.com/taosdata/libtaos-rs ${install_dir}/connector/rust + rm -rf ${install_dir}/connector/rust/.git ||: + # cp -r ${connector_dir}/python ${install_dir}/connector + # cp -r ${connector_dir}/nodejs ${install_dir}/connector + fi +fi + +# Copy release note +cp ${script_dir}/release_note ${install_dir} + +# exit 1 + +cd ${release_dir} + +# install_dir has been distinguishes cluster from edege, so comments this code +pkg_name=${install_dir}-${osType}-${cpuType} + +taostools_pkg_name=${taostools_install_dir}-${osType}-${cpuType} + +# if [ "$verMode" == "cluster" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# elif [ "$verMode" == "edge" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# else +# echo "unknow verMode, nor cluster or edge" +# exit 1 +# fi + +if [[ "$verType" == "beta" ]] || [[ "$verType" == "preRelease" ]]; then + pkg_name=${install_dir}-${verType}-${osType}-${cpuType} + taostools_pkg_name=${taostools_install_dir}-${verType}-${osType}-${cpuType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} + taostools_pkg_name=${taostools_pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" "$(basename ${install_dir})" --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +if [ -n "${taostools_bin_files}" ]; then + wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh" + tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || : + exitcode=$? + if [ "$exitcode" != "0" ]; then + echo "tar ${taostools_pkg_name}.tar.gz error !!!" + exit $exitcode + fi +fi + +cd ${curr_dir} diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh new file mode 100644 index 0000000000..59315dbe99 --- /dev/null +++ b/packaging/tools/post.sh @@ -0,0 +1,539 @@ +#!/bin/bash +# +# This file is used to install tdengine rpm package on centos systems. The operating system +# is required to use systemd to manage services at boot +# set -x + +iplist="" +serverFqdn="" + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) +# Dynamic directory +data_dir="/var/lib/taos" +log_dir="/var/log/taos" +data_link_dir="/usr/local/taos/data" +log_link_dir="/usr/local/taos/log" +install_main_dir="/usr/local/taos" + +# static directory +cfg_dir="/usr/local/taos/cfg" +bin_dir="/usr/local/taos/bin" +lib_dir="/usr/local/taos/driver" +init_d_dir="/usr/local/taos/init.d" +inc_dir="/usr/local/taos/include" + +cfg_install_dir="/etc/taos" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" + + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_taosadapter() { +# ${csudo}pkill -f taosadapter || : + pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function kill_taosd() { +# ${csudo}pkill -f taosd || : + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function install_include() { + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h|| : + ${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_lib() { + ${csudo}rm -f ${lib_link_dir}/libtaos* || : + ${csudo}rm -f ${lib64_link_dir}/libtaos* || : + + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + + ${csudo}ldconfig +} + +function install_bin() { + # Remove links + ${csudo}rm -f ${bin_link_dir}/taos || : + ${csudo}rm -f ${bin_link_dir}/taosd || : + ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : + ${csudo}rm -f ${bin_link_dir}/taosdemo || : + ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/rmtaos || : + ${csudo}rm -f ${bin_link_dir}/set_core || : + + ${csudo}chmod 0555 ${bin_dir}/* + + #Make link + [ -x ${bin_dir}/taos ] && ${csudo}ln -s ${bin_dir}/taos ${bin_link_dir}/taos || : + [ -x ${bin_dir}/taosd ] && ${csudo}ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : + [ -x ${bin_dir}/taosadapter ] && ${csudo}ln -s ${bin_dir}/taosadapter ${bin_link_dir}/taosadapter || : + [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosdemo || : + [ -x ${bin_dir}/TDinsight.sh ] && ${csudo}ln -sf ${bin_dir}/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : + [ -x ${bin_dir}/taosdump ] && ${csudo}ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : + [ -x ${bin_dir}/set_core.sh ] && ${csudo}ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : +} + +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in "${arr[@]}" + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo}echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo}hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo}echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in "${arr[@]}" + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + +function install_taosadapter_config() { + if [ ! -f "${cfg_install_dir}/taosadapter.toml" ]; then + [ ! -d %{cfg_install_dir} ] && + ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/taosadapter.toml ] && ${csudo}cp ${cfg_dir}/taosadapter.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/taosadapter.toml ] && + ${csudo}chmod 644 ${cfg_install_dir}/taosadapter.toml + fi + + [ -f ${cfg_dir}/taosadapter.toml ] && + ${csudo}mv ${cfg_dir}/taosadapter.toml ${cfg_dir}/taosadapter.toml.new + + [ -f ${cfg_install_dir}/taosadapter.toml ] && + ${csudo}ln -s ${cfg_install_dir}/taosadapter.toml ${cfg_dir} +} + +function install_config() { + if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then + ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/taos.cfg ] && ${csudo}cp ${cfg_dir}/taos.cfg ${cfg_install_dir} + ${csudo}chmod 644 ${cfg_install_dir}/* + fi + + # Save standard input to 6 and open / dev / TTY on standard input + exec 6<&0 0 ${email_file}" + break + #else + # read -p "Please enter the correct email address: " emailAddr + #fi + else + break + fi + done +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo}sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof taosd &> /dev/null; then + ${csudo}service taosd stop || : + fi + + if ((${initd_mod}==1)); then + ${csudo}chkconfig --del taosd || : + elif ((${initd_mod}==2)); then + ${csudo}insserv -r taosd || : + elif ((${initd_mod}==3)); then + ${csudo}update-rc.d -f taosd remove || : + fi + + ${csudo}rm -f ${service_config_dir}/taosd || : + + if $(which init &> /dev/null); then + ${csudo}init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + + sleep 1 + + # Install taosd service + ${csudo}cp %{init_d_dir}/taosd ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/taosd + + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo}grep -q -F "$restart_config_str" /etc/inittab || ${csudo}bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo}chkconfig --add taosd || : + ${csudo}chkconfig --level 2345 taosd on || : + elif ((${initd_mod}==2)); then + ${csudo}insserv taosd || : + ${csudo}insserv -d taosd || : + elif ((${initd_mod}==3)); then + ${csudo}update-rc.d taosd defaults || : + fi +} + +function clean_service_on_systemd() { + taosd_service_config="${service_config_dir}/taosd.service" + + # taosd service already is stoped before install in preinst script + #if systemctl is-active --quiet taosd; then + # echo "TDengine is running, stopping it..." + # ${csudo}systemctl stop taosd &> /dev/null || echo &> /dev/null + #fi + ${csudo}systemctl disable taosd &> /dev/null || echo &> /dev/null + + ${csudo}rm -f ${taosd_service_config} +} + +# taos:2345:respawn:/etc/init.d/taosd start + +function install_service_on_systemd() { + clean_service_on_systemd + + taosd_service_config="${service_config_dir}/taosd.service" + + ${csudo}bash -c "echo '[Unit]' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'After=network-online.target' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'Wants=network-online.target' >> ${taosd_service_config}" + ${csudo}bash -c "echo >> ${taosd_service_config}" + ${csudo}bash -c "echo '[Service]' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'Type=simple' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'Restart=always' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}" + ${csudo}bash -c "echo >> ${taosd_service_config}" + ${csudo}bash -c "echo '[Install]' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" + ${csudo}systemctl enable taosd +} + +function install_taosadapter_service() { + if ((${service_mod}==0)); then + [ -f ${script_dir}/../cfg/taosadapter.service ] &&\ + ${csudo}cp ${script_dir}/../cfg/taosadapter.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + fi +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # manual start taosd + kill_taosadapter + kill_taosd + fi +} + +function install_TDengine() { + echo -e "${GREEN}Start to install TDengine...${NC}" + + #install log and data dir , then ln to /usr/local/taos + ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} + ${csudo}mkdir -p ${data_dir} + + ${csudo}rm -rf ${log_link_dir} || : + ${csudo}rm -rf ${data_link_dir} || : + + ${csudo}ln -s ${log_dir} ${log_link_dir} || : + ${csudo}ln -s ${data_dir} ${data_link_dir} || : + + # Install include, lib, binary and service + install_include + install_lib + install_bin + install_config + install_taosadapter_config + install_taosadapter_service + install_service + + # Ask if to start the service + #echo + #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}systemctl start taosd${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}" + echo -e " : ${csudo}service taosd start ${RED} after${NC}" + else + echo -e "${GREEN_DARK}To start TDengine ${NC}: ./taosd${NC}" + fi + + + + if [ ! -z "$firstEp" ]; then + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" + echo + fi + echo + echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" +} + + +## ==============================Main program starts from here============================ +serverFqdn=$(hostname) +install_TDengine diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh new file mode 100644 index 0000000000..bc1727c4e4 --- /dev/null +++ b/packaging/tools/preun.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TSDB + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +data_link_dir="/usr/local/taos/data" +log_link_dir="/usr/local/taos/log" +cfg_link_dir="/usr/local/taos/cfg" + +service_config_dir="/etc/systemd/system" +taos_service_name="taosd" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_taosadapter() { + pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function kill_taosd() { + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function clean_service_on_systemd() { + taosadapter_service_config="${service_config_dir}/taosadapter.service" + if systemctl is-active --quiet taosadapter; then + echo "taosadapter is running, stopping it..." + ${csudo}systemctl stop taosadapter &> /dev/null || echo &> /dev/null + fi + + taosd_service_config="${service_config_dir}/${taos_service_name}.service" + + if systemctl is-active --quiet ${taos_service_name}; then + echo "TDengine taosd is running, stopping it..." + ${csudo}systemctl stop ${taos_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo}systemctl disable ${taos_service_name} &> /dev/null || echo &> /dev/null + + ${csudo}rm -f ${taosd_service_config} + + [ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config} + +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo}sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof taosd &> /dev/null; then + echo "TDengine taosd is running, stopping it..." + ${csudo}service taosd stop || : + fi + + if ((${initd_mod}==1)); then + ${csudo}chkconfig --del taosd || : + elif ((${initd_mod}==2)); then + ${csudo}insserv -r taosd || : + elif ((${initd_mod}==3)); then + ${csudo}update-rc.d -f taosd remove || : + fi + + ${csudo}rm -f ${service_config_dir}/taosd || : + + if $(which init &> /dev/null); then + ${csudo}init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop taosd + kill_taosadapter + kill_taosd + fi +} + +# Stop service and disable booting start. +clean_service + +# Remove all links +${csudo}rm -f ${bin_link_dir}/taos || : +${csudo}rm -f ${bin_link_dir}/taosd || : +${csudo}rm -f ${bin_link_dir}/taosadapter || : +${csudo}rm -f ${bin_link_dir}/taosBenchmark || : +${csudo}rm -f ${bin_link_dir}/taosdemo || : +${csudo}rm -f ${bin_link_dir}/set_core || : +${csudo}rm -f ${cfg_link_dir}/*.new || : +${csudo}rm -f ${inc_link_dir}/taos.h || : +${csudo}rm -f ${inc_link_dir}/taosdef.h || : +${csudo}rm -f ${inc_link_dir}/taoserror.h || : +${csudo}rm -f ${lib_link_dir}/libtaos.* || : +${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + +${csudo}rm -f ${log_link_dir} || : +${csudo}rm -f ${data_link_dir} || : + +if ((${service_mod}==2)); then + kill_taosadapter + kill_taosd +fi + +echo -e "${GREEN}TDengine is removed successfully!${NC}" diff --git a/packaging/tools/release_note b/packaging/tools/release_note new file mode 100644 index 0000000000..4a954b1f12 --- /dev/null +++ b/packaging/tools/release_note @@ -0,0 +1,136 @@ +taos-1.6.4.0 (Release on 2019-12-01) +Bug fixed: + 1.Look for possible causes of file corruption and fix them + 2.Encapsulate memory allocation functions to reduce the possibility of crashes + 3.Increase Arm64 compilation options + 4.Remove most of the warnings in the code + 5.Provide a variety of connector usage documents + 6.Network connection can be selected in udp and tcp + 7.Allow the maximum number of Tags to be 32 + 8.Bugs reported by the user + +taos-1.5.2.6 (Release on 2019-05-13) +Bug fixed: + - Nchar strings sometimes were wrongly truncated on Window + - Importing data from file throws an error of "invalid SQL" + +taos-1.5.2.5 (Release on 2019-05-13) +Bug fixed: + - Long timespan data import sometimes affects query result + - Synchronzation of cluster dnodes worked incorrectly when importing + +taos-1.5.2.4 (Release on 2019-05-10) +New Features: + - Optimized Windows client installation: now users don't need to copy taos.dll manually + - Changed the priority of taos.cfg and JDBC URL: parameters in JDCB URL now has a higher priority than parameters in taos.cfg +Bug fixed: + - Expired data files were not deleted corrected + - Occasionally importing returned "affected rows" which larger than 0, but 0 row was actually written into db + - Commit log is occupied by too many import-to-file requests, which blocked further data importing + - Cloud service shows a wrong number of available days with current balance + - Other minor issues + +taos-1.5.1 (Release on 2019-04-09) +New Features: + - Maximum number of rows returned by "top/bottom" methods increased from 20 to 100 + - Improved the performance of "first/last" methods + - Increased system stability +Bug fixed: + - Connection failure when query on huge STables through TPC + - The primary timestamp is occasionally returned as NULL in some queries + - Operation failure when updating a tag value to NULL + - Stream calculation couldn't start at certain occasions + +taos-1.5.0 (Release on 2019-03-11) +New Features: + - New syntax to automatically create tables when inserting values into non-existing tables + - New syntax "slimit/soffset" to pagenate groups in a query result set + - Support "top/bottom" queries on a supertable + - High performance statistic aggregation function "apercentile" + - Remove "first_t/last_t" functions; improve the performance of "first/last" function + - Add pre-aggregation for bool type values + - Supports fixed-length streaming computation, i.e. users may define an end time for a stream + - New JAVA API for SQL subscription, supports table/supertable/SQL query subscription +Bug fixed: + - Data file broken issue when frequently using "import" + - Using "spread" on a super table may return negative values + - RPC bug that random network packets might cause the RPC module to crush + +taos-1.4.15 (Released on 2019-01-23) +New Features: + - JDBC Driver now supports configuring timezone, locale, cfgdir in JDBC url + - A new API is added to validate if a table creation sql statement is correct in syntax without actually creating that table +Bugs Fixed: + - "select last(*) from STable" sometimest returned incorrect number of rows + - JDBC driver method ResultSetMetaData.getColumnClassName() returned wrong values. + - Web shell automatically changed query string to lower case + +taos-1.4.14 (Released on 2018-12-22) +New Features: + - C Driver support for integration with Python + - JDBC Driver support for integration with R and MATLAB + +taos-1.4.13 (Released on 2018-12-14) +Bugs Fixed: + - Clients failed to connect to server due to unexpected and invalid packets recieved by the server. +Features Added: + - Add support to HikariCP in TSDB JDBC driver. + +taos-1.4.12 (Released on 2018-12-08) +Bugs Fixed: + - Querying data while inserting into the database might return incomplete resultsets. +Features Added: + - A new python driver is added. + - Increased system stability. + - Changed meaning of database configuration paramerter 'ablocks'. 'ablocks' used to refer to the number of total cache blocks in memory, now it refers to average number of cache blocks for each table in memory. + +taos-1.4.11 (Released on 2018-11-23) +Bugs Fixed: + - Thread memory leaking during high-frequency committing. + - Master dnode selection failure caused by accidental network issues. +Features Added: + - Change keyword "metrics" to "stables", i.e. supertables; the previous query "show metrics" is now changed to "show stables". + - Add an error message mechanism in C# driver. An error with message "Failed to connect to server" is thrown when fetching data experienced a network connection interruption during data transmitting. + +taos-1.4.10 (Released on 2018-11-13) +Bugs Fixed: + - Taosdump failed while exporting extremely large datasets to a .sql file. + - Commit status did not change correctly if the last commit was triggered by commit threshold time (ctime) and no more new data was written to DB during the next ctime period. +Features Added: + - Support importing historical data from Telegraf interface. + - Support MyBatis framework in TSDB JDBC Driver. + - Change result set row indexing in JDBC Driver. Result set row indexes now starts from 1 instead of 0. + +taos-1.4.9 (Released on 2018-11-02) +Bugs Fixed: + - Dumping data using UTF-8 format in client shell failed. + - Tag query failed using C# Driver. + - Committing data to disk failed if DB files were corrupted. + - Continuously pressing Ctrl+c in client shell for multiple times produced a segmentation fault. +Features Added: + - Changed the display pattern in shell for taosdump. + - Add a check to the status of an existing resultset before firing a new query in a single JDBC connection. A connection can only have a single open resultset, and the resultset must be closed before one can execute new queries. + + +taos-1.4.7 (Released on 2018-10-25) +Bug Fixed: + - UTF-8 encoding in JDBC Driver did not give the correct Chinese characters. + - Fix crash error when where clause is too long. +Features Added: + - Add check on database properties, force ablocks to be at least (4 * tables) in a vnode. + - Check if pVgroup is empty in sdb. + +taos-1.4.6 (Released on 2018-10-21) +Bug Fixed: + - Fix wrong symbol addition while export csv file. +Features Added: + - Update grafana plugins. + - Update python drivers. + - Add error code explanation in JDBC Driver. + - Prohibit login while the version of server and client are not match. + +taos-1.4.5 (Released on 2018-10-17) +Bug Fixed: + - Fix HTTP request truncation bug in Telegraf interface. +Features Added: + - Support nchar and null object in JDBC Driver. diff --git a/packaging/remove.sh b/packaging/tools/remove.sh similarity index 100% rename from packaging/remove.sh rename to packaging/tools/remove.sh diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh new file mode 100644 index 0000000000..9b8e49d511 --- /dev/null +++ b/packaging/tools/remove_arbi.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TDengine's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tarbitrator" +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo}rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${inc_link_dir}/taosdef.h || : + ${csudo}rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo}rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo}systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo}systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo}rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "TDengine's tarbitrator is running, stopping it..." + ${csudo}service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo}rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo}init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo}rm -rf ${install_main_dir} + +echo -e "${GREEN}TDengine's arbitrator is removed successfully!${NC}" diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh new file mode 100644 index 0000000000..9e22f894ac --- /dev/null +++ b/packaging/tools/remove_client.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Script to stop the client and uninstall database, but retain the config and log files. +set -e +# set -x + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +installDir="/usr/local/taos" +clientName="taos" +uninstallScript="rmtaos" + +#install main path +install_main_dir=${installDir} + +log_link_dir=${installDir}/log +cfg_link_dir=${installDir}/cfg +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +function kill_client() { + if [ -n "$(pidof ${clientName})" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo}rm -f ${bin_link_dir}/${clientName} || : + ${csudo}rm -f ${bin_link_dir}/taosdemo || : + ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo}rm -rf ${v15_java_app_dir} || : +} + +function clean_header() { + # Remove link + ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${inc_link_dir}/taosdef.h || : + ${csudo}rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo}rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo}rm -rf ${log_link_dir} || : +} + +# Stop client. +kill_client +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config + +${csudo}rm -rf ${install_main_dir} + +echo -e "${GREEN}TDengine client is removed successfully!${NC}" +echo diff --git a/packaging/tools/repair_link.sh b/packaging/tools/repair_link.sh new file mode 100644 index 0000000000..8507b639ba --- /dev/null +++ b/packaging/tools/repair_link.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# This script is used to repaire links when you what to move TDengine +# data to other places and to access data. + +# Read link path +read -p "Please enter link directory such as /var/lib/taos/tsdb: " linkDir + +while true; do + if [ ! -d $linkDir ]; then + read -p "Paht not exists, please enter the correct link path:" linkDir + continue + fi + break +done + +declare -A dirHash + +for linkFile in $(find -L $linkDir -xtype l); do + targetFile=$(readlink -f $linkFile) + echo "targetFile: ${targetFile}" + # TODO : Extract directory part and basename part + dirName=$(dirname $(dirname ${targetFile})) + baseName=$(basename $(dirname ${targetFile}))/$(basename ${targetFile}) + + # TODO : + newDir="${dirHash["$dirName"]}" + if [ -z "${dirHash["$dirName"]}" ]; then + read -p "Please enter the directory to replace ${dirName}:" newDir + + read -p "Do you want to replcace all[y/N]?" replcaceAll + if [[ ( "${replcaceAll}" == "y") || ( "${replcaceAll}" == "Y") ]]; then + dirHash["$dirName"]="$newDir" + fi + fi + + # Replcace the file + ln -sf "${newDir}/${baseName}" "${linkFile}" +done diff --git a/packaging/tools/run_taosd_and_taosadapter.sh b/packaging/tools/run_taosd_and_taosadapter.sh new file mode 100644 index 0000000000..3dfab87d72 --- /dev/null +++ b/packaging/tools/run_taosd_and_taosadapter.sh @@ -0,0 +1,3 @@ +#!/bin/bash +[[ -x /usr/bin/taosadapter ]] && /usr/bin/taosadapter & +taosd diff --git a/packaging/tools/set_core.sh b/packaging/tools/set_core.sh new file mode 100644 index 0000000000..41e8aebec3 --- /dev/null +++ b/packaging/tools/set_core.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# This file is used to set config for core when taosd crash + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +# set -e +# set -x +corePath=$1 + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +if [[ ! -n ${corePath} ]]; then + echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:" + read corePath + while true; do + if [[ ! -z "$corePath" ]]; then + break + else + read -p "Please enter a file directory to save the coredump file:" corePath + fi + done +fi + +ulimit -c unlimited +${csudo}sed -i '/ulimit -c unlimited/d' /etc/profile ||: +${csudo}sed -i '$a\ulimit -c unlimited' /etc/profile ||: +source /etc/profile + +${csudo}mkdir -p ${corePath} ||: +${csudo}sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||: +${csudo}echo "${corePath}/core-%e-%p" | ${csudo}tee /proc/sys/kernel/core_pattern ||: diff --git a/packaging/tools/startPre.sh b/packaging/tools/startPre.sh new file mode 100644 index 0000000000..341e88ab2e --- /dev/null +++ b/packaging/tools/startPre.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# if enable core dump, set start count to 3, disable core dump, set start count to 20. +# set -e +# set -x + +serverName="taosd" +logDir="/var/log/taos" + +taosd=/etc/systemd/system/${serverName}.service +line=$(grep StartLimitBurst ${taosd}) +num=${line##*=} +#echo "burst num: ${num}" + +startSeqFile=${logDir}/.startSeq +recordFile=${logDir}/.startRecord + +startSeq=0 + +if [[ ! -e ${startSeqFile} ]]; then + startSeq=0 +else + startSeq=$(cat ${startSeqFile}) +fi + +nextSeq=$(expr $startSeq + 1) +echo "${nextSeq}" >${startSeqFile} + +curTime=$(date "+%Y-%m-%d %H:%M:%S") +echo "startSeq:${startSeq} startPre.sh exec ${curTime}, burstCnt:${num}" >>${recordFile} + +coreFlag=$(ulimit -c) +echo "coreFlag: ${coreFlag}" >>${recordFile} + +if [ ${coreFlag} = "0" ]; then + #echo "core is 0" + if [ ${num} != "20" ]; then + sed -i "s/^.*StartLimitBurst.*$/StartLimitBurst=20/" ${taosd} + systemctl daemon-reload + echo "modify burst count from ${num} to 20" >>${recordFile} + fi +fi + +if [ ${coreFlag} = "unlimited" ]; then + #echo "core is unlimited" + if [ ${num} != "3" ]; then + sed -i "s/^.*StartLimitBurst.*$/StartLimitBurst=3/" ${taosd} + systemctl daemon-reload + echo "modify burst count from ${num} to 3" >>${recordFile} + fi +fi diff --git a/packaging/tools/taosd-dump-cfg.gdb b/packaging/tools/taosd-dump-cfg.gdb new file mode 100644 index 0000000000..a7b143221d --- /dev/null +++ b/packaging/tools/taosd-dump-cfg.gdb @@ -0,0 +1,144 @@ +# Usage: +# sudo gdb -x ./taosd-dump-cfg.gdb + +define attach_pidof + if $argc != 1 + help attach_pidof + else + shell echo -e "\ +set \$PID = "$(echo $(pidof $arg0) 0 | cut -d " " -f 1)"\n\ +if \$PID > 0\n\ + attach "$(pidof -s $arg0)"\n\ +else\n\ + print \"Process '"$arg0"' not found\"\n\ +end" > /tmp/gdb.pidof + source /tmp/gdb.pidof + end +end + +document attach_pidof +Attach to process by name +Usage: attach_pidof PROG_NAME +end + +set $TAOS_CFG_VTYPE_INT8 = 0 +set $TAOS_CFG_VTYPE_INT16 = 1 +set $TAOS_CFG_VTYPE_INT32 = 2 +set $TAOS_CFG_VTYPE_FLOAT = 3 +set $TAOS_CFG_VTYPE_STRING = 4 +set $TAOS_CFG_VTYPE_IPSTR = 5 +set $TAOS_CFG_VTYPE_DIRECTORY = 6 + +set $TSDB_CFG_CTYPE_B_CONFIG = 1U +set $TSDB_CFG_CTYPE_B_SHOW = 2U +set $TSDB_CFG_CTYPE_B_LOG = 4U +set $TSDB_CFG_CTYPE_B_CLIENT = 8U +set $TSDB_CFG_CTYPE_B_OPTION = 16U +set $TSDB_CFG_CTYPE_B_NOT_PRINT = 32U + +set $TSDB_CFG_PRINT_LEN = 53 + +define print_blank + if $argc == 1 + set $blank_len = $arg0 + while $blank_len > 0 + printf "%s", " " + set $blank_len = $blank_len - 1 + end + end +end + +define dump_cfg + if $argc != 1 + help dump_cfg + else + set $blen = $TSDB_CFG_PRINT_LEN - (int)strlen($arg0.option) + if $blen < 0 + $blen = 0 + end + #printf "%s: %d\n", "******blen: ", $blen + printf "%s: ", $arg0.option + print_blank $blen + + if $arg0.valType == $TAOS_CFG_VTYPE_INT8 + printf "%d\n", *((int8_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_INT16 + printf "%d\n", *((int16_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_INT32 + printf "%d\n", *((int32_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_FLOAT + printf "%f\n", *((float *) $arg0.ptr) + else + printf "%s\n", $arg0.ptr + end + end + end + end + end +end + +document dump_cfg +Dump a cfg entry +Usage: dump_cfg cfg +end + +set pagination off + +attach_pidof taosd + +set $idx=0 +#print tsGlobalConfigNum +#set $end=$1 +set $end=tsGlobalConfigNum + +p "*=*=*=*=*=*=*=*=*= taos global config:" +#while ($idx .lt. $end) +while ($idx < $end) + # print tsGlobalConfig[$idx].option + set $cfg = tsGlobalConfig[$idx] + set $tsce = tscEmbedded +# p "1" + if ($tsce == 0) + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_CLIENT) + end + else + if $cfg.cfgType & $TSDB_CFG_CTYPE_B_NOT_PRINT + else + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_SHOW) + else + dump_cfg $cfg + end + end + end + + set $idx=$idx+1 +end + +set $idx=0 + +p "*=*=*=*=*=*=*=*=*= taos local config:" +while ($idx < $end) + set $cfg = tsGlobalConfig[$idx] + set $tsce = tscEmbedded + if ($tsce == 0) + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_CLIENT) + end + else + if $cfg.cfgType & $TSDB_CFG_CTYPE_B_NOT_PRINT + else + if ($cfg.cfgType & $TSDB_CFG_CTYPE_B_SHOW) + else + dump_cfg $cfg + end + end + end + + set $idx=$idx+1 +end + +detach + +quit From 8d1ddbd87788dbf0bfdd0fdd9b0b8dafb8566a8e Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 14 May 2022 15:42:21 +0800 Subject: [PATCH 23/71] fix: remove from task queue before signalling caller --- source/libs/function/src/tudf.c | 40 +++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 9fa20d3556..dd5bee208b 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -964,7 +964,7 @@ void udfcUvHandleError(SClientUvConn *conn) { uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); } -void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { +void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread); if (nread == 0) return; @@ -987,30 +987,32 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } -void onUdfClientWrite(uv_write_t *write, int status) { +void onUdfcPipetWrite(uv_write_t *write, int status) { SClientUvTaskNode *uvTask = write->data; uv_pipe_t *pipe = uvTask->pipe; + fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len); + SClientUvConn *conn = pipe->data; if (status == 0) { - SClientUvConn *conn = pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); } else { fnError("udfc client %p write error.", pipe); + udfcUvHandleError(conn); } - fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len); taosMemoryFree(write); taosMemoryFree(uvTask->reqBuf.base); } -void onUdfClientConnect(uv_connect_t *connect, int status) { +void onUdfcPipeConnect(uv_connect_t *connect, int status) { SClientUvTaskNode *uvTask = connect->data; - uvTask->errCode = status; if (status != 0) { - //TODO: LOG error + fnError("client connect error, task seq: %"PRId64", code: %s", uvTask->seqNum, uv_strerror(status)); } - uv_read_start((uv_stream_t *) uvTask->pipe, udfcAllocateBuffer, onUdfcRead); + uvTask->errCode = status; + + uv_read_start((uv_stream_t *)uvTask->pipe, udfcAllocateBuffer, onUdfcPipeRead); taosMemoryFree(connect); - uv_sem_post(&uvTask->taskSem); QUEUE_REMOVE(&uvTask->procTaskQueue); + uv_sem_post(&uvTask->taskSem); } int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { @@ -1070,6 +1072,7 @@ int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) { int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask); + int32_t code = 0; switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); @@ -1088,28 +1091,35 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); connReq->data = uvTask; - uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfClientConnect); + uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfcPipeConnect); + code = 0; break; } case UV_TASK_REQ_RSP: { uv_pipe_t *pipe = uvTask->pipe; uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); write->data = uvTask; - uv_write(write, (uv_stream_t *) pipe, &uvTask->reqBuf, 1, onUdfClientWrite); + int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipetWrite); + if (err != 0) { + fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err)); + } + code = err; break; } case UV_TASK_DISCONNECT: { SClientUvConn *conn = uvTask->pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); + code = 0; break; } default: { + fnError("udfc event loop unknown task type.") break; } } - return 0; + return code; } void udfClientAsyncCb(uv_async_t *async) { @@ -1124,8 +1134,10 @@ void udfClientAsyncCb(uv_async_t *async) { QUEUE* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); - udfcStartUvTask(task); - QUEUE_INSERT_TAIL(&udfc->uvProcTaskQueue, &task->procTaskQueue); + int32_t code = udfcStartUvTask(task); + if (code == 0) { + QUEUE_INSERT_TAIL(&udfc->uvProcTaskQueue, &task->procTaskQueue); + } } } From 234c0cd966eab7c8919178030b5ea41e954b65ef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 14 May 2022 16:28:34 +0800 Subject: [PATCH 24/71] fix(query): ignore the reserved column data when generating filtered result. --- source/libs/executor/src/executorimpl.c | 55 +++++++++++++++---------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f04ab24fe7..2d3c0d90d7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2114,6 +2114,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } } +static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { if (pFilterNode == NULL) { return; @@ -2128,43 +2129,53 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; + // todo the keep seems never to be True?? bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); filterFreeInfo(filter); - SSDataBlock* px = createOneDataBlock(pBlock, false); - blockDataEnsureCapacity(px, pBlock->info.rows); + extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); + blockDataUpdateTsWindow(pBlock); +} - // todo extract method - int32_t numOfRow = 0; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); - if (keep) { - colDataAssign(pDst, pSrc, pBlock->info.rows); - numOfRow = pBlock->info.rows; - } else if (NULL != rowRes) { - numOfRow = 0; +void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { + if (keep) { + return; + } + + if (rowRes != NULL) { + SSDataBlock* px = createOneDataBlock(pBlock, false); + blockDataEnsureCapacity(px, pBlock->info.rows); + + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); + + // For the reserved column, the value is not filled yet, so the whole column data may be NULL. + if (pSrc->pData == NULL) { + continue; + } + + int32_t numOfRows = 0; for (int32_t j = 0; j < pBlock->info.rows; ++j) { if (rowRes[j] == 0) { continue; } if (colDataIsNull_s(pSrc, j)) { - colDataAppendNULL(pDst, numOfRow); + colDataAppendNULL(pDst, numOfRows); } else { - colDataAppend(pDst, numOfRow, colDataGetData(pSrc, j), false); + colDataAppend(pDst, numOfRows, colDataGetData(pSrc, j), false); } - numOfRow += 1; + numOfRows += 1; } - } else { - numOfRow = 0; + + pBlock->info.rows = numOfRows; + *pSrc = *pDst; } - - *pSrc = *pDst; + } else { + // do nothing + pBlock->info.rows = 0; } - - pBlock->info.rows = numOfRow; - blockDataUpdateTsWindow(pBlock); } void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, From 46e76fc2eec22dac0b660cb619689520061b0343 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 14 May 2022 16:40:51 +0800 Subject: [PATCH 25/71] fix(query): fix a corner case for generating the filtered ssdatablock. --- source/libs/executor/src/executorimpl.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2d3c0d90d7..556f094528 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2146,6 +2146,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR SSDataBlock* px = createOneDataBlock(pBlock, false); blockDataEnsureCapacity(px, pBlock->info.rows); + int32_t totalRows = pBlock->info.rows; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); @@ -2156,7 +2158,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR } int32_t numOfRows = 0; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { + for (int32_t j = 0; j < totalRows; ++j) { if (rowRes[j] == 0) { continue; } @@ -2169,7 +2171,12 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR numOfRows += 1; } - pBlock->info.rows = numOfRows; + if (pBlock->info.rows == totalRows) { + pBlock->info.rows = numOfRows; + } else { + ASSERT(pBlock->info.rows == numOfRows); + } + *pSrc = *pDst; } } else { From 2a42fc66bc87803cabdb5a96df5f6aeb7cc7e0f9 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 14 May 2022 16:51:16 +0800 Subject: [PATCH 26/71] fix: udf task to queue whenever uv_write result --- source/libs/function/src/tudf.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index dd5bee208b..3a388d1c07 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1072,7 +1072,6 @@ int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) { int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask); - int32_t code = 0; switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); @@ -1092,7 +1091,6 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); connReq->data = uvTask; uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfcPipeConnect); - code = 0; break; } case UV_TASK_REQ_RSP: { @@ -1103,14 +1101,12 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { if (err != 0) { fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err)); } - code = err; break; } case UV_TASK_DISCONNECT: { SClientUvConn *conn = uvTask->pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); - code = 0; break; } default: { @@ -1119,7 +1115,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { } } - return code; + return 0; } void udfClientAsyncCb(uv_async_t *async) { From 1cfa16fcd8cbd3d6cdbc23143ebf53d73486b1b5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 14 May 2022 16:53:04 +0800 Subject: [PATCH 27/71] enh(sync): add syncConfigChangeTest --- source/libs/sync/test/CMakeLists.txt | 14 ++ .../libs/sync/test/syncConfigChangeTest.cpp | 217 ++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 source/libs/sync/test/syncConfigChangeTest.cpp diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 8afe9ff2a7..cfbdf0e961 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -37,6 +37,7 @@ add_executable(syncRaftCfgTest "") add_executable(syncRespMgrTest "") add_executable(syncSnapshotTest "") add_executable(syncApplyMsgTest "") +add_executable(syncConfigChangeTest "") target_sources(syncTest @@ -195,6 +196,10 @@ target_sources(syncApplyMsgTest PRIVATE "syncApplyMsgTest.cpp" ) +target_sources(syncConfigChangeTest + PRIVATE + "syncConfigChangeTest.cpp" +) target_include_directories(syncTest @@ -392,6 +397,11 @@ target_include_directories(syncApplyMsgTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncConfigChangeTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -550,6 +560,10 @@ target_link_libraries(syncApplyMsgTest sync gtest_main ) +target_link_libraries(syncConfigChangeTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp new file mode 100644 index 0000000000..0e94498a38 --- /dev/null +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -0,0 +1,217 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t gPorts[] = {7010, 7110, 7210, 7310, 7410}; +const char* gDir = "./syncReplicateTest"; +int32_t gVgId = 1234; +SyncIndex gSnapshotLastApplyIndex; + +void init() { + int code = walInit(); + assert(code == 0); + + code = syncInit(); + assert(code == 0); +} + +void cleanup() { walCleanUp(); } + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + SyncIndex beginIndex = SYNC_INDEX_INVALID; + if (pFsm->FpGetSnapshot != NULL) { + SSnapshot snapshot; + pFsm->FpGetSnapshot(pFsm, &snapshot); + beginIndex = snapshot.lastApplyIndex; + } + + if (cbMeta.index > beginIndex) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); + } else { + sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + } +} + +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = 100; + return 0; +} + +SSyncFSM* createFsm() { + SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; + pFsm->FpGetSnapshot = GetSnapshotCb; + return pFsm; +} + +SWal* createWal(char* path, int32_t vgId) { + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal* pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + return pWal; +} + +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path) { + SSyncInfo syncInfo; + syncInfo.vgId = vgId; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = createFsm(); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex); + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + } + + int64_t rid = syncOpen(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + syncNodeRelease(pSyncNode); + + return rid; +} + +void usage(char* exe) { printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum \n", exe); } + +SRpcMsg* createRpcMsg(int i, int count, int myIndex) { + SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 256; + pMsg->pCont = rpcMallocCont(pMsg->contLen); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + return pMsg; +} + +int main(int argc, char** argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + if (argc != 5) { + usage(argv[0]); + exit(-1); + } + int32_t replicaNum = atoi(argv[1]); + int32_t myIndex = atoi(argv[2]); + int32_t lastApplyIndex = atoi(argv[3]); + int32_t writeRecordNum = atoi(argv[4]); + gSnapshotLastApplyIndex = lastApplyIndex; + + assert(replicaNum >= 1 && replicaNum <= 5); + assert(myIndex >= 0 && myIndex < replicaNum); + assert(lastApplyIndex >= -1); + assert(writeRecordNum >= 0); + + init(); + int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); + assert(ret == 0); + + char walPath[128]; + snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); + SWal* pWal = createWal(walPath, gVgId); + + int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir); + assert(rid > 0); + syncStart(rid); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + //--------------------------- + int32_t alreadySend = 0; + while (1) { + char* s = syncNode2SimpleStr(pSyncNode); + + if (alreadySend < writeRecordNum) { + SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); + int32_t ret = syncPropose(rid, pRpcMsg, false); + if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + sTrace("%s value%d write not leader", s, alreadySend); + } else { + assert(ret == 0); + sTrace("%s value%d write ok", s, alreadySend); + } + alreadySend++; + + rpcFreeCont(pRpcMsg->pCont); + taosMemoryFree(pRpcMsg); + } else { + sTrace("%s", s); + } + + taosMsleep(1000); + taosMemoryFree(s); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + syncStop(rid); + walClose(pWal); + syncIOStop(); + cleanup(); + return 0; +} From 4a316eae3f5096b588cb05018651c5adf32b60a8 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 14 May 2022 17:03:05 +0800 Subject: [PATCH 28/71] Revert "feat: enhance udf scalar function calls" This reverts commit e8690dabefbe02a5d64615ddf7337f81e1546505. --- source/libs/scalar/inc/sclInt.h | 2 -- source/libs/scalar/src/scalar.c | 56 +++++++++------------------------ 2 files changed, 14 insertions(+), 44 deletions(-) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 99e61ad1db..9dbfeceb59 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -27,13 +27,11 @@ typedef struct SScalarCtx { SArray *pBlockList; /* element is SSDataBlock* */ SHashObj *pRes; /* element is SScalarParam */ void *param; // additional parameter (meta actually) for acquire value such as tbname/tags values - SHashObj *udf2Handle; } SScalarCtx; #define SCL_DATA_TYPE_DUMMY_HASH 9000 #define SCL_DEFAULT_OP_NUM 10 -#define SCL_DEFAULT_UDF_NUM 8 #define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type)) #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 8f4a9b9698..6c8326b09f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -154,18 +154,6 @@ void sclFreeRes(SHashObj *res) { taosHashCleanup(res); } -void sclFreeUdfHandles(SHashObj *udf2handle) { - void *pIter = taosHashIterate(udf2handle, NULL); - while (pIter) { - UdfcFuncHandle *handle = (UdfcFuncHandle *)pIter; - if (handle) { - teardownUdf(*handle); - } - pIter = taosHashIterate(udf2handle, pIter); - } - taosHashCleanup(udf2handle); -} - void sclFreeParam(SScalarParam *param) { if (param->columnData != NULL) { colDataDestroy(param->columnData); @@ -375,28 +363,22 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp if (fmIsUserDefinedFunc(node->funcId)) { UdfcFuncHandle udfHandle = NULL; - char* udfName = node->functionName; - if (ctx->udf2Handle) { - UdfcFuncHandle *pHandle = taosHashGet(ctx->udf2Handle, udfName, strlen(udfName)); - if (pHandle) { - udfHandle = *pHandle; - } - } - if (udfHandle == NULL) { - code = setupUdf(udfName, &udfHandle); - if (code != 0) { - sclError("fmExecFunction error. setupUdf. function name: %s, code:%d", udfName, code); - goto _return; - } - if (ctx->udf2Handle) { - taosHashPut(ctx->udf2Handle, udfName, strlen(udfName), &udfHandle, sizeof(UdfcFuncHandle)); - } + + code = setupUdf(node->functionName, &udfHandle); + if (code != 0) { + sclError("fmExecFunction error. setupUdf. function name: %s, code:%d", node->functionName, code); + goto _return; } code = callUdfScalarFunc(udfHandle, params, paramNum, output); if (code != 0) { sclError("fmExecFunction error. callUdfScalarFunc. function name: %s, udf code:%d", node->functionName, code); goto _return; } + code = teardownUdf(udfHandle); + if (code != 0) { + sclError("fmExecFunction error. callUdfScalarFunc. function name: %s, udf code:%d", node->functionName, code); + goto _return; + } } else { SScalarFuncExecFuncs ffpSet = {0}; code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); @@ -910,20 +892,15 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { SScalarCtx ctx = {0}; ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { - sclError("taosHashInit result map failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - ctx.udf2Handle = taosHashInit(SCL_DEFAULT_UDF_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (NULL == ctx.udf2Handle) { - sclError("taosHashInit udf to handle map failed, num:%d", SCL_DEFAULT_OP_NUM); + sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); SCL_ERR_JRET(ctx.code); *pRes = pNode; _return: - sclFreeUdfHandles(ctx.udf2Handle); sclFreeRes(ctx.pRes); return code; } @@ -939,14 +916,10 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { // TODO: OPT performance ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { - sclError("taosHashInit result map failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - ctx.udf2Handle = taosHashInit(SCL_DEFAULT_UDF_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (NULL == ctx.udf2Handle) { - sclError("taosHashInit udf to handle map failed, num:%d", SCL_DEFAULT_OP_NUM); + sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); SCL_ERR_JRET(ctx.code); @@ -964,7 +937,6 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { _return: //nodesDestroyNode(pNode); - sclFreeUdfHandles(ctx.udf2Handle); sclFreeRes(ctx.pRes); return code; } From 5b518821d7032aa6d4c9f998dacd5ca6e05dced7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 May 2022 09:16:57 +0000 Subject: [PATCH 29/71] test BDB --- cmake/{bdb_CMakeLists.txt.in.bak => bdb_CMakeLists.txt.in} | 0 cmake/cmake.options | 6 ++++++ contrib/CMakeLists.txt | 6 +++--- contrib/test/CMakeLists.txt | 6 +++--- 4 files changed, 12 insertions(+), 6 deletions(-) rename cmake/{bdb_CMakeLists.txt.in.bak => bdb_CMakeLists.txt.in} (100%) diff --git a/cmake/bdb_CMakeLists.txt.in.bak b/cmake/bdb_CMakeLists.txt.in similarity index 100% rename from cmake/bdb_CMakeLists.txt.in.bak rename to cmake/bdb_CMakeLists.txt.in diff --git a/cmake/cmake.options b/cmake/cmake.options index a60f5c7282..d83ab49fd5 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -78,6 +78,12 @@ option( OFF ) +option( + BUILD_WITH_BDB + "If build with BDB" + OFF +) + option( BUILD_WITH_LUCENE "If build with lucene" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 14a85ee4f6..97bfcfb8c0 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -78,9 +78,9 @@ if(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV}) # bdb -#if(${BUILD_WITH_BDB}) - #cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -#endif(${BUILD_WITH_BDB}) +if(${BUILD_WITH_BDB}) + cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif(${BUILD_WITH_BDB}) # sqlite if(${BUILD_WITH_SQLITE}) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 740488b39b..eacaeb9524 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -7,9 +7,9 @@ if(${BUILD_WITH_LUCENE}) add_subdirectory(lucene) endif(${BUILD_WITH_LUCENE}) -#if(${BUILD_WITH_BDB}) - #add_subdirectory(bdb) -#endif(${BUILD_WITH_BDB}) +if(${BUILD_WITH_BDB}) + add_subdirectory(bdb) +endif(${BUILD_WITH_BDB}) if(${BUILD_WITH_SQLITE}) add_subdirectory(sqlite) From 71619190864cdb73d4370c9a73063abf08c471c6 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 14 May 2022 17:20:22 +0800 Subject: [PATCH 30/71] fix: finish uv task when event loop start uv task failed --- source/libs/function/src/tudf.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 3a388d1c07..33614fadcb 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1072,6 +1072,8 @@ int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) { int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask); + int32_t code = 0; + switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); @@ -1091,6 +1093,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); connReq->data = uvTask; uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfcPipeConnect); + code = 0; break; } case UV_TASK_REQ_RSP: { @@ -1101,12 +1104,14 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { if (err != 0) { fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err)); } + code = err; break; } case UV_TASK_DISCONNECT: { SClientUvConn *conn = uvTask->pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); + code = 0; break; } default: { @@ -1115,7 +1120,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { } } - return 0; + return code; } void udfClientAsyncCb(uv_async_t *async) { @@ -1133,6 +1138,9 @@ void udfClientAsyncCb(uv_async_t *async) { int32_t code = udfcStartUvTask(task); if (code == 0) { QUEUE_INSERT_TAIL(&udfc->uvProcTaskQueue, &task->procTaskQueue); + } else { + task->errCode = code; + uv_sem_post(&task->taskSem); } } From a58a0e79c6bfdbc83e1f94685a95508ad52dc77d Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 14 May 2022 18:09:44 +0800 Subject: [PATCH 31/71] fix: uv task error when the pipe is NULL --- source/libs/function/src/tudf.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 33614fadcb..390f60e56c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1098,20 +1098,29 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { } case UV_TASK_REQ_RSP: { uv_pipe_t *pipe = uvTask->pipe; - uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); - write->data = uvTask; - int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipetWrite); - if (err != 0) { - fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err)); + if (pipe == NULL) { + code = TSDB_CODE_UDF_PIPE_NO_PIPE; + } else { + uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); + write->data = uvTask; + int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipetWrite); + if (err != 0) { + fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err)); + } + code = err; } - code = err; break; } case UV_TASK_DISCONNECT: { - SClientUvConn *conn = uvTask->pipe->data; - QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); - uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); - code = 0; + uv_pipe_t *pipe = uvTask->pipe; + if (pipe == NULL) { + code = TSDB_CODE_UDF_PIPE_NO_PIPE; + } else { + SClientUvConn *conn = pipe->data; + QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); + uv_close((uv_handle_t *)uvTask->pipe, onUdfcPipeClose); + code = 0; + } break; } default: { From a2d43fb96fb8fc72d340c56693305fae40eb793c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 14 May 2022 18:12:53 +0800 Subject: [PATCH 32/71] enh(sync): add syncStartStandBy --- source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/inc/syncUtil.h | 1 - source/libs/sync/src/syncAppendEntries.c | 12 ++-- source/libs/sync/src/syncCommit.c | 14 ++--- source/libs/sync/src/syncIO.c | 2 + source/libs/sync/src/syncMain.c | 11 ++-- source/libs/sync/src/syncRaftLog.c | 59 ++++++++++--------- .../libs/sync/test/syncConfigChangeTest.cpp | 41 +++++++++---- 8 files changed, 82 insertions(+), 60 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 9d090adda5..9b655fb0fa 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -272,7 +272,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); char* syncNode2SimpleStr(const SSyncNode* pSyncNode); -void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig); +void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig); SSyncNode* syncNodeAcquire(int64_t rid); void syncNodeRelease(SSyncNode* pNode); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 159af1610e..1b08d3f7a1 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -62,7 +62,6 @@ bool syncUtilUserPreCommit(tmsg_t msgType); bool syncUtilUserCommit(tmsg_t msgType); bool syncUtilUserRollback(tmsg_t msgType); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index aed19d042e..34fc7d4eb4 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -15,11 +15,11 @@ #include "syncAppendEntries.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" -#include "syncRaftCfg.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == @@ -200,7 +200,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); assert(pRollBackEntry != NULL); - //if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { + // if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { if (syncUtilUserRollback(pRollBackEntry->msgType)) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); @@ -229,7 +229,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pAppendEntry->index; @@ -261,7 +261,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pAppendEntry->index; @@ -324,7 +324,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - //if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; @@ -338,7 +338,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // config change if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { SSyncCfg newSyncCfg; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); ASSERT(ret == 0); syncNodeUpdateConfig(ths, &newSyncCfg); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 620f0e9cd2..5d7b5c1b21 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -16,10 +16,10 @@ #include "syncCommit.h" #include "syncIndexMgr.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" -#include "syncRaftCfg.h" // \* Leader i advances its commitIndex. // \* This is done as a separate step from handling AppendEntries responses, @@ -102,7 +102,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - //if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (pSyncNode->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; @@ -114,12 +114,12 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { } // config change - if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { - SSyncCfg newSyncCfg; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); - ASSERT(ret == 0); + if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { + SSyncCfg newSyncCfg; + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); + ASSERT(ret == 0); - syncNodeUpdateConfig(pSyncNode, &newSyncCfg); + syncNodeUpdateConfig(pSyncNode, &newSyncCfg); } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index deb158cbae..209bd25c2c 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -20,6 +20,7 @@ #include "tglobal.h" #include "ttimer.h" #include "tutil.h" +#include "os.h" SSyncIO *gSyncIO = NULL; @@ -198,6 +199,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); + snprintf(rpcInit.localFqdn, sizeof(rpcInit.localFqdn), "%s", "127.0.0.1"); rpcInit.localPort = io->myAddr.eps[0].port; rpcInit.label = "SYNC-IO-SERVER"; rpcInit.numOfThreads = 1; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0740a73fd0..34313baaa9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -126,7 +126,7 @@ void syncStop(int64_t rid) { int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { int32_t ret = 0; - char *configChange = syncCfg2Str((SSyncCfg*)pSyncCfg); + char* configChange = syncCfg2Str((SSyncCfg*)pSyncCfg); SRpcMsg rpcMsg = {0}; rpcMsg.msgType = TDMT_VND_SYNC_CONFIG_CHANGE; rpcMsg.noResp = 1; @@ -198,10 +198,9 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { (pEpSet->numOfEps)++; sInfo("syncGetEpSet index:%d %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; - + sInfo("syncGetEpSet pEpSet->inUse:%d ", pEpSet->inUse); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -879,7 +878,7 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { return s; } -void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig) { +void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig) { pSyncNode->pRaftCfg->cfg = *newConfig; int32_t ret = raftCfgPersist(pSyncNode->pRaftCfg); ASSERT(ret == 0); @@ -1266,7 +1265,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; @@ -1288,7 +1287,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 8aeb9c4856..07a9397a58 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -58,14 +58,15 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); if (code != 0) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t linuxErr = errno; - const char *linuxErrMsg = strerror(errno); - sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); ASSERT(0); - } - //assert(code == 0); + } + // assert(code == 0); walFsync(pWal, true); return code; @@ -77,16 +78,17 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - int32_t code = walReadWithHandle(pWalHandle, index); + int32_t code = walReadWithHandle(pWalHandle, index); if (code != 0) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t linuxErr = errno; - const char *linuxErrMsg = strerror(errno); - sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); ASSERT(0); - } - //assert(walReadWithHandle(pWalHandle, index) == 0); + } + // assert(walReadWithHandle(pWalHandle, index) == 0); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); assert(pEntry != NULL); @@ -112,16 +114,17 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - //assert(walRollback(pWal, fromIndex) == 0); + // assert(walRollback(pWal, fromIndex) == 0); int32_t code = walRollback(pWal, fromIndex); if (code != 0) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t linuxErr = errno; - const char *linuxErrMsg = strerror(errno); - sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); ASSERT(0); - } + } return 0; // to avoid compiler error } @@ -145,16 +148,16 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - //assert(walCommit(pWal, index) == 0); + // assert(walCommit(pWal, index) == 0); int32_t code = walCommit(pWal, index); if (code != 0) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t linuxErr = errno; - const char *linuxErrMsg = strerror(errno); + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); - } + } return 0; // to avoid compiler error } diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index 0e94498a38..e91c6d33fb 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -5,6 +5,7 @@ #include "syncInt.h" #include "syncUtil.h" #include "wal.h" +#include "os.h" void logTest() { sTrace("--- sync log test: trace"); @@ -26,6 +27,8 @@ void init() { code = syncInit(); assert(code == 0); + + sprintf(tsTempDir, "%s", "."); } void cleanup() { walCleanUp(); } @@ -94,7 +97,7 @@ SWal* createWal(char* path, int32_t vgId) { return pWal; } -int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path) { +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy) { SSyncInfo syncInfo; syncInfo.vgId = vgId; syncInfo.rpcClient = gSyncIO->clientRpc; @@ -106,13 +109,22 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* syncInfo.pWal = pWal; SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = gPorts[i]; - taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); - // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + if (isStandBy) { + pCfg->myIndex = 0; + pCfg->replicaNum = 1; + pCfg->nodeInfo[0].nodePort = gPorts[myIndex]; + taosGetFqdn(pCfg->nodeInfo[myIndex].nodeFqdn); + + } else { + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + } } int64_t rid = syncOpen(&syncInfo); @@ -136,7 +148,7 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* return rid; } -void usage(char* exe) { printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum \n", exe); } +void usage(char* exe) { printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy \n", exe); } SRpcMsg* createRpcMsg(int i, int count, int myIndex) { SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); @@ -151,14 +163,16 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) { int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; - if (argc != 5) { + if (argc != 6) { usage(argv[0]); exit(-1); } + int32_t replicaNum = atoi(argv[1]); int32_t myIndex = atoi(argv[2]); int32_t lastApplyIndex = atoi(argv[3]); int32_t writeRecordNum = atoi(argv[4]); + bool isStandBy = atoi(argv[5]); gSnapshotLastApplyIndex = lastApplyIndex; assert(replicaNum >= 1 && replicaNum <= 5); @@ -174,9 +188,14 @@ int main(int argc, char** argv) { snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); SWal* pWal = createWal(walPath, gVgId); - int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir); + int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy); assert(rid > 0); - syncStart(rid); + + if (isStandBy) { + syncStartStandBy(rid); + } else { + syncStart(rid); + } SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); assert(pSyncNode != NULL); From 56cc2c06f9a3941bf9fa9dbc6c33b81e3457dee1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 14 May 2022 18:28:24 +0800 Subject: [PATCH 33/71] fix: udf aggregate function handle does not exist --- source/libs/function/src/tudf.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 390f60e56c..94599fa0f5 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1500,6 +1500,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SClientUdfUvSession *session = udfRes->session; + if (session == NULL) { + return TSDB_CODE_UDF_PIPE_NO_PIPE; + } udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; @@ -1552,6 +1555,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SClientUdfUvSession *session = udfRes->session; + if (session == NULL) { + return TSDB_CODE_UDF_PIPE_NO_PIPE; + } udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; From cef0f74cb39438a6413bf8119c7836d20e9c2e64 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 14 May 2022 18:28:51 +0800 Subject: [PATCH 34/71] fix: udf aggregate function handle does not exist --- include/util/taoserror.h | 1 + source/libs/function/src/tudf.c | 4 ++-- source/util/src/terror.c | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b73d39090b..a0d7b59cf9 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -657,6 +657,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_UDF_LOAD_UDF_FAILURE TAOS_DEF_ERROR_CODE(0, 0x2905) #define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906) #define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2907) +#define TSDB_CODE_UDF_NO_FUNC_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2908) #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) #define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 94599fa0f5..d9e3ff0a5b 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1501,7 +1501,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SClientUdfUvSession *session = udfRes->session; if (session == NULL) { - return TSDB_CODE_UDF_PIPE_NO_PIPE; + return TSDB_CODE_UDF_NO_FUNC_HANDLE; } udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; @@ -1556,7 +1556,7 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SClientUdfUvSession *session = udfRes->session; if (session == NULL) { - return TSDB_CODE_UDF_PIPE_NO_PIPE; + return TSDB_CODE_UDF_NO_FUNC_HANDLE; } udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6c0a9b1324..a2b0648a4b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -463,7 +463,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") - +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle") //schemaless TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") From 60e8bc24cd768131023f1cc7670520f4c78b203a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 14 May 2022 19:17:53 +0800 Subject: [PATCH 35/71] fix: some problems of parser and planner --- include/common/ttime.h | 15 +-- include/libs/qcom/query.h | 119 +++++++++++------- include/util/taoserror.h | 2 + source/libs/nodes/src/nodesCodeFuncs.c | 98 +++++++++++++-- source/libs/parser/src/parAstCreater.c | 17 +++ source/libs/parser/src/parAuthenticator.c | 13 +- source/libs/parser/src/parCalcConst.c | 4 +- source/libs/parser/src/parTranslater.c | 35 +++++- source/libs/parser/src/parUtil.c | 4 + source/libs/parser/test/parSelectTest.cpp | 6 +- source/libs/planner/src/planOptimizer.c | 10 +- source/libs/planner/src/planPhysiCreater.c | 18 +++ source/libs/planner/test/planJoinTest.cpp | 14 ++- source/libs/planner/test/planSubqueryTest.cpp | 8 +- source/libs/planner/test/planTestMain.cpp | 32 ++++- source/libs/planner/test/planTestUtil.cpp | 1 + source/libs/planner/test/planTestUtil.h | 2 + 17 files changed, 315 insertions(+), 83 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 3de0b98d85..cd704bb1f7 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -59,10 +59,11 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) { * precision == TSDB_TIME_PRECISION_NANO, it returns timestamp in nanosecond. */ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { - int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 : - (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000; - time_t t = taosTime(NULL); - struct tm * tm= taosLocalTime(&t, NULL); + int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 + : 1000000000; + time_t t = taosTime(NULL); + struct tm* tm = taosLocalTime(&t, NULL); tm->tm_hour = 0; tm->tm_min = 0; tm->tm_sec = 0; @@ -79,13 +80,13 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); void deltaToUtcInitOnce(); -char getPrecisionUnit(int32_t precision); +char getPrecisionUnit(int32_t precision); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit); -int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal); +int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal); -void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision); +void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t time, int32_t precision); #ifdef __cplusplus } diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c390f67153..711db65e97 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -51,14 +51,12 @@ typedef struct STableComInfo { } STableComInfo; typedef struct SIndexMeta { - #ifdef WINDOWS size_t avoidCompilationErrors; #endif } SIndexMeta; - /* * ASSERT(sizeof(SCTableMeta) == 24) * ASSERT(tableType == TSDB_CHILD_TABLE) @@ -95,7 +93,7 @@ typedef struct SDBVgInfo { int32_t vgVersion; int8_t hashMethod; int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT - SHashObj *vgHash; //key:vgId, value:SVgroupInfo + SHashObj* vgHash; // key:vgId, value:SVgroupInfo } SDBVgInfo; typedef struct SUseDbOutput { @@ -135,7 +133,7 @@ typedef struct SMsgSendInfo { } SMsgSendInfo; typedef struct SQueryNodeStat { - int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT + int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT } SQueryNodeStat; int32_t initTaskQueue(); @@ -172,7 +170,7 @@ const SSchema* tGetTbnameColumnSchema(); bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); -char *jobTaskStatusStr(int32_t status); +char* jobTaskStatusStr(int32_t status); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); @@ -184,62 +182,87 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE -#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST) -#define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) +#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \ + ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST) +#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ + ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED) -#define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) +#define NEED_CLIENT_HANDLE_ERROR(_code) \ + (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ + NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) -#define NEED_SCHEDULER_RETRY_ERROR(_code) ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) +#define NEED_SCHEDULER_RETRY_ERROR(_code) \ + ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) #define REQUEST_MAX_TRY_TIMES 5 -#define qFatal(...) \ - do { \ - if (qDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \ - } \ +#define qFatal(...) \ + do { \ + if (qDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("QRY FATAL ", DEBUG_FATAL, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qError(...) \ - do { \ - if (qDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \ - } \ +#define qError(...) \ + do { \ + if (qDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("QRY ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qWarn(...) \ - do { \ - if (qDebugFlag & DEBUG_WARN) { \ - taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \ - } \ +#define qWarn(...) \ + do { \ + if (qDebugFlag & DEBUG_WARN) { \ + taosPrintLog("QRY WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qInfo(...) \ - do { \ - if (qDebugFlag & DEBUG_INFO) { \ - taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \ - } \ +#define qInfo(...) \ + do { \ + if (qDebugFlag & DEBUG_INFO) { \ + taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qDebug(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebug(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qTrace(...) \ - do { \ - if (qDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ - } \ +#define qTrace(...) \ + do { \ + if (qDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qDebugL(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebugL(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) -#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) -#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) - +#define QRY_ERR_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + terrno = _code; \ + return _code; \ + } \ + } while (0) +#define QRY_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + terrno = _code; \ + } \ + return _code; \ + } while (0) +#define QRY_ERR_JRET(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + terrno = code; \ + goto _return; \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index aa2b32daab..ae083780e4 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -635,6 +635,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_PERMISSION_DENIED TAOS_DEF_ERROR_CODE(0, 0x2644) #define TSDB_CODE_PAR_INVALID_STREAM_QUERY TAOS_DEF_ERROR_CODE(0, 0x2645) #define TSDB_CODE_PAR_INVALID_INTERNAL_PK TAOS_DEF_ERROR_CODE(0, 0x2646) +#define TSDB_CODE_PAR_INVALID_TIMELINE_FUNC TAOS_DEF_ERROR_CODE(0, 0x2647) +#define TSDB_CODE_PAR_INVALID_PASSWD TAOS_DEF_ERROR_CODE(0, 0x2648) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 71b0774ca6..dba202a45f 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1142,9 +1142,13 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { return code; } -static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { return physiTableScanNodeToJson(pObj, pJson); } +static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { + return physiTableScanNodeToJson(pObj, pJson); +} -static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiTableScanNode(pJson, pObj); } +static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { + return jsonToPhysiTableScanNode(pJson, pObj); +} static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet"; static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite"; @@ -2347,6 +2351,30 @@ static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkTempTableSubquery = "Subquery"; + +static int32_t tempTableNodeToJson(const void* pObj, SJson* pJson) { + const STempTableNode* pNode = (const STempTableNode*)pObj; + + int32_t code = tableNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkTempTableSubquery, nodeToJson, pNode->pSubquery); + } + + return code; +} + +static int32_t jsonToTempTableNode(const SJson* pJson, void* pObj) { + STempTableNode* pNode = (STempTableNode*)pObj; + + int32_t code = jsonToTableNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkTempTableSubquery, &pNode->pSubquery); + } + + return code; +} + static const char* jkGroupingSetType = "GroupingSetType"; static const char* jkGroupingSetParameter = "Parameters"; @@ -2659,6 +2687,59 @@ static int32_t jsonToDataBlockDescNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkSetOperatorOpType = "OpType"; +static const char* jkSetOperatorProjections = "Projections"; +static const char* jkSetOperatorLeft = "Left"; +static const char* jkSetOperatorRight = "Right"; +static const char* jkSetOperatorOrderByList = "OrderByList"; +static const char* jkSetOperatorLimit = "Limit"; + +static int32_t setOperatorToJson(const void* pObj, SJson* pJson) { + const SSetOperator* pNode = (const SSetOperator*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSetOperatorOpType, pNode->opType); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkSetOperatorProjections, pNode->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSetOperatorLeft, nodeToJson, pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSetOperatorRight, nodeToJson, pNode->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkSetOperatorOrderByList, pNode->pOrderByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSetOperatorLimit, nodeToJson, pNode->pLimit); + } + + return code; +} + +static int32_t jsonToSetOperator(const SJson* pJson, void* pObj) { + SSetOperator* pNode = (SSetOperator*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkSetOperatorOpType, pNode->opType); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSetOperatorProjections, &pNode->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSetOperatorLeft, &pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSetOperatorRight, &pNode->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSetOperatorOrderByList, &pNode->pOrderByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSetOperatorLimit, &pNode->pLimit); + } + + return code; +} + static const char* jkSelectStmtDistinct = "Distinct"; static const char* jkSelectStmtProjections = "Projections"; static const char* jkSelectStmtFrom = "From"; @@ -2673,7 +2754,7 @@ static const char* jkSelectStmtSlimit = "Slimit"; static const char* jkSelectStmtStmtName = "StmtName"; static const char* jkSelectStmtHasAggFuncs = "HasAggFuncs"; -static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { +static int32_t selectStmtToJson(const void* pObj, SJson* pJson) { const SSelectStmt* pNode = (const SSelectStmt*)pObj; int32_t code = tjsonAddBoolToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); @@ -2815,6 +2896,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_REAL_TABLE: return realTableNodeToJson(pObj, pJson); case QUERY_NODE_TEMP_TABLE: + return tempTableNodeToJson(pObj, pJson); case QUERY_NODE_JOIN_TABLE: break; case QUERY_NODE_GROUPING_SET: @@ -2844,9 +2926,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_DOWNSTREAM_SOURCE: return downstreamSourceNodeToJson(pObj, pJson); case QUERY_NODE_SET_OPERATOR: - break; + return setOperatorToJson(pObj, pJson); case QUERY_NODE_SELECT_STMT: - return selectStmtTojson(pObj, pJson); + return selectStmtToJson(pObj, pJson); case QUERY_NODE_VNODE_MODIF_STMT: case QUERY_NODE_CREATE_DATABASE_STMT: case QUERY_NODE_CREATE_TABLE_STMT: @@ -2914,7 +2996,6 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN: return planToJson(pObj, pJson); default: - // assert(0); break; } nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj))); @@ -2935,6 +3016,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToFunctionNode(pJson, pObj); case QUERY_NODE_REAL_TABLE: return jsonToRealTableNode(pJson, pObj); + case QUERY_NODE_TEMP_TABLE: + return jsonToTempTableNode(pJson, pObj); case QUERY_NODE_ORDER_BY_EXPR: return jsonToOrderByExprNode(pJson, pObj); case QUERY_NODE_INTERVAL_WINDOW: @@ -2951,6 +3034,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToSlotDescNode(pJson, pObj); case QUERY_NODE_DOWNSTREAM_SOURCE: return jsonToDownstreamSourceNode(pJson, pObj); + case QUERY_NODE_SET_OPERATOR: + return jsonToSetOperator(pJson, pObj); case QUERY_NODE_SELECT_STMT: return jsonToSelectStmt(pJson, pObj); case QUERY_NODE_CREATE_TOPIC_STMT: @@ -3003,7 +3088,6 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN: return jsonToPlan(pJson, pObj); default: - assert(0); break; } nodesWarn("jsonToSpecificNode unknown node = %s", nodesNodeName(nodeType(pObj))); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 639da98f48..7dc2978ec7 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -14,6 +14,8 @@ * along with this program. If not, see . */ +#include + #include "parAst.h" #include "parUtil.h" #include "ttime.h" @@ -76,6 +78,19 @@ static bool checkUserName(SAstCreateContext* pCxt, SToken* pUserName) { return TSDB_CODE_SUCCESS == pCxt->errCode; } +static bool invalidPassword(const char* pPassword) { + regex_t regex; + + if (regcomp(®ex, "[ '\"`\\]", REG_EXTENDED | REG_ICASE) != 0) { + return false; + } + + /* Execute regular expression */ + int32_t res = regexec(®ex, pPassword, 0, NULL, 0); + regfree(®ex); + return 0 == res; +} + static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, char* pPassword) { if (NULL == pPasswordToken) { pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; @@ -86,6 +101,8 @@ static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, strdequote(pPassword); if (strtrim(pPassword) <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); + } else if (invalidPassword(pPassword)) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PASSWD); } } return TSDB_CODE_SUCCESS == pCxt->errCode; diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 8f686cefce..ad1dca6857 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -65,13 +65,19 @@ static int32_t authSetOperator(SAuthCxt* pCxt, SSetOperator* pSetOper) { return code; } +static int32_t authDropUser(SAuthCxt* pCxt, SDropUserReq* pStmt) { + if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->user, TSDB_DEFAULT_USER)) { + return TSDB_CODE_PAR_PERMISSION_DENIED; + } + return TSDB_CODE_SUCCESS; +} + static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { switch (nodeType(pStmt)) { case QUERY_NODE_SET_OPERATOR: return authSetOperator(pCxt, (SSetOperator*)pStmt); case QUERY_NODE_SELECT_STMT: return authSelect(pCxt, (SSelectStmt*)pStmt); - case QUERY_NODE_VNODE_MODIF_STMT: case QUERY_NODE_CREATE_DATABASE_STMT: case QUERY_NODE_DROP_DATABASE_STMT: case QUERY_NODE_ALTER_DATABASE_STMT: @@ -84,7 +90,10 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_ALTER_TABLE_STMT: case QUERY_NODE_CREATE_USER_STMT: case QUERY_NODE_ALTER_USER_STMT: - case QUERY_NODE_DROP_USER_STMT: + break; + case QUERY_NODE_DROP_USER_STMT: { + return authDropUser(pCxt, (SDropUserReq*)pStmt); + } case QUERY_NODE_USE_DATABASE_STMT: case QUERY_NODE_CREATE_DNODE_STMT: case QUERY_NODE_DROP_DNODE_STMT: diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 9c2bd10686..646ef4cf62 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -262,9 +262,9 @@ static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subque break; case QUERY_NODE_SET_OPERATOR: { SSetOperator* pSetOp = (SSetOperator*)pStmt; - code = calcConstQuery(pCxt, pSetOp->pLeft, subquery); + code = calcConstQuery(pCxt, pSetOp->pLeft, false); if (TSDB_CODE_SUCCESS == code) { - code = calcConstQuery(pCxt, pSetOp->pRight, subquery); + code = calcConstQuery(pCxt, pSetOp->pRight, false); } break; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c7f0c13de0..bc4fa9169c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -480,6 +480,31 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { return res; } +static int32_t parseTimeFromValueNode(SValueNode* pVal) { + if (IS_SIGNED_NUMERIC_TYPE(pVal->node.resType.type)) { + return TSDB_CODE_SUCCESS; + } else if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) { + pVal->datum.i = pVal->datum.u; + return TSDB_CODE_SUCCESS; + } else if (IS_FLOAT_TYPE(pVal->node.resType.type)) { + pVal->datum.i = pVal->datum.d; + return TSDB_CODE_SUCCESS; + } else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) { + pVal->datum.i = pVal->datum.b; + return TSDB_CODE_SUCCESS; + } else if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) { + if (TSDB_CODE_SUCCESS == taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes, + pVal->node.resType.precision, tsDaylight)) { + return TSDB_CODE_SUCCESS; + } + char* pEnd = NULL; + pVal->datum.i = strtoll(pVal->literal, &pEnd, 10); + return (NULL != pEnd && '\0' == *pEnd) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; + } else { + return TSDB_CODE_FAILED; + } +} + static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SDataType targetDt) { uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : targetDt.precision); pVal->node.resType.precision = precision; @@ -571,7 +596,7 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD break; } case TSDB_DATA_TYPE_TIMESTAMP: { - if (taosParseTime(pVal->literal, &pVal->datum.i, targetDt.bytes, precision, tsDaylight) != TSDB_CODE_SUCCESS) { + if (TSDB_CODE_SUCCESS != parseTimeFromValueNode(pVal)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(int64_t*)&pVal->typeData = pVal->datum.i; @@ -1658,10 +1683,10 @@ static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* p if (NULL == pCol) { return TSDB_CODE_OUT_OF_MEMORY; } - if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); - } else { - // todo + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + strcpy(pCol->colName, PK_TS_COL_INTERNAL_NAME); + if (!findAndSetColumn(pCol, pTable)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_FUNC); } *pPrimaryKey = (SNode*)pCol; return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 43aea8de7c..676fe5dbfd 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -148,6 +148,10 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid number of tag columns"; case TSDB_CODE_PAR_INVALID_INTERNAL_PK: return "Invalid _c0 or _rowts expression"; + case TSDB_CODE_PAR_INVALID_TIMELINE_FUNC: + return "Invalid timeline function"; + case TSDB_CODE_PAR_INVALID_PASSWD: + return "Invalid password"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 0ba062ebe4..23b82d54bd 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -235,9 +235,11 @@ TEST_F(ParserSelectTest, semanticError) { TEST_F(ParserSelectTest, setOperator) { useDb("root", "test"); - run("SELECT * FROM t1 UNION ALL SELECT * FROM t1"); + // run("SELECT * FROM t1 UNION ALL SELECT * FROM t1"); - run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)"); + // run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)"); + + run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); } } // namespace ParserTest diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 9968f63c5d..edac2a879f 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -582,7 +582,7 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { return false; } - SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions; + SOperatorNode* pOper = (SOperatorNode*)pCond; if (OP_TYPE_EQUAL != pOper->opType) { return false; } @@ -608,12 +608,16 @@ static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, if (LOGIC_COND_TYPE_AND != pOnCond->condType) { return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } + bool hasPrimaryKeyEqualCond = false; SNode* pCond = NULL; FOREACH(pCond, pOnCond->pParameterList) { - if (!cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { - return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); + if (cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { + hasPrimaryKeyEqualCond = true; } } + if (!hasPrimaryKeyEqualCond) { + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index edf44424e3..affe9ef2f6 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -261,6 +261,22 @@ typedef struct SSetSlotIdCxt { SHashObj* pRightHash; } SSetSlotIdCxt; +static void dumpSlots(const char* pName, SHashObj* pHash) { + if (NULL == pHash) { + return; + } + planDebug("%s", pName); + void* pIt = taosHashIterate(pHash, NULL); + while (NULL != pIt) { + size_t len = 0; + char* pKey = taosHashGetKey(pIt, &len); + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN] = {0}; + strncpy(name, pKey, len); + planDebug("\tslot name = %s", name); + pIt = taosHashIterate(pHash, pIt); + } +} + static EDealRes doSetSlotId(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) { SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; @@ -273,6 +289,8 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { // pIndex is definitely not NULL, otherwise it is a bug if (NULL == pIndex) { planError("doSetSlotId failed, invalid slot name %s", name); + dumpSlots("left datablock desc", pCxt->pLeftHash); + dumpSlots("right datablock desc", pCxt->pRightHash); pCxt->errCode = TSDB_CODE_PLAN_INTERNAL_ERROR; return DEAL_RES_ERROR; } diff --git a/source/libs/planner/test/planJoinTest.cpp b/source/libs/planner/test/planJoinTest.cpp index 4098d383f8..714900c4e5 100644 --- a/source/libs/planner/test/planJoinTest.cpp +++ b/source/libs/planner/test/planJoinTest.cpp @@ -23,10 +23,16 @@ class PlanJoinTest : public PlannerTestBase {}; TEST_F(PlanJoinTest, basic) { useDb("root", "test"); - run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + run("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); - run("select t1.*, t2.* from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + run("SELECT t1.*, t2.* FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); - // run("select t1.c1, t2.c1 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1 and t1.c2 = 'abc' and " - // "t2.c2 = 'qwe'"); + run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts"); +} + +TEST_F(PlanJoinTest, withWhere) { + useDb("root", "test"); + + run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts " + "WHERE t1.c1 > t2.c1 AND t1.c2 = 'abc' AND t2.c2 = 'qwe'"); } diff --git a/source/libs/planner/test/planSubqueryTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp index f45cbc6f8f..11e5e98052 100644 --- a/source/libs/planner/test/planSubqueryTest.cpp +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -23,9 +23,13 @@ class PlanSubqeuryTest : public PlannerTestBase {}; TEST_F(PlanSubqeuryTest, basic) { useDb("root", "test"); - run("SELECT * FROM (SELECT * FROM t1)"); + if (0 == g_skipSql) { + run("SELECT * FROM (SELECT * FROM t1)"); - // run("SELECT LAST(c1) FROM ( SELECT * FROM t1)"); + run("SELECT LAST(c1) FROM (SELECT * FROM t1)"); + } + + run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); } TEST_F(PlanSubqeuryTest, doubleGroupBy) { diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 464c636b66..36f66ddff6 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -25,23 +25,53 @@ class PlannerEnv : public testing::Environment { virtual void SetUp() { initMetaDataEnv(); generateMetaData(); + initLog("/tmp/td"); } virtual void TearDown() { destroyMetaDataEnv(); } PlannerEnv() {} virtual ~PlannerEnv() {} + + private: + void initLog(const char* path) { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 143; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 135; + uDebugFlag = 135; + rpcDebugFlag = 143; + qDebugFlag = 143; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + tsLogEmbedded = 1; + tsAsyncLog = 0; + + taosRemoveDir(path); + taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); + if (taosInitLog("taoslog", 1) != 0) { + std::cout << "failed to init log file" << std::endl; + } + } }; static void parseArg(int argc, char* argv[]) { int opt = 0; const char* optstring = ""; - static struct option long_options[] = {{"dump", optional_argument, NULL, 'd'}, {0, 0, 0, 0}}; + static struct option long_options[] = { + {"dump", optional_argument, NULL, 'd'}, {"skipSql", optional_argument, NULL, 's'}, {0, 0, 0, 0}}; while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { switch (opt) { case 'd': setDumpModule(optarg); break; + case 's': + g_skipSql = 1; + break; default: break; } diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index b2c590667e..6b038ae8ea 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -47,6 +47,7 @@ enum DumpModule { }; DumpModule g_dumpModule = DUMP_MODULE_NOTHING; +int32_t g_skipSql = 0; void setDumpModule(const char* pModule) { if (NULL == pModule) { diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 7913ef531f..a63bba1a97 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -32,6 +32,8 @@ class PlannerTestBase : public testing::Test { std::unique_ptr impl_; }; +extern int32_t g_skipSql; + extern void setDumpModule(const char* pModule); #endif // PLAN_TEST_UTIL_H From 8242cac94ae82da998a32bd7ed2a4a9ba8cb05ef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 14 May 2022 19:21:12 +0800 Subject: [PATCH 36/71] fix(query): fill the pseudo column before applying filter in table scanner. --- source/client/src/clientImpl.c | 3 +-- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 20 +++++++++++++------- source/libs/executor/src/groupoperator.c | 2 +- source/libs/executor/src/scanoperator.c | 20 ++++++++++++-------- tools/shell/src/shellEngine.c | 1 + 6 files changed, 29 insertions(+), 19 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f879838d63..94238073c6 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -947,8 +947,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR // TODO handle the compressed case pResultInfo->totalRows += pResultInfo->numOfRows; - return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, - convertUcs4); + return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4); } TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index dd0bcbff0e..a032645493 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -651,7 +651,7 @@ void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 556f094528..cb8bee1e6a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2115,7 +2115,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) { if (pFilterNode == NULL) { return; } @@ -2129,14 +2129,25 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; + // todo the keep seems never to be True?? - bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); + bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); filterFreeInfo(filter); extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); blockDataUpdateTsWindow(pBlock); } +static int32_t colIdSearchCompar(const void* p1, const void* p2) { + int32_t colId = *(int32_t*)p1; + SColMatchInfo* pInfo = (SColMatchInfo*)p2; + if (colId == pInfo->targetSlotId) { + return 0; + } + + return (colId < pInfo->colId) ? -1 : 1; +} + void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { if (keep) { return; @@ -2152,11 +2163,6 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); - // For the reserved column, the value is not filled yet, so the whole column data may be NULL. - if (pSrc->pData == NULL) { - continue; - } - int32_t numOfRows = 0; for (int32_t j = 0; j < totalRows; ++j) { if (rowRes[j] == 0) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index d8ccac8cea..483ac67e5e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -318,7 +318,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { while(1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes); + doFilter(pInfo->pCondition, pRes, NULL); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); if (!hasRemain) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c6cb01e8fb..65bd8f4bda 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -159,6 +159,8 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn return false; } +static void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock); + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -238,8 +240,15 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols); + + // currently only the tbname pseudo column + if (pTableScanInfo->numOfPseudoExpr > 0) { + addTagPseudoColumnData(pTableScanInfo, pBlock); + } + // todo record the filter time cost - doFilter(pTableScanInfo->pFilterNode, pBlock); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), @@ -260,7 +269,7 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction pTableScanInfo->cond.order = TSDB_ORDER_DESC; } -static void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { +void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { // currently only the tbname pseudo column if (pTableScanInfo->numOfPseudoExpr == 0) { return; @@ -330,11 +339,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - // currently only the tbname pseudo column - if (pTableScanInfo->numOfPseudoExpr > 0) { - addTagPseudoColumnData(pTableScanInfo, pBlock); - } - return pBlock; } @@ -750,7 +754,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { return NULL; } rows = pBlockInfo->rows; - doFilter(pInfo->pCondition, pInfo->pRes); + doFilter(pInfo->pCondition, pInfo->pRes, NULL); break; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 39b97004ff..4825aae699 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -604,6 +604,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { case TSDB_DATA_TYPE_DOUBLE: return TMAX(25, width); + case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_BINARY: if (field->bytes > shell.args.displayWidth) { return TMAX(shell.args.displayWidth, width); From 2d456be89e65bc294ce3ea78eca19c08e4f5bf89 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 14 May 2022 19:31:53 +0800 Subject: [PATCH 37/71] fix: some problems of parser and planner --- source/libs/nodes/src/nodesCodeFuncs.c | 108 ++++++++++++++++--------- 1 file changed, 72 insertions(+), 36 deletions(-) diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 6a80c29303..57dfcaeddd 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -318,15 +318,19 @@ static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) { STableComInfo* pNode = (STableComInfo*)pObj; int32_t code; - tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code);; + tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code); + ; if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code);; + tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code);; + tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code);; + tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code); + ; } return code; @@ -358,12 +362,15 @@ static int32_t jsonToSchema(const SJson* pJson, void* pObj) { SSchema* pNode = (SSchema*)pObj; int32_t code; - tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code);; + tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code); + ; if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code);; + tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code);; + tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name); @@ -415,21 +422,27 @@ static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) { STableMeta* pNode = (STableMeta*)pObj; int32_t code; - tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code);; + tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code); + ; if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code);; + tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code);; + tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code);; + tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code);; + tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code);; + tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo); @@ -605,7 +618,8 @@ static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) { int32_t code = jsonToLogicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code);; + tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs); @@ -881,7 +895,8 @@ static int32_t jsonToLogicSubplan(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkLogicSubplanRootNode, (SNode**)&pNode->pNode); } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkLogicSubplanType, pNode->subplanType, code);; + tjsonGetNumberValue(pJson, jkLogicSubplanType, pNode->subplanType, code); + ; } int32_t objSize = 0; if (TSDB_CODE_SUCCESS == code) { @@ -1121,25 +1136,31 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanRatio, &pNode->ratio); } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired, code);; + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkTableScanPhysiPlanDynamicScanFuncs, &pNode->pDynamicScanFuncs); } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableScanPhysiPlanInterval, pNode->interval, code);; + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanInterval, pNode->interval, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableScanPhysiPlanOffset, pNode->offset, code);; + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanOffset, pNode->offset, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSliding, pNode->sliding, code);; + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSliding, pNode->sliding, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableScanPhysiPlanIntervalUnit, pNode->intervalUnit, code);; + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanIntervalUnit, pNode->intervalUnit, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit, code);; + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit, code); + ; } return code; @@ -1185,7 +1206,8 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkSysTableScanPhysiPlanShowRewrite, &pNode->showRewrite); } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId, code);; + tjsonGetNumberValue(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId, code); + ; } return code; @@ -1269,7 +1291,8 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);; + tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions); @@ -1431,10 +1454,12 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkWindowPhysiPlanTsPk, (SNode**)&pNode->pTspk); } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkWindowPhysiPlanTriggerType, pNode->triggerType, code);; + tjsonGetNumberValue(pJson, jkWindowPhysiPlanTriggerType, pNode->triggerType, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkWindowPhysiPlanWatermark, pNode->watermark, code);; + tjsonGetNumberValue(pJson, jkWindowPhysiPlanWatermark, pNode->watermark, code); + ; } return code; @@ -1530,7 +1555,8 @@ static int32_t jsonToPhysiFillNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode, code);; + tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillPhysiPlanWStartTs, &pNode->pWStartTs); @@ -1569,7 +1595,8 @@ static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysiWindowNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap, code);; + tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap, code); + ; } return code; @@ -1731,7 +1758,8 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { int32_t code = tjsonToObject(pJson, jkSubplanId, jsonToSubplanId, &pNode->id); if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkSubplanType, pNode->subplanType, code);; + tjsonGetNumberValue(pJson, jkSubplanType, pNode->subplanType, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanMsgType, &pNode->msgType); @@ -1921,7 +1949,8 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId); } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code);; + tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName); @@ -2175,7 +2204,8 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code);; + tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft); @@ -2209,7 +2239,8 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code);; + tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code); + ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList); @@ -2415,10 +2446,12 @@ static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) { int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr); if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code);; + tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code); + ; } if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code);; + tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code); + ; } return code; @@ -2525,7 +2558,8 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) { SFillNode* pNode = (SFillNode*)pObj; int32_t code; - tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code);; + tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code); + ; if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues); } @@ -2724,7 +2758,8 @@ static int32_t setOperatorToJson(const void* pObj, SJson* pJson) { static int32_t jsonToSetOperator(const SJson* pJson, void* pObj) { SSetOperator* pNode = (SSetOperator*)pObj; - int32_t code = tjsonGetNumberValue(pJson, jkSetOperatorOpType, pNode->opType); + int32_t code = TSDB_CODE_SUCCESS; + tjsonGetNumberValue(pJson, jkSetOperatorOpType, pNode->opType, code); if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkSetOperatorProjections, &pNode->pProjectionList); } @@ -3122,7 +3157,8 @@ static int32_t jsonToNode(const SJson* pJson, void* pObj) { SNode* pNode = (SNode*)pObj; int32_t code; - tjsonGetNumberValue(pJson, jkNodeType, pNode->type, code);; + tjsonGetNumberValue(pJson, jkNodeType, pNode->type, code); + ; if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, nodesNodeName(pNode->type), jsonToSpecificNode, pNode); if (TSDB_CODE_SUCCESS != code) { From 3e65caccaa07ed9b76232bc7147fcf616318c3bd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 14 May 2022 19:54:18 +0800 Subject: [PATCH 38/71] enh(sync): add syncStartStandBy --- source/libs/sync/src/syncIO.c | 2 +- .../libs/sync/test/syncConfigChangeTest.cpp | 39 +++++++++++++++---- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 209bd25c2c..1c5bb2914c 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -15,12 +15,12 @@ #include "syncIO.h" #include +#include "os.h" #include "syncMessage.h" #include "syncUtil.h" #include "tglobal.h" #include "ttimer.h" #include "tutil.h" -#include "os.h" SSyncIO *gSyncIO = NULL; diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index e91c6d33fb..4455d25c59 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -1,11 +1,11 @@ #include #include +#include "os.h" #include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" #include "syncUtil.h" #include "wal.h" -#include "os.h" void logTest() { sTrace("--- sync log test: trace"); @@ -114,7 +114,7 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pCfg->myIndex = 0; pCfg->replicaNum = 1; pCfg->nodeInfo[0].nodePort = gPorts[myIndex]; - taosGetFqdn(pCfg->nodeInfo[myIndex].nodeFqdn); + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); } else { pCfg->myIndex = myIndex; @@ -148,7 +148,23 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* return rid; } -void usage(char* exe) { printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy \n", exe); } +void configChange(int64_t rid, int32_t replicaNum, int32_t myIndex) { + SSyncCfg syncCfg; + + syncCfg.myIndex = myIndex; + syncCfg.replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + syncCfg.nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(syncCfg.nodeInfo[i].nodeFqdn); + } + + syncReconfig(rid, &syncCfg); +} + +void usage(char* exe) { + printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy isConfigChange \n", exe); +} SRpcMsg* createRpcMsg(int i, int count, int myIndex) { SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); @@ -163,7 +179,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) { int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; - if (argc != 6) { + if (argc != 7) { usage(argv[0]); exit(-1); } @@ -173,12 +189,15 @@ int main(int argc, char** argv) { int32_t lastApplyIndex = atoi(argv[3]); int32_t writeRecordNum = atoi(argv[4]); bool isStandBy = atoi(argv[5]); + bool isConfigChange = atoi(argv[6]); gSnapshotLastApplyIndex = lastApplyIndex; - assert(replicaNum >= 1 && replicaNum <= 5); - assert(myIndex >= 0 && myIndex < replicaNum); - assert(lastApplyIndex >= -1); - assert(writeRecordNum >= 0); + if (!isStandBy) { + assert(replicaNum >= 1 && replicaNum <= 5); + assert(myIndex >= 0 && myIndex < replicaNum); + assert(lastApplyIndex >= -1); + assert(writeRecordNum >= 0); + } init(); int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); @@ -200,6 +219,10 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); assert(pSyncNode != NULL); + if (isConfigChange) { + configChange(rid, replicaNum, myIndex); + } + //--------------------------- int32_t alreadySend = 0; while (1) { From 24aba2479146f679091eb51c873586827a9fea31 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 14 May 2022 20:20:13 +0800 Subject: [PATCH 39/71] fix: some problems of parser and planner --- source/client/src/clientImpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f879838d63..090638a4d1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -172,7 +172,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = pStmtCb, - .pUser = pTscObj->user}; + .pUser = pTscObj->user, + .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER))}; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); From cc021699a84bf9b109731e9bde2e426f054f738a Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 14 May 2022 20:34:59 +0800 Subject: [PATCH 40/71] fix(os): make tdb lib to static --- packaging/release.sh | 1 - packaging/tools/install.sh | 6 ------ source/libs/tdb/CMakeLists.txt | 2 +- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index ef3018a913..9230cafa85 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -67,7 +67,6 @@ bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos ${compi cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : cp ${compile_dir}/build/lib/libtaos.so ${install_dir}/lib/ -cp ${compile_dir}/build/lib/libtdb.so ${install_dir}/lib/ cp ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries" cp -rf ${compile_dir}/build/lib/pkgconfig ${install_dir}/lib/ > /dev/null || echo -e "failed to copy pkgconfig directory" diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 740d356f80..d2d52af955 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -215,15 +215,9 @@ function install_lib() { ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - ${csudo} ln -s ${install_main_dir}/lib/libtdb.* ${lib_link_dir}/libtdb.so.1 - ${csudo} ln -s ${lib_link_dir}/libtdb.so.1 ${lib_link_dir}/libtdb.so - if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : - - ${csudo} ln -s ${install_main_dir}/lib/libtdb.* ${lib64_link_dir}/libtdb.so.1 || : - ${csudo} ln -s ${lib64_link_dir}/libtdb.so.1 ${lib64_link_dir}/libtdb.so || : fi ${csudo} ldconfig diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 722f6bddef..01490030f2 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ # tdb -add_library(tdb SHARED "") +add_library(tdb STATIC "") target_sources(tdb PRIVATE "src/db/tdbPCache.c" From 71c63a9bb39110a5184682f32a06fc85f5b56b9e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 14 May 2022 20:41:05 +0800 Subject: [PATCH 41/71] fix: some problems of parser and planner --- source/libs/parser/src/parTranslater.c | 2 +- source/libs/parser/test/parSelectTest.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6f9faaa6f0..6427e13ae9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -492,7 +492,7 @@ static int32_t parseTimeFromValueNode(SValueNode* pVal) { } else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) { pVal->datum.i = pVal->datum.b; return TSDB_CODE_SUCCESS; - } else if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) { + } else if (IS_VAR_DATA_TYPE(pVal->node.resType.type) || TSDB_DATA_TYPE_TIMESTAMP == pVal->node.resType.type) { if (TSDB_CODE_SUCCESS == taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes, pVal->node.resType.precision, tsDaylight)) { return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 23b82d54bd..821f480b20 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -187,7 +187,7 @@ TEST_F(ParserSelectTest, semanticError) { run("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_WRONG_VALUE_TYPE - run("SELECT timestamp '2010' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); + run("SELECT timestamp '2010a' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, From 4a6bd8b9931df5befe4272c2a58da08365c4e076 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sat, 14 May 2022 20:43:50 +0800 Subject: [PATCH 42/71] test : add case for UDF functions --- tests/system-test/0-others/udfTest.py | 543 ++++++++++++++++++++++++++ 1 file changed, 543 insertions(+) create mode 100644 tests/system-test/0-others/udfTest.py diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py new file mode 100644 index 0000000000..c2ae837160 --- /dev/null +++ b/tests/system-test/0-others/udfTest.py @@ -0,0 +1,543 @@ +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +import subprocess + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def prepare_udf_so(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + + libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + os.system("mkdir /tmp/udf/") + os.system("sudo cp %s /tmp/udf/ "%libudf1.replace("\n" ,"")) + os.system("sudo cp %s /tmp/udf/ "%libudf2.replace("\n" ,"")) + + + def prepare_data(self): + + tdSql.execute("use db") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdSql.execute("create table tb (ts timestamp , num1 int , num2 int, num3 double , num4 binary(30))") + tdSql.execute( + f'''insert into tb values + ( '2020-04-21 01:01:01.000', NULL, 1, 1, "binary1" ) + ( '2020-10-21 01:01:01.000', 1, 1, 1.11, "binary1" ) + ( '2020-12-31 01:01:01.000', 2, 22222, 22, "binary1" ) + ( '2021-01-01 01:01:06.000', 3, 33333, 33, "binary1" ) + ( '2021-05-07 01:01:10.000', 4, 44444, 44, "binary1" ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ( '2021-09-30 01:01:16.000', 5, 55555, 55, "binary1" ) + ( '2022-02-01 01:01:20.000', 6, 66666, 66, "binary1" ) + ( '2022-10-28 01:01:26.000', 0, 00000, 00, "binary1" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -88, "binary1" ) + ( '2022-12-31 01:01:36.000', 9, -9999999, -99, "binary1" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ''' + ) + + + def create_udf_function(self): + + for i in range(10): + # create scalar functions + tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + # drop functions + + tdSql.execute("drop function udf1") + tdSql.execute("drop function udf2") + + functions = tdSql.getResult("show functions") + for function in functions: + if "udf1" in function[0] or "udf2" in function[0]: + tdLog.info("drop udf functions failed ") + tdLog.exit("drop udf functions failed") + + tdLog.info("drop two udf functions success ") + + # create scalar functions + tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + def basic_udf_query(self): + + # scalar functions + + tdSql.execute("use db ") + tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(0,2,1) + tdSql.checkData(0,3,88) + tdSql.checkData(0,4,1.000000000) + tdSql.checkData(0,5,88) + tdSql.checkData(0,6,"binary1") + tdSql.checkData(0,7,88) + + tdSql.checkData(3,0,3) + tdSql.checkData(3,1,88) + tdSql.checkData(3,2,33333) + tdSql.checkData(3,3,88) + tdSql.checkData(3,4,33.000000000) + tdSql.checkData(3,5,88) + tdSql.checkData(3,6,"binary1") + tdSql.checkData(3,7,88) + + tdSql.checkData(11,0,None) + tdSql.checkData(11,1,None) + tdSql.checkData(11,2,None) + tdSql.checkData(11,3,None) + tdSql.checkData(11,4,None) + tdSql.checkData(11,5,None) + tdSql.checkData(11,6,"binary1") + tdSql.checkData(11,7,88) + + tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(0,2,None) + tdSql.checkData(0,3,None) + tdSql.checkData(0,4,None) + tdSql.checkData(0,5,None) + tdSql.checkData(0,6,None) + tdSql.checkData(0,7,None) + + tdSql.checkData(20,0,8) + tdSql.checkData(20,1,88) + tdSql.checkData(20,2,88888) + tdSql.checkData(20,3,88) + tdSql.checkData(20,4,888) + tdSql.checkData(20,5,88) + tdSql.checkData(20,6,88) + tdSql.checkData(20,7,88) + + + # aggregate functions + tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb") + tdSql.checkData(0,0,15.362291496) + tdSql.checkData(0,1,10000949.553189287) + tdSql.checkData(0,2,168.633425216) + + # Arithmetic compute + tdSql.query("select udf2(num1)+100 ,udf2(num2)-100, udf2(num3)*100 ,udf2(num3)/100 from tb") + tdSql.checkData(0,0,115.362291496) + tdSql.checkData(0,1,10000849.553189287) + tdSql.checkData(0,2,16863.342521576) + tdSql.checkData(0,3,1.686334252) + + tdSql.query("select udf2(c1) ,udf2(c6) from stb1 ") + tdSql.checkData(0,0,25.514701644) + tdSql.checkData(0,1,265.247614504) + + tdSql.query("select udf2(c1)+100 ,udf2(c6)-100 ,udf2(c1)*100 ,udf2(c6)/100 from stb1 ") + tdSql.checkData(0,0,125.514701644) + tdSql.checkData(0,1,165.247614504) + tdSql.checkData(0,2,2551.470164435) + tdSql.checkData(0,3,2.652476145) + + # # bug for crash when query sub table + # tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") + # tdSql.checkData(0,0,378.215547010) + # tdSql.checkData(0,1,353.808067460) + # tdSql.checkData(0,2,2114.237451187) + # tdSql.checkData(0,3,2.125468151) + + tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 ") + tdSql.checkData(0,0,490.358032462) + tdSql.checkData(0,1,400.460106627) + tdSql.checkData(0,2,2551.470164435) + tdSql.checkData(0,3,2.652476145) + + + # regular table with aggregate functions + + tdSql.error("select udf1(num1) , count(num1) from tb;") + tdSql.error("select udf1(num1) , avg(num1) from tb;") + tdSql.error("select udf1(num1) , twa(num1) from tb;") + tdSql.error("select udf1(num1) , irate(num1) from tb;") + tdSql.error("select udf1(num1) , sum(num1) from tb;") + tdSql.error("select udf1(num1) , stddev(num1) from tb;") + tdSql.error("select udf1(num1) , mode(num1) from tb;") + tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") + # stable + tdSql.error("select udf1(c1) , count(c1) from stb1;") + tdSql.error("select udf1(c1) , avg(c1) from stb1;") + tdSql.error("select udf1(c1) , twa(c1) from stb1;") + tdSql.error("select udf1(c1) , irate(c1) from stb1;") + tdSql.error("select udf1(c1) , sum(c1) from stb1;") + tdSql.error("select udf1(c1) , stddev(c1) from stb1;") + tdSql.error("select udf1(c1) , mode(c1) from stb1;") + tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") + + # regular table with select functions + + tdSql.query("select udf1(num1) , max(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select floor(num1) , max(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select udf1(num1) , min(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select ceil(num1) , min(num1) from tb;") + tdSql.checkRows(1) + tdSql.error("select udf1(num1) , first(num1) from tb;") + + tdSql.error("select abs(num1) , first(num1) from tb;") + + tdSql.error("select udf1(num1) , last(num1) from tb;") + + tdSql.error("select round(num1) , last(num1) from tb;") + + tdSql.query("select udf1(num1) , top(num1,1) from tb;") + tdSql.checkRows(1) + tdSql.query("select udf1(num1) , bottom(num1,1) from tb;") + tdSql.checkRows(1) + tdSql.error("select udf1(num1) , last_row(num1) from tb;") + + tdSql.error("select round(num1) , last_row(num1) from tb;") + + + # stable + tdSql.query("select udf1(c1) , max(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select abs(c1) , max(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select udf1(c1) , min(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select floor(c1) , min(c1) from stb1;") + tdSql.checkRows(1) + tdSql.error("select udf1(c1) , first(c1) from stb1;") + + tdSql.error("select udf1(c1) , last(c1) from stb1;") + + tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select abs(c1) , top(c1 ,1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select udf1(c1) , bottom(c1,1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;") + tdSql.checkRows(1) + + tdSql.error("select udf1(c1) , last_row(c1) from stb1;") + tdSql.error("select ceil(c1) , last_row(c1) from stb1;") + + # regular table with compute functions + + tdSql.query("select udf1(num1) , abs(num1) from tb;") + tdSql.checkRows(12) + tdSql.query("select floor(num1) , abs(num1) from tb;") + tdSql.checkRows(12) + + # # bug need fix + + # tdSql.query("select udf1(num1) , csum(num1) from tb;") + # tdSql.checkRows(12) + # tdSql.query("select ceil(num1) , csum(num1) from tb;") + # tdSql.checkRows(12) + # tdSql.query("select udf1(c1) , csum(c1) from stb1;") + # tdSql.checkRows(25) + # tdSql.query("select floor(c1) , csum(c1) from stb1;") + # tdSql.checkRows(25) + + # stable with compute functions + tdSql.query("select udf1(c1) , abs(c1) from stb1;") + tdSql.checkRows(25) + tdSql.query("select abs(c1) , ceil(c1) from stb1;") + tdSql.checkRows(25) + + # nest query + tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 order by ts;") + tdSql.checkRows(25) + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(1,0,88) + tdSql.checkData(1,1,8) + + tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") + tdSql.checkRows(13) + tdSql.checkData(0,0,88) + tdSql.checkData(0,1,8) + tdSql.checkData(1,0,88) + tdSql.checkData(1,1,7) + + # bug fix for crash + # order by udf function result + # for _ in range(50): + # tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)") + + # udf functions with filter + + tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 where c1 is null order by ts;") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + + tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") + tdSql.checkRows(3) + tdSql.checkData(0,0,9) + tdSql.checkData(0,1,88) + tdSql.checkData(0,2,-99.990000000) + tdSql.checkData(0,3,88) + + # udf functions with join + ts_start = 1652517451000 + tdSql.execute("create stable st (ts timestamp , c1 int , c2 int ,c3 double ,c4 double ) tags(ind int)") + tdSql.execute("create table sub1 using st tags(1)") + tdSql.execute("create table sub2 using st tags(2)") + + for i in range(10): + ts = ts_start + i *1000 + tdSql.execute(" insert into sub1 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + tdSql.execute(" insert into sub2 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + + tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,0) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,10) + + tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,88) + tdSql.checkData(0,1,88) + tdSql.checkData(1,0,88) + tdSql.checkData(1,1,88) + + tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,88) + tdSql.checkData(0,2,0) + tdSql.checkData(0,3,88) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,88) + tdSql.checkData(1,2,10) + tdSql.checkData(1,3,88) + + tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,16.881943016) + tdSql.checkData(0,1,168.819430161) + tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + + # udf functions with group by + tdSql.query("select udf1(c1) from ct1 group by c1") + tdSql.checkRows(10) + tdSql.query("select udf1(c1) from stb1 group by c1") + tdSql.checkRows(11) + tdSql.query("select c1,c2, udf1(c1,c2) from ct1 group by c1,c2") + tdSql.checkRows(10) + tdSql.query("select c1,c2, udf1(c1,c2) from stb1 group by c1,c2") + tdSql.checkRows(11) + + tdSql.query("select udf2(c1) from ct1 group by c1") + tdSql.checkRows(10) + tdSql.query("select udf2(c1) from stb1 group by c1") + tdSql.checkRows(11) + tdSql.query("select c1,c2, udf2(c1,c6) from ct1 group by c1,c2") + tdSql.checkRows(10) + tdSql.query("select c1,c2, udf2(c1,c6) from stb1 group by c1,c2") + tdSql.checkRows(11) + tdSql.query("select udf2(c1) from stb1 group by udf1(c1)") + tdSql.checkRows(2) + tdSql.query("select udf2(c1) from stb1 group by floor(c1)") + tdSql.checkRows(11) + + # udf mix with order by + tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)") + tdSql.checkRows(11) + + + def multi_cols_udf(self): + tdSql.query("select num1,num2,num3,udf1(num1,num2,num3) from tb") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,1) + tdSql.checkData(0,2,1.000000000) + tdSql.checkData(0,3,None) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) + tdSql.checkData(1,2,1.110000000) + tdSql.checkData(1,3,88) + + tdSql.query("select c1,c6,udf1(c1,c6) from stb1 order by ts") + tdSql.checkData(1,0,8) + tdSql.checkData(1,1,88.880000000) + tdSql.checkData(1,2,88) + + tdSql.query("select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;") + tdSql.checkRows(22) + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + + + def unexpected_create(self): + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + + def loop_kill_udfd(self): + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + cfgPath = buildPath + "/../sim/dnode1/cfg" + udfdPath = buildPath +'/build/bin/udfd' + + for i in range(5): + + tdLog.info(" loop restart udfd %d_th" % i) + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + # stop udfd cmds + get_processID = "ps -ef | grep -w udfd | grep 'root' | grep -v grep| grep -v defunct | awk '{print $2}'" + processID = subprocess.check_output(get_processID, shell=True).decode("utf-8") + stop_udfd = " kill -9 %s" % processID + os.system(stop_udfd) + + time.sleep(2) + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + + # # start udfd cmds + # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &" + # tdLog.info("start udfd : %s " % start_udfd) + + + def restart_taosd_query_udf(self): + + for i in range(5): + time.sleep(5) + tdLog.info(" this is %d_th restart taosd " %i) + tdSql.execute("use db ") + tdSql.query("select count(*) from stb1") + tdSql.checkRows(1) + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + tdDnodes.stop(1) + time.sleep(2) + tdDnodes.start(1) + time.sleep(5) + + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + self.prepare_udf_so() + self.prepare_data() + self.create_udf_function() + self.basic_udf_query() + self.loop_kill_udfd() + # self.restart_taosd_query_udf() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 70171a66e018cea12d1f9f46a3b059113d91e6bb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 14 May 2022 20:52:46 +0800 Subject: [PATCH 43/71] enh(index): fix sanitizer error --- source/libs/executor/src/indexoperator.c | 4 + .../executor/test/index_executor_tests.cpp | 4 +- source/libs/index/inc/indexComm.h | 2 + source/libs/index/inc/indexInt.h | 9 +- source/libs/index/src/index.c | 92 +++++++++++--- source/libs/index/src/indexCache.c | 3 + source/libs/index/src/indexComm.c | 114 +++++++++++++++++- source/libs/index/src/indexFstUtil.c | 5 +- source/libs/index/src/indexTfile.c | 13 +- source/libs/index/test/jsonUT.cc | 20 +++ 10 files changed, 233 insertions(+), 33 deletions(-) diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index c17fcacf1f..2c204e9356 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -398,6 +398,10 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { output->status = SFLT_ACCURATE_INDEX; } + if (ctx->noExec) { + SIF_RET(code); + } + return operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); _return: taosMemoryFree(params); diff --git a/source/libs/executor/test/index_executor_tests.cpp b/source/libs/executor/test/index_executor_tests.cpp index 5b03da034e..2449bd1da1 100644 --- a/source/libs/executor/test/index_executor_tests.cpp +++ b/source/libs/executor/test/index_executor_tests.cpp @@ -249,7 +249,7 @@ TEST(testCase, index_filter_varify) { sifMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); SIdxFltStatus st = idxGetFltStatus(opNode); - EXPECT_EQ(st, SFLT_COARSE_INDEX); + EXPECT_EQ(st, SFLT_ACCURATE_INDEX); nodesDestroyNode(res); } { @@ -269,7 +269,7 @@ TEST(testCase, index_filter_varify) { sifMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); SIdxFltStatus st = idxGetFltStatus(opNode); - EXPECT_EQ(st, SFLT_COARSE_INDEX); + EXPECT_EQ(st, SFLT_ACCURATE_INDEX); nodesDestroyNode(res); } } diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 4cab71f92c..043404f48f 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -37,6 +37,8 @@ TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b); _cache_range_compare indexGetCompare(RangeType ty); +int32_t indexConvertData(void* src, int8_t type, void** dst); + #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 7b7050d80e..27c380beaf 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -46,9 +46,7 @@ typedef struct SIndexStat { } SIndexStat; struct SIndex { -#ifdef USE_LUCENE - index_t* index; -#endif + int64_t refId; void* cache; void* tindex; SHashObj* colObj; // < field name, field id> @@ -124,6 +122,11 @@ typedef struct TFileCacheKey { int indexFlushCacheToTFile(SIndex* sIdx, void*); +int64_t indexAddRef(void* p); +int32_t indexRemoveRef(int64_t ref); +void indexAcquireRef(int64_t ref); +void indexReleaseRef(int64_t ref); + int32_t indexSerialCacheKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index d56413f840..46f2f7a93b 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -19,7 +19,10 @@ #include "indexInt.h" #include "indexTfile.h" #include "indexUtil.h" +#include "tcoding.h" +#include "tdataformat.h" #include "tdef.h" +#include "tref.h" #include "tsched.h" #ifdef USE_LUCENE @@ -27,36 +30,40 @@ #endif #define INDEX_NUM_OF_THREADS 4 -#define INDEX_QUEUE_SIZE 200 +#define INDEX_QUEUE_SIZE 200 -void* indexQhandle = NULL; - -#define INDEX_DATA_BOOL_NULL 0x02 -#define INDEX_DATA_TINYINT_NULL 0x80 -#define INDEX_DATA_SMALLINT_NULL 0x8000 -#define INDEX_DATA_INT_NULL 0x80000000L -#define INDEX_DATA_BIGINT_NULL 0x8000000000000000L +#define INDEX_DATA_BOOL_NULL 0x02 +#define INDEX_DATA_TINYINT_NULL 0x80 +#define INDEX_DATA_SMALLINT_NULL 0x8000 +#define INDEX_DATA_INT_NULL 0x80000000L +#define INDEX_DATA_BIGINT_NULL 0x8000000000000000L #define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL -#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN -#define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF -#define INDEX_DATA_BINARY_NULL 0xFF -#define INDEX_DATA_JSON_NULL 0xFFFFFFFF -#define INDEX_DATA_JSON_null 0xFFFFFFFE +#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF +#define INDEX_DATA_BINARY_NULL 0xFF +#define INDEX_DATA_JSON_NULL 0xFFFFFFFF +#define INDEX_DATA_JSON_null 0xFFFFFFFE #define INDEX_DATA_JSON_NOT_NULL 0x01 -#define INDEX_DATA_UTINYINT_NULL 0xFF +#define INDEX_DATA_UTINYINT_NULL 0xFF #define INDEX_DATA_USMALLINT_NULL 0xFFFF -#define INDEX_DATA_UINT_NULL 0xFFFFFFFF -#define INDEX_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL +#define INDEX_DATA_UINT_NULL 0xFFFFFFFF +#define INDEX_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL -#define INDEX_DATA_NULL_STR "NULL" +#define INDEX_DATA_NULL_STR "NULL" #define INDEX_DATA_NULL_STR_L "null" +void* indexQhandle = NULL; +int32_t indexRefMgt; + +static void indexDestroy(void* sIdx); + void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); + indexRefMgt = taosOpenRef(10, indexDestroy); } void indexCleanUp() { // refacto later @@ -100,7 +107,12 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { sIdx->cVersion = 1; sIdx->path = tstrdup(path); taosThreadMutexInit(&sIdx->mtx, NULL); + + sIdx->refId = indexAddRef(sIdx); + taosAcquireRef(indexRefMgt, sIdx->refId); + *index = sIdx; + return 0; END: @@ -112,8 +124,9 @@ END: return -1; } -void indexClose(SIndex* sIdx) { - void* iter = taosHashIterate(sIdx->colObj, NULL); +void indexDestroy(void* handle) { + SIndex* sIdx = handle; + void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; if (*pCache) { @@ -128,6 +141,27 @@ void indexClose(SIndex* sIdx) { taosMemoryFree(sIdx); return; } +void indexClose(SIndex* sIdx) { + indexReleaseRef(sIdx->refId); + indexRemoveRef(sIdx->refId); +} +int64_t indexAddRef(void* p) { + // impl + return taosAddRef(indexRefMgt, p); +} +int32_t indexRemoveRef(int64_t ref) { + // impl later + return taosRemoveRef(indexRefMgt, ref); +} + +void indexAcquireRef(int64_t ref) { + // impl + taosAcquireRef(indexRefMgt, ref); +} +void indexReleaseRef(int64_t ref) { + // impl + taosReleaseRef(indexRefMgt, ref); +} int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { // TODO(yihao): reduce the lock range @@ -222,6 +256,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy tm->operType = oper; tm->colType = colType; +#if 0 tm->colName = (char*)taosMemoryCalloc(1, nColName + 1); memcpy(tm->colName, colName, nColName); tm->nColName = nColName; @@ -229,6 +264,22 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); memcpy(tm->colVal, colVal, nColVal); tm->nColVal = nColVal; +#endif + +#if 1 + + tm->colName = (char*)taosMemoryCalloc(1, nColName + 1); + memcpy(tm->colName, colName, nColName); + tm->nColName = nColName; + + char* buf = NULL; + int32_t len = indexConvertData((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf); + assert(len != -1); + + tm->colVal = buf; + tm->nColVal = len; + +#endif return tm; } @@ -457,6 +508,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { } else { indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000); } + indexReleaseRef(sIdx->refId); return ret; } void iterateValueDestroy(IterateValue* value, bool destroy) { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 5294ac8c19..d4231619ec 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -460,8 +460,11 @@ int indexCacheSchedToMerge(IndexCache* pCache) { schedMsg.fp = doMergeWork; schedMsg.ahandle = pCache; schedMsg.thandle = NULL; + // schedMsg.thandle = taosMemoryCalloc(1, sizeof(int64_t)); + // memcpy((char*)(schedMsg.thandle), (char*)&(pCache->index->refId), sizeof(int64_t)); schedMsg.msg = NULL; + indexAcquireRef(pCache->index->refId); taosScheduleTask(indexQhandle, &schedMsg); return 0; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 9e85a6680a..ac26ed1fab 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -16,25 +16,33 @@ #include "indexComm.h" #include "index.h" #include "indexInt.h" +#include "tcoding.h" #include "tcompare.h" +#include "tdataformat.h" char JSON_COLUMN[] = "JSON"; char JSON_VALUE_DELIM = '&'; +static __compar_fn_t indexGetCompar(int8_t type) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + return (__compar_fn_t)strcmp; + } + return getComparFunc(type, 0); +} static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); + __compar_fn_t func = indexGetCompar(type); return tDoCommpare(func, QUERY_LESS_THAN, a, b); } static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); + __compar_fn_t func = indexGetCompar(type); return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); } static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); + __compar_fn_t func = indexGetCompar(type); return tDoCommpare(func, QUERY_GREATER_THAN, a, b); } static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); + __compar_fn_t func = indexGetCompar(type); return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); } @@ -120,3 +128,101 @@ char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { return buf; } + +int32_t indexConvertData(void* src, int8_t type, void** dst) { + int tlen = -1; + switch (type) { + case TSDB_DATA_TYPE_TIMESTAMP: + tlen = taosEncodeFixedI64(NULL, *(int64_t*)src); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeFixedI64(dst, *(int64_t*)src); + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeFixedU8(dst, *(uint8_t*)src); + break; + case TSDB_DATA_TYPE_TINYINT: + tlen = taosEncodeFixedI8(NULL, *(uint8_t*)src); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeFixedI8(dst, *(uint8_t*)src); + break; + case TSDB_DATA_TYPE_SMALLINT: + tlen = taosEncodeFixedI16(NULL, *(int16_t*)src); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeFixedI16(dst, *(int16_t*)src); + break; + case TSDB_DATA_TYPE_USMALLINT: + tlen = taosEncodeFixedU16(NULL, *(uint16_t*)src); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeFixedU16(dst, *(uint16_t*)src); + break; + case TSDB_DATA_TYPE_INT: + tlen = taosEncodeFixedI32(NULL, *(int32_t*)src); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeFixedI32(dst, *(int32_t*)src); + break; + case TSDB_DATA_TYPE_FLOAT: + tlen = taosEncodeBinary(NULL, src, sizeof(float)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, src, sizeof(float)); + break; + case TSDB_DATA_TYPE_UINT: + tlen = taosEncodeFixedU32(NULL, *(uint32_t*)src); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeFixedU32(dst, *(uint32_t*)src); + break; + case TSDB_DATA_TYPE_BIGINT: + tlen = taosEncodeFixedI64(NULL, *(uint32_t*)src); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeFixedI64(dst, *(uint32_t*)src); + break; + case TSDB_DATA_TYPE_DOUBLE: + tlen = taosEncodeBinary(NULL, src, sizeof(double)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, src, sizeof(double)); + break; + case TSDB_DATA_TYPE_UBIGINT: + tlen = taosEncodeFixedU64(NULL, *(uint32_t*)src); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeFixedU64(dst, *(uint32_t*)src); + break; + case TSDB_DATA_TYPE_NCHAR: { + tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); + + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY +#if 1 + tlen = taosEncodeBinary(NULL, src, strlen(src)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, src, strlen(src)); + break; +#endif + } + case TSDB_DATA_TYPE_VARBINARY: +#if 1 + tlen = taosEncodeBinary(NULL, src, strlen(src)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, src, strlen(src)); + break; +#endif + default: + TASSERT(0); + break; + } + *dst = *dst - tlen; + if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY && + type == TSDB_DATA_TYPE_VARCHAR) { + uint8_t* p = *dst; + for (int i = 0; i < tlen; i++) { + if (p[i] == 0) { + p[i] = (uint8_t)'0'; + } + } + } + return tlen; +} diff --git a/source/libs/index/src/indexFstUtil.c b/source/libs/index/src/indexFstUtil.c index ec9a6943dc..a980c6b740 100644 --- a/source/libs/index/src/indexFstUtil.c +++ b/source/libs/index/src/indexFstUtil.c @@ -82,7 +82,10 @@ FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { str->ref = 1; str->len = len; str->data = taosMemoryMalloc(len * sizeof(uint8_t)); - memcpy(str->data, data, len); + + if (data != NULL) { + memcpy(str->data, data, len); + } FstSlice s = {.str = str, .start = 0, .end = len - 1}; return s; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 4cc2a4975f..b787da117d 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -469,13 +469,19 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { FstSlice* s = &rt->data; - char* ch = (char*)fstSliceData(s, NULL); - if (0 != strncmp(ch, p, skip)) { + int32_t sz = 0; + char* ch = (char*)fstSliceData(s, &sz); + char* tmp = taosMemoryCalloc(1, sz + 1); + memcpy(tmp, ch, sz); + + if (0 != strncmp(tmp, p, skip)) { swsResultDestroy(rt); + taosMemoryFree(tmp); break; } - TExeCond cond = cmpFn(ch + skip, tem->colVal, tem->colType); + TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType)); + if (MATCH == cond) { tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); } else if (CONTINUE == cond) { @@ -483,6 +489,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR swsResultDestroy(rt); break; } + taosMemoryFree(tmp); swsResultDestroy(rt); } streamWithStateDestroy(st); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 08d58da07f..3de7cb66f2 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -17,12 +17,32 @@ #include "tutil.h" static std::string dir = "/tmp/json"; +static std::string logDir = "/tmp/log"; + +static void initLog() { + const char* defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + sDebugFlag = 143; + strcpy(tsLogDir, logDir.c_str()); + taosRemoveDir(tsLogDir); + taosMkDir(tsLogDir); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} class JsonEnv : public ::testing::Test { protected: virtual void SetUp() { + taosRemoveDir(logDir.c_str()); + taosMkDir(logDir.c_str()); taosRemoveDir(dir.c_str()); taosMkDir(dir.c_str()); printf("set up\n"); + + initLog(); opts = indexOptsCreate(); int ret = tIndexJsonOpen(opts, dir.c_str(), &index); assert(ret == 0); From b6677e1a5d9ba4cce15033e781bb6e148b8aed93 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 14 May 2022 22:13:27 +0800 Subject: [PATCH 44/71] fix(query): remove expired moving data operation during extract data from in-memory buffer. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 12 ------------ source/libs/executor/src/executorimpl.c | 10 ---------- 2 files changed, 22 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b293f1399d..927babc26c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2769,20 +2769,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } while (moveToNextRowInMem(pCheckInfo)); taosMemoryFreeClear(pSchema); // free the STSChema - assert(numOfRows <= maxRowsToRead); - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && numOfRows < maxRowsToRead) { - int32_t emptySize = maxRowsToRead - numOfRows; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, - numOfRows * pColInfo->info.bytes); - } - } - int64_t elapsedTime = taosGetTimestampUs() - st; tsdbDebug("%p build data block from cache completed, elapsed time:%" PRId64 " us, numOfRows:%d, numOfCols:%d, %s", pTsdbReadHandle, elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->idStr); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index cb8bee1e6a..2cbfb9b344 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2138,16 +2138,6 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchIn blockDataUpdateTsWindow(pBlock); } -static int32_t colIdSearchCompar(const void* p1, const void* p2) { - int32_t colId = *(int32_t*)p1; - SColMatchInfo* pInfo = (SColMatchInfo*)p2; - if (colId == pInfo->targetSlotId) { - return 0; - } - - return (colId < pInfo->colId) ? -1 : 1; -} - void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { if (keep) { return; From 65edd3296e0af441a7f66d8bf434738bec348997 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 May 2022 14:29:04 +0000 Subject: [PATCH 45/71] feat: alter super table --- source/dnode/mnode/impl/src/mndStb.c | 3 +- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaTable.c | 69 ++++++++++++++++++++++++ source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 10 ++-- source/dnode/vnode/src/tsdb/tsdbSma.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 44 +++++++++------ source/libs/parser/src/parTranslater.c | 2 +- 9 files changed, 110 insertions(+), 27 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 12e89277f4..5f1c4d93ce 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -383,9 +383,10 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.suid = pStb->uid; req.rollup = pStb->ast1Len > 0 ? 1 : 0; req.schema.nCols = pStb->numOfColumns; - req.schema.sver = 0; + req.schema.sver = pStb->version; req.schema.pSchema = pStb->pColumns; req.schemaTag.nCols = pStb->numOfTags; + req.schemaTag.nCols = 0; req.schemaTag.pSchema = pStb->pTags; if (req.rollup) { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9a36fc6eae..cb99a96923 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -77,6 +77,7 @@ int metaClose(SMeta* pMeta); int metaBegin(SMeta* pMeta); int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index d666bd22c1..e5377d543a 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -131,6 +131,75 @@ _err: return -1; } +int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + TDBC *pUidIdxc = NULL; + TDBC *pTbDbc = NULL; + const void *pData; + int nData; + int64_t oversion; + SDecoder dc = {0}; + int32_t ret; + int32_t c; + + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + if (ret < 0 || c) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + oversion = *(int64_t *)pData; + + tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); + ASSERT(ret == 0 && c == 0); + + ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + ASSERT(ret == 0); + + tDecoderInit(&dc, pData, nData); + metaDecodeEntry(&dc, &oStbEntry); + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = pReq->suid; + nStbEntry.name = pReq->name; + nStbEntry.stbEntry.schema = pReq->schema; + nStbEntry.stbEntry.schemaTag = pReq->schemaTag; + + metaWLock(pMeta); + // compare two entry + if (oStbEntry.stbEntry.schema.sver != pReq->schema.sver) { + if (oStbEntry.stbEntry.schema.nCols != pReq->schema.nCols) { + metaSaveToSkmDb(pMeta, &nStbEntry); + } + } + + // if (oStbEntry.stbEntry.schemaTag.sver != pReq->schemaTag.sver) { + // // change tag schema + // } + + // update table.db + metaSaveToTbDb(pMeta, &nStbEntry); + + // update uid index + tdbDbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); + + metaULock(pMeta); + tDecoderClear(&dc); + tdbDbcClose(pTbDbc); + tdbDbcClose(pUidIdxc); + return 0; +} + int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { SMetaEntry me = {0}; SMetaReader mr = {0}; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 996d789e24..8fbd1e24e1 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -91,7 +91,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p // TODO set to real sversion *pUid = 0; - int32_t sversion = 0; + int32_t sversion = 1; if (pHandle->sver != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d180799e58..76d5c3cb3a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -465,7 +465,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pTbData = (STbData *)pNode->pData; pCommitIter = pCommith->iters + i; - pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); // TODO: schema version + pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 1); // TODO: schema version if (pTSchema) { pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); @@ -912,7 +912,7 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) { while (bidx < nBlocks) { if (!pTSchema && !tsdbCommitIsSameFile(pCommith, bidx)) { // Set commit table - pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 0); // TODO: schema version + pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 1); // TODO: schema version if (!pTSchema) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b293f1399d..4e66477629 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -490,7 +490,7 @@ tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableG STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); - pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 0); + pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 1); int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn); int16_t* ids = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData; @@ -1618,7 +1618,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (pSchema1 == NULL) { // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); // TODO: use the real schemaVersion - pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); + pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1); } #ifdef TD_DEBUG_PRINT_ROW @@ -1637,7 +1637,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (pSchema2 == NULL) { // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); // TODO: use the real schemaVersion - pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); + pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1); } if (isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -2755,7 +2755,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->ekey = key; if (rv != TD_ROW_SVER(row)) { - pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); + pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 1); rv = TD_ROW_SVER(row); } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, @@ -3889,7 +3889,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch // NOTE: not add ref count for super table SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); - SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true); + SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 1, true); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index e878668654..1589513110 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -2084,7 +2084,7 @@ static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { // TODO: use the proper schema instead of 0, and cache STSchema in cache - STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 0); + STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 1); if (!pTSchema) { terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4f76bc5386..fe496d137f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -16,7 +16,7 @@ #include "vnd.h" static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); -static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -72,7 +72,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_STB: - if (vnodeProcessAlterStbReq(pVnode, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessAlterStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_STB: if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; @@ -398,20 +398,32 @@ _exit: return rcode; } -static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { - // ASSERT(0); -#if 0 - SVCreateTbReq vAlterTbReq = {0}; - vTrace("vgId:%d, process alter stb req", TD_VID(pVnode)); - tDeserializeSVCreateTbReq(pReq, &vAlterTbReq); - // TODO: to encapsule a free API - taosMemoryFree(vAlterTbReq.stbCfg.pSchema); - taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema); - if (vAlterTbReq.stbCfg.pRSmaParam) { - taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); +static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SVCreateStbReq req = {0}; + SDecoder dc = {0}; + + pRsp->msgType = TDMT_VND_ALTER_STB_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + tDecoderInit(&dc, pReq, len); + + // decode req + if (tDecodeSVCreateStbReq(&dc, &req) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&dc); + return -1; } - taosMemoryFree(vAlterTbReq.name); -#endif + + if (metaAlterSTable(pVnode->pMeta, version, &req) < 0) { + pRsp->code = terrno; + tDecoderClear(&dc); + return -1; + } + + tDecoderClear(&dc); + return 0; } @@ -514,7 +526,7 @@ static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSub if (pSchema) { taosMemoryFreeClear(pSchema); } - pSchema = metaGetTbTSchema(pMeta, msgIter->suid, 0); // TODO: use the real schema + pSchema = metaGetTbTSchema(pMeta, msgIter->suid, 1); // TODO: use the real schema if (pSchema) { suid = msgIter->suid; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 95ab1500cc..d848fb05e8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3688,7 +3688,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* req.type = TD_NORMAL_TABLE; req.name = strdup(pStmt->tableName); req.ntb.schema.nCols = LIST_LENGTH(pStmt->pCols); - req.ntb.schema.sver = 0; + req.ntb.schema.sver = 1; req.ntb.schema.pSchema = taosMemoryCalloc(req.ntb.schema.nCols, sizeof(SSchema)); if (NULL == req.name || NULL == req.ntb.schema.pSchema) { destroyCreateTbReq(&req); From 911cd1fe6694f74d843d2b943ba0531c3cae3a1b Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 14 May 2022 22:51:13 +0800 Subject: [PATCH 46/71] fix: reuse existing udf handles and teardown the handle later --- include/libs/function/function.h | 2 +- include/libs/function/tudf.h | 32 +++++----- source/libs/function/inc/udfc.h | 2 +- source/libs/function/src/tudf.c | 99 +++++++++++++++++++++++------- source/libs/function/test/runUdf.c | 6 +- source/libs/scalar/src/scalar.c | 14 +---- 6 files changed, 99 insertions(+), 56 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 8d0b93dde2..616aec8c02 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -309,7 +309,7 @@ void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); /** - * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf + * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf * @return error code */ int32_t udfcOpen(); diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index b5c38e14f4..b37dcd2b61 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -39,16 +39,6 @@ extern "C" { //====================================================================================== //begin API to taosd and qworker -typedef void *UdfcFuncHandle; - -/** - * setup udf - * @param udf, in - * @param handle, out - * @return error code - */ -int32_t setupUdf(char udfName[], UdfcFuncHandle *handle); - typedef struct SUdfColumnMeta { int16_t type; int32_t bytes; @@ -95,32 +85,42 @@ typedef struct SUdfInterBuf { char* buf; int8_t numOfResult; //zero or one } SUdfInterBuf; +typedef void *UdfcFuncHandle; +/** + * setup udf + * @param udf, in + * @param funcHandle, out + * @return error code + */ +int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle); // output: interBuf -int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf); +int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf); // input: block, state // output: newState -int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); +int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); // input: interBuf // output: resultData -int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); +int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); // input: interbuf1, interbuf2 // output: resultBuf -int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); +int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); // input: block // output: resultData -int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output); +int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output); /** * tearn down udf * @param handle * @return */ -int32_t teardownUdf(UdfcFuncHandle handle); +int32_t doTeardownUdf(UdfcFuncHandle handle); bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); + +int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output); // end API to taosd and qworker //============================================================================================================================= // begin API to UDF writer. diff --git a/source/libs/function/inc/udfc.h b/source/libs/function/inc/udfc.h index a693e476e8..f414c2b29e 100644 --- a/source/libs/function/inc/udfc.h +++ b/source/libs/function/inc/udfc.h @@ -43,7 +43,7 @@ int32_t setupUdf(SUdfInfo* udf, UdfcFuncHandle* handle); int32_t callUdf(UdfcFuncHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, int32_t *newStateSize, SSDataBlock *output); -int32_t teardownUdf(UdfcFuncHandle handle); +int32_t doTeardownUdf(UdfcFuncHandle handle); typedef struct SUdfSetupRequest { char udfName[16]; // diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index d9e3ff0a5b..f2a0b4ec2c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -310,6 +310,11 @@ enum { }; int64_t gUdfTaskSeqNum = 0; +typedef struct SUdfcFuncStub { + char udfName[TSDB_FUNC_NAME_LEN]; + UdfcFuncHandle handle; +} SUdfcFuncStub; + typedef struct SUdfcProxy { char udfdPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2]; uv_barrier_t initBarrier; @@ -325,6 +330,9 @@ typedef struct SUdfcProxy { QUEUE taskQueue; QUEUE uvProcTaskQueue; + uv_mutex_t udfStubsMutex; + SArray* udfStubs; // SUdfcFuncStub + int8_t initialized; } SUdfcProxy; @@ -1222,6 +1230,8 @@ int32_t udfcOpen() { atomic_store_8(&proxy->udfcState, UDFC_STATE_READY); proxy->udfcState = UDFC_STATE_READY; uv_barrier_wait(&proxy->initBarrier); + uv_mutex_init(&proxy->udfStubsMutex); + proxy->udfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub)); fnInfo("udfc initialized") return 0; } @@ -1238,6 +1248,8 @@ int32_t udfcClose() { uv_thread_join(&udfc->loopThread); uv_mutex_destroy(&udfc->taskQueueMutex); uv_barrier_destroy(&udfc->initBarrier); + taosArrayDestroy(udfc->udfStubs); + uv_mutex_destroy(&udfc->udfStubsMutex); udfc->udfcState = UDFC_STATE_INITAL; fnInfo("udfc cleaned up"); return 0; @@ -1259,7 +1271,7 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { return task->errCode; } -int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { +int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { fnInfo("udfc setup udf. udfName: %s", udfName); if (gUdfdProxy.udfcState != UDFC_STATE_READY) { return TSDB_CODE_UDF_INVALID_STATE; @@ -1287,7 +1299,7 @@ int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { task->session->outputLen = rsp->outputLen; task->session->bufSize = rsp->bufSize; if (task->errCode != 0) { - fnError("failed to setup udf. err: %d", task->errCode) + fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode) } else { fnInfo("sucessfully setup udf func handle. handle: %p", task->session); *funcHandle = task->session; @@ -1373,7 +1385,7 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf return err; } -int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { +int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { int8_t callType = TSDB_UDF_CALL_AGG_INIT; int32_t err = callUdf(handle, callType, NULL, NULL, NULL, NULL, interBuf); @@ -1383,7 +1395,7 @@ int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { // input: block, state // output: interbuf, -int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) { +int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) { int8_t callType = TSDB_UDF_CALL_AGG_PROC; int32_t err = callUdf(handle, callType, block, state, NULL, NULL, newState); return err; @@ -1391,7 +1403,7 @@ int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBu // input: interbuf1, interbuf2 // output: resultBuf -int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) { +int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) { int8_t callType = TSDB_UDF_CALL_AGG_MERGE; int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf); return err; @@ -1399,13 +1411,13 @@ int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInte // input: interBuf // output: resultData -int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { +int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { int8_t callType = TSDB_UDF_CALL_AGG_FIN; int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData); return err; } -int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam* output) { +int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam* output) { int8_t callType = TSDB_UDF_CALL_SCALA_PROC; SSDataBlock inputBlock = {0}; convertScalarParamToDataBlock(input, numOfCols, &inputBlock); @@ -1417,7 +1429,50 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu return err; } -int32_t teardownUdf(UdfcFuncHandle handle) { +int compareUdfcFuncSub(const void* elem1, const void* elem2) { + SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1; + SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2; + return strcmp(stub1->udfName, stub2->udfName); +} + +int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) { + int32_t code = 0; + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + SUdfcFuncStub key = {0}; + strcpy(key.udfName, udfName); + SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); + if (foundStub != NULL) { + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); + *pHandle = foundStub->handle; + return 0; + } + *pHandle = NULL; + code = doSetupUdf(udfName, pHandle); + if (code == TSDB_CODE_SUCCESS) { + SUdfcFuncStub stub = {0}; + strcpy(stub.udfName, udfName); + stub.handle = *pHandle; + taosArrayPush(gUdfdProxy.udfStubs, &stub); + taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub); + } else { + *pHandle = NULL; + } + + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); + return code; +} + +int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) { + UdfcFuncHandle handle = NULL; + int32_t code = setupUdf(udfName, &handle); + if (code != 0) { + return code; + } + code = doCallUdfScalarFunc(handle, input, numOfCols, output); + return code; +} + +int32_t doTeardownUdf(UdfcFuncHandle handle) { fnInfo("tear down udf. udf func handle: %p", handle); SClientUdfUvSession *session = (SClientUdfUvSession *) handle; @@ -1471,8 +1526,8 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } UdfcFuncHandle handle; int32_t udfCode = 0; - if ((udfCode = setupUdf((char*)pCtx->udfName, &handle)) != 0) { - fnError("udfAggInit error. step setupUdf. udf code: %d", udfCode); + if ((udfCode = setupUdf((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode); return false; } SClientUdfUvSession *session = (SClientUdfUvSession *)handle; @@ -1485,8 +1540,8 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult udfRes->session = (SClientUdfUvSession *)handle; SUdfInterBuf buf = {0}; - if ((udfCode = callUdfAggInit(handle, &buf)) != 0) { - fnError("udfAggInit error. step callUdfAggInit. udf code: %d", udfCode); + if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { + fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); return false; } udfRes->interResNum = buf.numOfResult; @@ -1533,7 +1588,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { .numOfResult = udfRes->interResNum}; SUdfInterBuf newState = {0}; - int32_t udfCode = callUdfAggProcess(session, inputBlock, &state, &newState); + int32_t udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); if (udfCode != 0) { fnError("udfAggProcess error. code: %d", udfCode); newState.numOfResult = 0; @@ -1567,9 +1622,9 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { .bufLen = session->bufSize, .numOfResult = udfRes->interResNum}; int32_t udfCallCode= 0; - udfCallCode= callUdfAggFinalize(session, &state, &resultBuf); - if (udfCallCode!= 0) { - fnError("udfAggFinalize error. callUdfAggFinalize step. udf code:%d", udfCallCode); + udfCallCode= doCallUdfAggFinalize(session, &state, &resultBuf); + if (udfCallCode != 0) { + fnError("udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d", udfCallCode); GET_RES_INFO(pCtx)->numOfRes = 0; } else { memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen); @@ -1577,11 +1632,11 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; } - int32_t code = teardownUdf(session); - if (code != 0) { - fnError("udfAggFinalize error. teardownUdf step. udf code: %d", code); - } - - return functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); +// int32_t code = doTeardownUdf(session); +// if (code != 0) { +// fnError("udfAggFinalize error. doTeardownUdf step. udf code: %d", code); +// } + int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); + return udfCallCode == 0 ? numOfResults : udfCallCode; } \ No newline at end of file diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index a8d6fbd715..d7c539e5c2 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -47,7 +47,7 @@ int main(int argc, char *argv[]) { UdfcFuncHandle handle; - setupUdf("udf1", &handle); + doSetupUdf("udf1", &handle); SSDataBlock block = {0}; SSDataBlock *pBlock = █ @@ -73,12 +73,12 @@ int main(int argc, char *argv[]) { input.numOfRows = pBlock->info.rows; input.columnData = taosArrayGet(pBlock->pDataBlock, 0); SScalarParam output = {0}; - callUdfScalarFunc(handle, &input, 1, &output); + doCallUdfScalarFunc(handle, &input, 1, &output); SColumnInfoData *col = output.columnData; for (int32_t i = 0; i < output.numOfRows; ++i) { fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t))); } - teardownUdf(handle); + doTeardownUdf(handle); udfcClose(); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 6c8326b09f..7e3dbaf7d0 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -362,19 +362,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, ¶mNum, &rowNum)); if (fmIsUserDefinedFunc(node->funcId)) { - UdfcFuncHandle udfHandle = NULL; - - code = setupUdf(node->functionName, &udfHandle); - if (code != 0) { - sclError("fmExecFunction error. setupUdf. function name: %s, code:%d", node->functionName, code); - goto _return; - } - code = callUdfScalarFunc(udfHandle, params, paramNum, output); - if (code != 0) { - sclError("fmExecFunction error. callUdfScalarFunc. function name: %s, udf code:%d", node->functionName, code); - goto _return; - } - code = teardownUdf(udfHandle); + code = callUdfScalarFunc(node->functionName, params, paramNum, output); if (code != 0) { sclError("fmExecFunction error. callUdfScalarFunc. function name: %s, udf code:%d", node->functionName, code); goto _return; From 5a72ca804b7432f3b4783c4fc1cb6f1e740747b2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 14 May 2022 23:03:02 +0800 Subject: [PATCH 47/71] fix(query): set the correct row index during generating join results. --- source/libs/executor/src/executorimpl.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2cbfb9b344..d08345a02f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5500,18 +5500,21 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId; int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId; + int32_t rowIndex = -1; SColumnInfoData* pSrc = NULL; if (pJoinInfo->pLeft->info.blockId == blockId) { pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId); + rowIndex = pJoinInfo->leftPos; } else { pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId); + rowIndex = pJoinInfo->rightPos; } - if (colDataIsNull_s(pSrc, pJoinInfo->leftPos)) { + if (colDataIsNull_s(pSrc, rowIndex)) { colDataAppendNULL(pDst, nrows); } else { - char* p = colDataGetData(pSrc, pJoinInfo->leftPos); + char* p = colDataGetData(pSrc, rowIndex); colDataAppend(pDst, nrows, p, false); } } From 692ae29c6cdaada1191fcd7d3dd66482ccc3aa80 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 14 May 2022 23:39:10 +0800 Subject: [PATCH 48/71] fix: clean the udf handles cache when pipe error --- source/libs/function/src/tudf.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index f2a0b4ec2c..3b83531e9f 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -825,6 +825,11 @@ void onUdfcPipeClose(uv_handle_t *handle) { taosMemoryFree(conn->readBuf.buf); taosMemoryFree(conn); taosMemoryFree((uv_pipe_t *) handle); + + //clear the udf handles cache + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + taosArrayClear(gUdfdProxy.udfStubs); + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); } int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *uvTask) { @@ -1140,7 +1145,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { return code; } -void udfClientAsyncCb(uv_async_t *async) { +void udfcAsyncTaskCb(uv_async_t *async) { SUdfcProxy *udfc = async->data; QUEUE wq; @@ -1204,7 +1209,7 @@ void constructUdfService(void *argsThread) { SUdfcProxy *udfc = (SUdfcProxy *)argsThread; uv_loop_init(&udfc->uvLoop); - uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfClientAsyncCb); + uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfcAsyncTaskCb); udfc->loopTaskAync.data = udfc; uv_async_init(&udfc->uvLoop, &udfc->loopStopAsync, udfStopAsyncCb); udfc->loopStopAsync.data = udfc; @@ -1472,6 +1477,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, return code; } +//TODO: when to teardown udf. teardown udf is not called int32_t doTeardownUdf(UdfcFuncHandle handle) { fnInfo("tear down udf. udf func handle: %p", handle); From e155e5637138b9d3784153357447a07f9315b5de Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sat, 14 May 2022 23:53:06 +0800 Subject: [PATCH 49/71] fix: pass udf test test case modification --- tests/system-test/0-others/udfTest.py | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index c2ae837160..3850e1eb0f 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -235,11 +235,11 @@ class TDTestCase: tdSql.checkData(0,3,2.652476145) # # bug for crash when query sub table - # tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") - # tdSql.checkData(0,0,378.215547010) - # tdSql.checkData(0,1,353.808067460) - # tdSql.checkData(0,2,2114.237451187) - # tdSql.checkData(0,3,2.125468151) + tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") + tdSql.checkData(0,0,378.215547010) + tdSql.checkData(0,1,353.808067460) + tdSql.checkData(0,2,2114.237451187) + tdSql.checkData(0,3,2.125468151) tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 ") tdSql.checkData(0,0,490.358032462) @@ -329,14 +329,14 @@ class TDTestCase: # # bug need fix - # tdSql.query("select udf1(num1) , csum(num1) from tb;") - # tdSql.checkRows(12) - # tdSql.query("select ceil(num1) , csum(num1) from tb;") - # tdSql.checkRows(12) - # tdSql.query("select udf1(c1) , csum(c1) from stb1;") - # tdSql.checkRows(25) - # tdSql.query("select floor(c1) , csum(c1) from stb1;") - # tdSql.checkRows(25) + tdSql.query("select udf1(num1) , csum(num1) from tb;") + tdSql.checkRows(9) + tdSql.query("select ceil(num1) , csum(num1) from tb;") + tdSql.checkRows(9) + tdSql.query("select udf1(c1) , csum(c1) from stb1;") + tdSql.checkRows(22) + tdSql.query("select floor(c1) , csum(c1) from stb1;") + tdSql.checkRows(22) # stable with compute functions tdSql.query("select udf1(c1) , abs(c1) from stb1;") @@ -361,8 +361,8 @@ class TDTestCase: # bug fix for crash # order by udf function result - # for _ in range(50): - # tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)") + for _ in range(50): + tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)") # udf functions with filter From 18fcb334653bde2992e84c03915f0f1b954e5631 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 15 May 2022 00:04:51 +0800 Subject: [PATCH 50/71] refactor: do some internal refactor. --- source/libs/executor/src/executorimpl.c | 263 ------------------ source/libs/executor/src/joinoperator.c | 184 ++++++++++++ source/libs/executor/src/timewindowoperator.c | 4 - 3 files changed, 184 insertions(+), 267 deletions(-) create mode 100644 source/libs/executor/src/joinoperator.c diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d08345a02f..0ab172f03a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "filter.h" #include "function.h" #include "functionMgt.h" @@ -1601,9 +1600,6 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { } } -static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); -static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant* tag, int16_t type, int16_t bytes); - static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; uint32_t status = BLK_DATA_NOT_LOAD; @@ -1771,100 +1767,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc return TSDB_CODE_SUCCESS; } -/* - * set tag value in SqlFunctionCtx - * e.g.,tag information into input buffer - */ -static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant* tag, int16_t type, int16_t bytes) { - taosVariantDestroy(tag); - - char* val = NULL; - // if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { - // val = tsdbGetTableName(pTable); - // assert(val != NULL); - // } else { - // val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); - // } - - if (val == NULL || isNull(val, type)) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t maxLen = bytes - VARSTR_HEADER_SIZE; - int32_t len = (varDataLen(val) > maxLen) ? maxLen : varDataLen(val); - taosVariantCreateFromBinary(tag, varDataVal(val), len, type); - // taosVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); - } else { - taosVariantCreateFromBinary(tag, val, bytes, type); - } -} - -static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { - assert(pTagColList != NULL && numOfTags > 0); - - for (int32_t i = 0; i < numOfTags; ++i) { - if (pTagColList[i].colId == colId) { - return &pTagColList[i]; - } - } - - return NULL; -} - -void setTagValue(SOperatorInfo* pOperatorInfo, void* pTable, SqlFunctionCtx* pCtx, int32_t numOfOutput) { - SExprInfo* pExpr = pOperatorInfo->pExpr; - SExprInfo* pExprInfo = &pExpr[0]; - int32_t functionId = getExprFunctionId(pExprInfo); -#if 0 - if (pQueryAttr->numOfOutput == 1 && functionId == FUNCTION_TS_COMP && pQueryAttr->stableQuery) { - assert(pExprInfo->base.numOfParams == 1); - - // int16_t tagColId = (int16_t)pExprInfo->base.param[0].i; - int16_t tagColId = -1; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); - - doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes); - - } else { - // set tag value, by which the results are aggregated. - int32_t offset = 0; - memset(pRuntimeEnv->tagVal, 0, pQueryAttr->tagLen); - - for (int32_t idx = 0; idx < numOfOutput; ++idx) { - SExprInfo* pLocalExprInfo = &pExpr[idx]; - - // ts_comp column required the tag value for join filter - if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.pParam[0].pCol->flag)) { - continue; - } - - // todo use tag column index to optimize performance - doSetTagValueInParam(pTable, pLocalExprInfo->base.pParam[0].pCol->colId, &pCtx[idx].tag, - pLocalExprInfo->base.resSchema.type, pLocalExprInfo->base.resSchema.bytes); - - if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resSchema.type) || - pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_BOOL || - pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_TIMESTAMP) { - memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i, pLocalExprInfo->base.resSchema.bytes); - } else { - if (pCtx[idx].tag.pz != NULL) { - memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen); - } - } - - offset += pLocalExprInfo->base.resSchema.bytes; - } - } - - // set the tsBuf start position before check each data block - if (pRuntimeEnv->pTsBuf != NULL) { - setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable); - } -#endif -} - void copyToSDataBlock(SSDataBlock* pBlock, int32_t* offset, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pResBuf) { pBlock->info.rows = 0; @@ -4038,12 +3940,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { if (pProjectInfo->existDataBlock) { // TODO refactor SSDataBlock* pBlock = pProjectInfo->existDataBlock; pProjectInfo->existDataBlock = NULL; - *newgroup = true; - - // todo dynamic set tags - // if (pTableQueryInfo != NULL) { - // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs); - // } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); @@ -4084,13 +3980,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } } - // todo set tags - - // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - // if (pTableQueryInfo != NULL) { - // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs); - // } - // the pDataBlock are always the same one, no need to call this again int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag); @@ -4430,10 +4319,6 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { doDestroyBasicInfo(pInfo, numOfOutput); } -void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { - SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; -} - void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); @@ -4778,7 +4663,6 @@ static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* createSortInfo(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); -static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { @@ -5447,150 +5331,3 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } - -static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { - SJoinOperatorInfo* pJoinInfo = pOperator->info; - - SSDataBlock* pRes = pJoinInfo->pRes; - blockDataCleanup(pRes); - blockDataEnsureCapacity(pRes, 4096); - - int32_t nrows = 0; - - while (1) { - if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { - SOperatorInfo* ds1 = pOperator->pDownstream[0]; - publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1); - publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC); - - pJoinInfo->leftPos = 0; - if (pJoinInfo->pLeft == NULL) { - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - break; - } - } - - if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { - SOperatorInfo* ds2 = pOperator->pDownstream[1]; - publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2); - publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC); - - pJoinInfo->rightPos = 0; - if (pJoinInfo->pRight == NULL) { - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - break; - } - } - - SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId); - char* pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos); - - SColumnInfoData* pRightCol = taosArrayGet(pJoinInfo->pRight->pDataBlock, pJoinInfo->rightCol.slotId); - char* pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos); - - // only the timestamp match support for ordinary table - ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) { - for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { - SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i); - - SExprInfo* pExprInfo = &pOperator->pExpr[i]; - - int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId; - int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId; - int32_t rowIndex = -1; - - SColumnInfoData* pSrc = NULL; - if (pJoinInfo->pLeft->info.blockId == blockId) { - pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId); - rowIndex = pJoinInfo->leftPos; - } else { - pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId); - rowIndex = pJoinInfo->rightPos; - } - - if (colDataIsNull_s(pSrc, rowIndex)) { - colDataAppendNULL(pDst, nrows); - } else { - char* p = colDataGetData(pSrc, rowIndex); - colDataAppend(pDst, nrows, p, false); - } - } - - pJoinInfo->leftPos += 1; - pJoinInfo->rightPos += 1; - - nrows += 1; - } else if (*(int64_t*)pLeftVal < *(int64_t*)pRightVal) { - pJoinInfo->leftPos += 1; - - if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { - continue; - } - } else if (*(int64_t*)pLeftVal > *(int64_t*)pRightVal) { - pJoinInfo->rightPos += 1; - if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { - continue; - } - } - - // the pDataBlock are always the same one, no need to call this again - pRes->info.rows = nrows; - if (pRes->info.rows >= pOperator->resultInfo.threshold) { - break; - } - } - - return (pRes->info.rows > 0) ? pRes : NULL; -} - -SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, - int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, - SExecTaskInfo* pTaskInfo) { - SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pOperator == NULL || pInfo == NULL) { - goto _error; - } - - initResultSizeInfo(pOperator, 4096); - - pInfo->pRes = pResBlock; - pOperator->name = "MergeJoinOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfExprs = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - SOperatorNode* pNode = (SOperatorNode*)pOnCondition; - setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft); - setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight); - - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL, NULL, NULL); - int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; -} - -void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { - pColumn->slotId = pColumnNode->slotId; - pColumn->type = pColumnNode->node.resType.type; - pColumn->bytes = pColumnNode->node.resType.bytes; - pColumn->precision = pColumnNode->node.resType.precision; - pColumn->scale = pColumnNode->node.resType.scale; -} diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c new file mode 100644 index 0000000000..82bb435a2c --- /dev/null +++ b/source/libs/executor/src/joinoperator.c @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "function.h" +#include "os.h" +#include "querynodes.h" +#include "tdatablock.h" +#include "tmsg.h" +#include "executorimpl.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" + +static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); +static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); +static void destroyMergeJoinOperator(void* param, int32_t numOfOutput); + +SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, + int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, + SExecTaskInfo* pTaskInfo) { + SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pOperator == NULL || pInfo == NULL) { + goto _error; + } + + initResultSizeInfo(pOperator, 4096); + + pInfo->pRes = pResBlock; + pOperator->name = "MergeJoinOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + if (nodeType(pOnCondition) == QUERY_NODE_OPERATOR) { + SOperatorNode* pNode = (SOperatorNode*)pOnCondition; + setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft); + setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight); + } else if (nodeType(pOnCondition) == QUERY_NODE_LOGIC_CONDITION) { + ASSERT(0); + } + + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL, NULL, NULL); + int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} + +void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { + pColumn->slotId = pColumnNode->slotId; + pColumn->type = pColumnNode->node.resType.type; + pColumn->bytes = pColumnNode->node.resType.bytes; + pColumn->precision = pColumnNode->node.resType.precision; + pColumn->scale = pColumnNode->node.resType.scale; +} + +void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { + SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; +} + +SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { + SJoinOperatorInfo* pJoinInfo = pOperator->info; + + SSDataBlock* pRes = pJoinInfo->pRes; + blockDataCleanup(pRes); + blockDataEnsureCapacity(pRes, 4096); + + int32_t nrows = 0; + + while (1) { + // todo extract method + if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { + SOperatorInfo* ds1 = pOperator->pDownstream[0]; + publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC); + pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1); + publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC); + + pJoinInfo->leftPos = 0; + if (pJoinInfo->pLeft == NULL) { + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); + break; + } + } + + if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { + SOperatorInfo* ds2 = pOperator->pDownstream[1]; + publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC); + pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2); + publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC); + + pJoinInfo->rightPos = 0; + if (pJoinInfo->pRight == NULL) { + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); + break; + } + } + + SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId); + char* pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos); + + SColumnInfoData* pRightCol = taosArrayGet(pJoinInfo->pRight->pDataBlock, pJoinInfo->rightCol.slotId); + char* pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos); + + // only the timestamp match support for ordinary table + ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); + if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) { + for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i); + + SExprInfo* pExprInfo = &pOperator->pExpr[i]; + + int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId; + int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId; + int32_t rowIndex = -1; + + SColumnInfoData* pSrc = NULL; + if (pJoinInfo->pLeft->info.blockId == blockId) { + pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId); + rowIndex = pJoinInfo->leftPos; + } else { + pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId); + rowIndex = pJoinInfo->rightPos; + } + + if (colDataIsNull_s(pSrc, rowIndex)) { + colDataAppendNULL(pDst, nrows); + } else { + char* p = colDataGetData(pSrc, rowIndex); + colDataAppend(pDst, nrows, p, false); + } + } + + pJoinInfo->leftPos += 1; + pJoinInfo->rightPos += 1; + + nrows += 1; + } else if (*(int64_t*)pLeftVal < *(int64_t*)pRightVal) { + pJoinInfo->leftPos += 1; + + if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { + continue; + } + } else if (*(int64_t*)pLeftVal > *(int64_t*)pRightVal) { + pJoinInfo->rightPos += 1; + if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { + continue; + } + } + + // the pDataBlock are always the same one, no need to call this again + pRes->info.rows = nrows; + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } + } + + return (pRes->info.rows > 0) ? pRes : NULL; +} diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 10dc482462..79e675e2df 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -773,7 +773,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { break; } - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true); STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; @@ -1062,8 +1061,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the // caller. Note that all the time window are not close till now. - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true); if (pInfo->invertible) { @@ -1377,7 +1374,6 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { break; } - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfExprs); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true); // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); From a6015d2f39c6264c43a8529bf4a607eb539715ba Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sun, 15 May 2022 08:05:26 +0800 Subject: [PATCH 51/71] refactor(stream) --- include/common/tmsg.h | 8 + include/libs/stream/tstream.h | 99 +++++- include/os/osAtomic.h | 90 +++--- source/dnode/mnode/impl/src/mndDef.c | 282 +++++++++--------- source/dnode/mnode/impl/test/trans/trans2.cpp | 2 +- source/dnode/vnode/src/tq/tq.c | 34 +++ source/libs/stream/src/tstream.c | 261 +++++++++++++++- tests/script/tsim/tstream/basic1.sim | 2 +- 8 files changed, 578 insertions(+), 200 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ae21986c56..72ec017980 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2574,6 +2574,14 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); } +typedef struct { + void* data; +} SStreamDispatchReq; + +typedef struct { + int8_t status; +} SStreamDispatchRsp; + #define TD_AUTO_CREATE_TABLE 0x1 typedef struct { int64_t suid; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 56e6a39ce8..796fee4f89 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "tdatablock.h" #include "tmsg.h" #include "tmsgcb.h" @@ -29,8 +30,22 @@ extern "C" { typedef struct SStreamTask SStreamTask; enum { - STREAM_TASK_STATUS__RUNNING = 1, - STREAM_TASK_STATUS__STOP, + TASK_STATUS__IDLE = 1, + TASK_STATUS__EXECUTING, + TASK_STATUS__CLOSING, +}; + +enum { + TASK_INPUT_STATUS__NORMAL = 1, + TASK_INPUT_STATUS__BLOCKED, + TASK_INPUT_STATUS__RECOVER, + TASK_INPUT_STATUS__STOP, +}; + +enum { + TASK_OUTPUT_STATUS__NORMAL = 1, + TASK_OUTPUT_STATUS__WAIT, + TASK_OUTPUT_STATUS__BLOCKED, }; enum { @@ -38,10 +53,64 @@ enum { STREAM_CREATED_BY__SMA, }; +enum { + STREAM_INPUT__DATA_SUBMIT = 1, + STREAM_INPUT__DATA_BLOCK, + STREAM_INPUT__CHECKPOINT, +}; + typedef struct { - int32_t nodeId; // 0 for snode - SEpSet epSet; -} SStreamTaskEp; + int8_t type; + + int32_t sourceVg; + int64_t sourceVer; + + int32_t* dataRef; + SSubmitReq* data; +} SStreamDataSubmit; + +typedef struct { + int8_t type; + + int32_t sourceVg; + int64_t sourceVer; + + SArray* blocks; // SArray +} SStreamDataBlock; + +typedef struct { + int8_t type; +} SStreamCheckpoint; + +static FORCE_INLINE SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) { + SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosMemoryCalloc(1, sizeof(SStreamDataSubmit)); + if (pDataSubmit == NULL) return NULL; + pDataSubmit->data = pReq; + pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); + if (pDataSubmit->data == NULL) goto FAIL; + *pDataSubmit->dataRef = 1; + return pDataSubmit; +FAIL: + taosMemoryFree(pDataSubmit); + return NULL; +} + +static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) { + // + atomic_add_fetch_32(pDataSubmit->dataRef, 1); +} + +static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { + int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); + ASSERT(ref >= 0); + if (ref == 0) { + taosMemoryFree(pDataSubmit->data); + taosMemoryFree(pDataSubmit->dataRef); + } +} + +int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput); +void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput); typedef struct { void* inputHandle; @@ -122,9 +191,15 @@ enum { TASK_SINK__FETCH, }; +enum { + TASK_INPUT_TYPE__SUMBIT_BLOCK = 1, + TASK_INPUT_TYPE__DATA_BLOCK, +}; + struct SStreamTask { int64_t streamId; int32_t taskId; + int8_t inputType; int8_t status; int8_t sourceType; @@ -155,9 +230,11 @@ struct SStreamTask { STaskDispatcherShuffle shuffleDispatcher; }; - // msg buffer - int32_t memUsed; + int8_t inputStatus; + int8_t outputStatus; + STaosQueue* inputQ; + STaosQueue* outputQ; // application storage void* ahandle; @@ -199,10 +276,16 @@ typedef struct { SArray* res; // SArray } SStreamSinkReq; -int32_t streamEnqueueData(SStreamTask* pTask, const void* input, int32_t inputType); +int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input); +int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input); +int32_t streamDequeueOutput(SStreamTask* pTask, void** output); int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId); +int32_t streamTaskExecNew(SStreamTask* pTask); + +int32_t streamTaskHandleInput(SStreamTask* pTask, void* data); + #ifdef __cplusplus } #endif diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index e2a122a0fe..8600992d68 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -23,92 +23,92 @@ extern "C" { // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC - #define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID - #define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID - #define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID - #define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID - #define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID - #define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID - #define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID - #define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID - #define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID - #define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID - #define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID - #define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID - #define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID - #define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID +#define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID +#define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID +#define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID +#define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID +#define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID +#define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID +#define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID +#define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID +#define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID +#define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID +#define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID +#define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID +#define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID +#define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID #endif -int8_t atomic_load_8(int8_t volatile *ptr); +int8_t atomic_load_8(int8_t volatile *ptr); int16_t atomic_load_16(int16_t volatile *ptr); int32_t atomic_load_32(int32_t volatile *ptr); int64_t atomic_load_64(int64_t volatile *ptr); -void* atomic_load_ptr(void *ptr); -void atomic_store_8(int8_t volatile *ptr, int8_t val); -void atomic_store_16(int16_t volatile *ptr, int16_t val); -void atomic_store_32(int32_t volatile *ptr, int32_t val); -void atomic_store_64(int64_t volatile *ptr, int64_t val); -void atomic_store_ptr(void *ptr, void *val); -int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); +void *atomic_load_ptr(void *ptr); +void atomic_store_8(int8_t volatile *ptr, int8_t val); +void atomic_store_16(int16_t volatile *ptr, int16_t val); +void atomic_store_32(int32_t volatile *ptr, int32_t val); +void atomic_store_64(int64_t volatile *ptr, int64_t val); +void atomic_store_ptr(void *ptr, void *val); +int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val); int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val); int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val); -void* atomic_exchange_ptr(void *ptr, void *val); -int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); +void *atomic_exchange_ptr(void *ptr, void *val); +int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval); int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval); int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval); -void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval); -int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val); +void *atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval); +int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_add_fetch_ptr(void *ptr, void *val); -int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); +void *atomic_add_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_add_ptr(void *ptr, void *val); -int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); +void *atomic_fetch_add_ptr(void *ptr, void *val); +int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_sub_fetch_ptr(void *ptr, void *val); -int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); +void *atomic_sub_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_sub_ptr(void *ptr, void *val); -int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); +void *atomic_fetch_sub_ptr(void *ptr, void *val); +int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_and_fetch_ptr(void *ptr, void *val); -int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val); +void *atomic_and_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_and_ptr(void *ptr, void *val); -int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val); +void *atomic_fetch_and_ptr(void *ptr, void *val); +int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_or_fetch_ptr(void *ptr, void *val); -int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val); +void *atomic_or_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_or_ptr(void *ptr, void *val); -int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val); +void *atomic_fetch_or_ptr(void *ptr, void *val); +int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_xor_fetch_ptr(void *ptr, void *val); -int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val); +void *atomic_xor_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_xor_ptr(void *ptr, void *val); +void *atomic_fetch_xor_ptr(void *ptr, void *val); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 8225eca659..35ba25acd5 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -17,6 +17,147 @@ #include "mndDef.h" #include "mndConsumer.h" +int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { + int32_t sz = 0; + /*int32_t outputNameSz = 0;*/ + if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1; + if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; + if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1; + if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; + /*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/ + if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + // TODO encode tasks + if (pObj->tasks) { + sz = taosArrayGetSize(pObj->tasks); + } + if (tEncodeI32(pEncoder, sz) < 0) return -1; + + for (int32_t i = 0; i < sz; i++) { + SArray *pArray = taosArrayGetP(pObj->tasks, i); + int32_t innerSz = taosArrayGetSize(pArray); + if (tEncodeI32(pEncoder, innerSz) < 0) return -1; + for (int32_t j = 0; j < innerSz; j++) { + SStreamTask *pTask = taosArrayGetP(pArray, j); + if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1; + } + } + + if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; + +#if 0 + if (pObj->ColAlias != NULL) { + outputNameSz = taosArrayGetSize(pObj->ColAlias); + } + if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1; + for (int32_t i = 0; i < outputNameSz; i++) { + char *name = taosArrayGetP(pObj->ColAlias, i); + if (tEncodeCStr(pEncoder, name) < 0) return -1; + } +#endif + return pEncoder->pos; +} + +int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { + if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1; + if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; + if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1; + if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; + /*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/ + if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; + pObj->tasks = NULL; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + if (sz != 0) { + pObj->tasks = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + int32_t innerSz; + if (tDecodeI32(pDecoder, &innerSz) < 0) return -1; + SArray *pArray = taosArrayInit(innerSz, sizeof(void *)); + for (int32_t j = 0; j < innerSz; j++) { + SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); + if (pTask == NULL) return -1; + if (tDecodeSStreamTask(pDecoder, pTask) < 0) return -1; + taosArrayPush(pArray, &pTask); + } + taosArrayPush(pObj->tasks, &pArray); + } + } + + if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; +#if 0 + int32_t outputNameSz; + if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1; + if (outputNameSz != 0) { + pObj->ColAlias = taosArrayInit(outputNameSz, sizeof(void *)); + if (pObj->ColAlias == NULL) { + return -1; + } + } + for (int32_t i = 0; i < outputNameSz; i++) { + char *name; + if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1; + taosArrayPush(pObj->ColAlias, &name); + } +#endif + return 0; +} + +SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { + SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); + if (pVgEpNew == NULL) return NULL; + pVgEpNew->vgId = pVgEp->vgId; + pVgEpNew->qmsg = strdup(pVgEp->qmsg); + pVgEpNew->epSet = pVgEp->epSet; + return pVgEpNew; +} + +void tDeleteSMqVgEp(SMqVgEp *pVgEp) { + if (pVgEp->qmsg) taosMemoryFree(pVgEp->qmsg); +} + +int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeString(buf, pVgEp->qmsg); + tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); + return tlen; +} + +void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) { + buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeString(buf, &pVgEp->qmsg); + buf = taosDecodeSEpSet(buf, &pVgEp->epSet); + return (void *)buf; +} + SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) { SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { @@ -187,34 +328,6 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { return (void *)buf; } -SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { - SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); - if (pVgEpNew == NULL) return NULL; - pVgEpNew->vgId = pVgEp->vgId; - pVgEpNew->qmsg = strdup(pVgEp->qmsg); - pVgEpNew->epSet = pVgEp->epSet; - return pVgEpNew; -} - -void tDeleteSMqVgEp(SMqVgEp *pVgEp) { - if (pVgEp->qmsg) taosMemoryFree(pVgEp->qmsg); -} - -int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pVgEp->vgId); - tlen += taosEncodeString(buf, pVgEp->qmsg); - tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); - return tlen; -} - -void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) { - buf = taosDecodeFixedI32(buf, &pVgEp->vgId); - buf = taosDecodeString(buf, &pVgEp->qmsg); - buf = taosDecodeSEpSet(buf, &pVgEp->epSet); - return (void *)buf; -} - SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); if (pConsumerEpNew == NULL) return NULL; @@ -413,119 +526,6 @@ void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) { return (void *)buf; } -int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { - int32_t sz = 0; - /*int32_t outputNameSz = 0;*/ - if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1; - if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; - if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1; - if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; - /*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/ - if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; - // TODO encode tasks - if (pObj->tasks) { - sz = taosArrayGetSize(pObj->tasks); - } - if (tEncodeI32(pEncoder, sz) < 0) return -1; - - for (int32_t i = 0; i < sz; i++) { - SArray *pArray = taosArrayGetP(pObj->tasks, i); - int32_t innerSz = taosArrayGetSize(pArray); - if (tEncodeI32(pEncoder, innerSz) < 0) return -1; - for (int32_t j = 0; j < innerSz; j++) { - SStreamTask *pTask = taosArrayGetP(pArray, j); - if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1; - } - } - - if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; - -#if 0 - if (pObj->ColAlias != NULL) { - outputNameSz = taosArrayGetSize(pObj->ColAlias); - } - if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1; - for (int32_t i = 0; i < outputNameSz; i++) { - char *name = taosArrayGetP(pObj->ColAlias, i); - if (tEncodeCStr(pEncoder, name) < 0) return -1; - } -#endif - return pEncoder->pos; -} - -int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { - if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1; - if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; - if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1; - if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; - if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; - /*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/ - if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; - pObj->tasks = NULL; - int32_t sz; - if (tDecodeI32(pDecoder, &sz) < 0) return -1; - if (sz != 0) { - pObj->tasks = taosArrayInit(sz, sizeof(void *)); - for (int32_t i = 0; i < sz; i++) { - int32_t innerSz; - if (tDecodeI32(pDecoder, &innerSz) < 0) return -1; - SArray *pArray = taosArrayInit(innerSz, sizeof(void *)); - for (int32_t j = 0; j < innerSz; j++) { - SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); - if (pTask == NULL) return -1; - if (tDecodeSStreamTask(pDecoder, pTask) < 0) return -1; - taosArrayPush(pArray, &pTask); - } - taosArrayPush(pObj->tasks, &pArray); - } - } - - if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; -#if 0 - int32_t outputNameSz; - if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1; - if (outputNameSz != 0) { - pObj->ColAlias = taosArrayInit(outputNameSz, sizeof(void *)); - if (pObj->ColAlias == NULL) { - return -1; - } - } - for (int32_t i = 0; i < outputNameSz; i++) { - char *name; - if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1; - taosArrayPush(pObj->ColAlias, &name); - } -#endif - return 0; -} - int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { int32_t tlen = 0; tlen += taosEncodeString(buf, pOffset->key); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 974c86b423..061ede345d 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -510,4 +510,4 @@ TEST_F(MndTestTrans2, 04_Conflict) { ASSERT_EQ(pUser, nullptr); mndReleaseUser(pMnode, pUser); } -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9526451907..ac8a271a4f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1031,3 +1031,37 @@ int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId) } return 0; } + +int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) { + SStreamTaskExecReq req = {0}; + tDecodeSStreamTaskExecReq(msg, &req); + int32_t taskId = req.taskId; + + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + ASSERT(pTask); + ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK); + + // enqueue + int32_t inputStatus = streamEnqueueDataBlk(pTask, (SStreamDataBlock*)req.data); + if (inputStatus == TASK_INPUT_STATUS__BLOCKED) { + // TODO rsp blocked + return 0; + } + + // try exec + int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); + if (execStatus == TASK_STATUS__IDLE) { + if (streamTaskExecNew(pTask) < 0) { + atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); + + goto FAIL; + } + } else if (execStatus == TASK_STATUS__EXECUTING) { + return 0; + } + + // TODO rsp success + return 0; +FAIL: + return -1; +} diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index 08093c8b18..33147a0c0a 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -16,6 +16,25 @@ #include "tstream.h" #include "executor.h" +int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) { + int32_t tlen = 0; + tlen += taosEncodeFixedI8(buf, pOutput->type); + tlen += taosEncodeFixedI32(buf, pOutput->sourceVg); + tlen += taosEncodeFixedI64(buf, pOutput->sourceVer); + ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK); + tlen += tEncodeDataBlocks(buf, pOutput->blocks); + return tlen; +} + +void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) { + buf = taosDecodeFixedI8(buf, &pInput->type); + buf = taosDecodeFixedI32(buf, &pInput->sourceVg); + buf = taosDecodeFixedI64(buf, &pInput->sourceVer); + ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); + buf = tDecodeDataBlocks(buf, &pInput->blocks); + return (void*)buf; +} + static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { SStreamTaskExecReq req = { .streamId = pTask->streamId, @@ -97,6 +116,226 @@ static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashOb return 0; } +int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input) { + ASSERT(pTask->inputType == TASK_INPUT_TYPE__SUMBIT_BLOCK); + int8_t inputStatus = atomic_load_8(&pTask->inputStatus); + if (inputStatus == TASK_INPUT_STATUS__NORMAL) { + streamDataSubmitRefInc(input); + taosWriteQitem(pTask->inputQ, input); + } + return inputStatus; +} + +int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) { + ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK); + taosWriteQitem(pTask->inputQ, input); + int8_t inputStatus = atomic_load_8(&pTask->inputStatus); + return inputStatus; +} + +int32_t streamTaskProcessTriggerReq(SStreamTask* pTask, SMsgCb* pMsgCb, char* msg, int32_t msgLen) { + // + return 0; +} + +int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDataBlock* pBlock, SRpcMsg* pRsp) { + // 1. handle input + // 1.1 enqueue + taosWriteQitem(pTask->inputQ, pBlock); + // 1.2 calc back pressure + // 1.3 rsp by input status + int8_t inputStatus = atomic_load_8(&pTask->inputStatus); + SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp)); + pCont->status = inputStatus; + pRsp->pCont = pCont; + pRsp->contLen = sizeof(SStreamDispatchRsp); + tmsgSendRsp(pRsp); + // 2. try exec + // 2.1. idle: exec + // 2.2. executing: return + // 2.3. closing: keep trying + while (1) { + int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); + void* exec = pTask->exec.runners[0].executor; + if (execStatus == TASK_STATUS__IDLE) { + SArray* pRes = taosArrayInit(0, sizeof(void*)); + const SArray* blocks = pBlock->blocks; + qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK); + while (1) { + SSDataBlock* output; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) break; + taosArrayPush(pRes, &output); + } + // TODO: wrap destroy block + taosArrayDestroyP(pBlock->blocks, (FDelete)blockDataDestroy); + + if (taosArrayGetSize(pRes) != 0) { + SArray** resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + *resQ = pRes; + taosWriteQitem(pTask->outputQ, resQ); + } + + } else if (execStatus == TASK_STATUS__CLOSING) { + continue; + } else if (execStatus == TASK_STATUS__EXECUTING) + break; + else { + ASSERT(0); + } + } + // 3. handle output + // 3.1 check and set status + // 3.2 dispatch / sink + STaosQall* qall = taosAllocateQall(); + taosReadAllQitems(pTask->outputQ, qall); + SArray** ppRes = NULL; + while (1) { + taosGetQitem(qall, (void**)&ppRes); + if (ppRes == NULL) break; + + SArray* pRes = *ppRes; + if (pTask->sinkType == TASK_SINK__TABLE) { + pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, pBlock->sourceVer, pRes); + } else if (pTask->sinkType == TASK_SINK__SMA) { + pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes); + } else { + } + + // dispatch + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + SRpcMsg dispatchMsg = {0}; + if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, NULL) < 0) { + ASSERT(0); + return -1; + } + + int32_t qType; + if (pTask->dispatchMsgType == TDMT_VND_TASK_PIPE_EXEC || pTask->dispatchMsgType == TDMT_SND_TASK_PIPE_EXEC) { + qType = FETCH_QUEUE; + } else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC || + pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) { + qType = MERGE_QUEUE; + } else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) { + qType = WRITE_QUEUE; + } else { + ASSERT(0); + } + tmsgPutToQueue(pMsgCb, qType, &dispatchMsg); + + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + SRpcMsg dispatchMsg = {0}; + SEpSet* pEpSet = NULL; + if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) { + ASSERT(0); + return -1; + } + + tmsgSendReq(pMsgCb, pEpSet, &dispatchMsg); + + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (pShuffleRes == NULL) { + return -1; + } + + int32_t sz = taosArrayGetSize(pRes); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pRes, i); + SArray* pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t)); + if (pArray == NULL) { + pArray = taosArrayInit(0, sizeof(SSDataBlock)); + if (pArray == NULL) { + return -1; + } + taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*)); + } + taosArrayPush(pArray, pDataBlock); + } + + if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) { + return -1; + } + + } else { + ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); + } + } + // + return 0; +} + +int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, char* msg, int32_t msgLen) { + // + return 0; +} + +int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, char* msg) { + // + return 0; +} + +int32_t streamTaskExecNew(SStreamTask* pTask) { + SArray* pRes = NULL; + if (pTask->execType == TASK_EXEC__PIPE || pTask->execType == TASK_EXEC__MERGE) { + // TODO remove multi runner + void* exec = pTask->exec.runners[0].executor; + + int8_t status = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); + if (status == TASK_STATUS__IDLE) { + pRes = taosArrayInit(0, sizeof(void*)); + if (pRes == NULL) { + return -1; + } + + void* input = NULL; + taosWriteQitem(pTask->inputQ, &input); + if (input == NULL) return 0; + + // TODO: fix type + if (pTask->sourceType == TASK_SOURCE__SCAN) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input; + qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK); + while (1) { + SSDataBlock* output; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) break; + taosArrayPush(pRes, &output); + } + streamDataSubmitRefDec(pSubmit); + } else { + SStreamDataBlock* pStreamBlock = (SStreamDataBlock*)input; + const SArray* blocks = pStreamBlock->blocks; + qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK); + while (1) { + SSDataBlock* output; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) break; + taosArrayPush(pRes, &output); + } + // TODO: wrap destroy block + taosArrayDestroyP(pStreamBlock->blocks, (FDelete)blockDataDestroy); + } + + if (taosArrayGetSize(pRes) != 0) { + SArray** resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + *resQ = pRes; + taosWriteQitem(pTask->outputQ, resQ); + } + } + } + return 0; +} + int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) { SArray* pRes = NULL; // source @@ -251,9 +490,17 @@ SStreamTask* tNewSStreamTask(int64_t streamId) { } pTask->taskId = tGenIdPI32(); pTask->streamId = streamId; - pTask->status = STREAM_TASK_STATUS__RUNNING; - /*pTask->qmsg = NULL;*/ + pTask->status = TASK_STATUS__IDLE; + + pTask->inputQ = taosOpenQueue(); + pTask->outputQ = taosOpenQueue(); + if (pTask->inputQ == NULL || pTask->outputQ == NULL) goto FAIL; return pTask; +FAIL: + if (pTask->inputQ) taosCloseQueue(pTask->inputQ); + if (pTask->outputQ) taosCloseQueue(pTask->outputQ); + if (pTask) taosMemoryFree(pTask); + return NULL; } int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { @@ -349,10 +596,16 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { } void tFreeSStreamTask(SStreamTask* pTask) { + taosCloseQueue(pTask->inputQ); + taosCloseQueue(pTask->outputQ); // TODO - /*taosMemoryFree(pTask->qmsg);*/ + if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); + for (int32_t i = 0; i < pTask->exec.numOfRunners; i++) { + qDestroyTask(pTask->exec.runners[i].executor); + } + taosMemoryFree(pTask->exec.runners); /*taosMemoryFree(pTask->executor);*/ - /*taosMemoryFree(pTask);*/ + taosMemoryFree(pTask); } #if 0 diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/tstream/basic1.sim index 3bb5943b3b..37f9cb94c9 100644 --- a/tests/script/tsim/tstream/basic1.sim +++ b/tests/script/tsim/tstream/basic1.sim @@ -136,7 +136,7 @@ if $data35 != 3 then endi sql insert into t1 values(1648791223001,12,14,13,11.1); -sleep 100 +sleep 500 sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; if $rows != 4 then From 89a998221794db044707b37df9de433fb717a66b Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sun, 15 May 2022 13:46:44 +0800 Subject: [PATCH 52/71] fix: print query result --- tests/system-test/0-others/udfTest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 3850e1eb0f..0a998aee2b 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -363,6 +363,7 @@ class TDTestCase: # order by udf function result for _ in range(50): tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)") + print(tdSql.queryResult) # udf functions with filter From 2df2e1a84dc352160c38fa52a7ac38b2652b6ca5 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sun, 15 May 2022 13:50:13 +0800 Subject: [PATCH 53/71] test: add udfTest.py to fulltest.sh --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index c80206abbc..2b813f83ca 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -7,6 +7,7 @@ python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/telemetry.py python3 ./test.py -f 0-others/taosdMonitor.py +python3 ./test.py -f 0-others/udfTest.py python3 ./test.py -f 0-others/user_control.py From 89ba94398165a34fa5c513fd7308b40ebbdfd35d Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 15 May 2022 14:15:46 +0800 Subject: [PATCH 54/71] fix: change udf comment --- source/libs/function/src/tudf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 3b83531e9f..35702eb71f 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -826,7 +826,7 @@ void onUdfcPipeClose(uv_handle_t *handle) { taosMemoryFree(conn); taosMemoryFree((uv_pipe_t *) handle); - //clear the udf handles cache + //clear the udf handles cache TODO move to other thread uv_mutex_lock(&gUdfdProxy.udfStubsMutex); taosArrayClear(gUdfdProxy.udfStubs); uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); From 069ce051f0e5fd3d49521fdc93c5fb05ab463264 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 15 May 2022 20:17:20 +0800 Subject: [PATCH 55/71] feat: move sma module from tsdb to vnode --- include/common/taosdef.h | 3 +- include/common/tmsg.h | 605 ++++----- include/util/taoserror.h | 3 +- include/util/tlog.h | 1 + source/common/src/tglobal.c | 2 + source/common/src/tmsg.c | 108 +- source/dnode/mnode/impl/src/mndSma.c | 72 +- source/dnode/mnode/impl/src/mndStb.c | 4 + source/dnode/vnode/CMakeLists.txt | 13 +- source/dnode/vnode/inc/vnode.h | 3 + source/dnode/vnode/src/inc/meta.h | 7 +- source/dnode/vnode/src/inc/sma.h | 225 ++++ source/dnode/vnode/src/inc/tsdb.h | 80 +- source/dnode/vnode/src/inc/vnodeInt.h | 80 +- source/dnode/vnode/src/meta/metaEntry.c | 2 + source/dnode/vnode/src/meta/metaIdx.c | 31 - source/dnode/vnode/src/meta/metaOpen.c | 24 +- source/dnode/vnode/src/meta/metaSma.c | 201 +++ .../vnode/src/{inc/tsdbSma.h => sma/sma.c} | 33 +- source/dnode/vnode/src/sma/smaEnv.c | 463 +++++++ source/dnode/vnode/src/sma/smaOpen.c | 137 ++ source/dnode/vnode/src/sma/smaRollup.c | 484 ++++++++ source/dnode/vnode/src/sma/smaTDBImpl.c | 128 ++ source/dnode/vnode/src/sma/smaTimeRange.c | 1103 +++++++++++++++++ source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 2 + source/dnode/vnode/src/tsdb/tsdbFS.c | 16 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 10 +- source/dnode/vnode/src/tsdb/tsdbOpen.c | 123 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 38 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 50 +- source/libs/parser/src/parTranslater.c | 18 +- source/util/src/terror.c | 1 + source/util/src/tlog.c | 2 + 35 files changed, 3367 insertions(+), 709 deletions(-) create mode 100644 source/dnode/vnode/src/inc/sma.h create mode 100644 source/dnode/vnode/src/meta/metaSma.c rename source/dnode/vnode/src/{inc/tsdbSma.h => sma/sma.c} (50%) create mode 100644 source/dnode/vnode/src/sma/smaEnv.c create mode 100644 source/dnode/vnode/src/sma/smaOpen.c create mode 100644 source/dnode/vnode/src/sma/smaRollup.c create mode 100644 source/dnode/vnode/src/sma/smaTDBImpl.c create mode 100644 source/dnode/vnode/src/sma/smaTimeRange.c diff --git a/include/common/taosdef.h b/include/common/taosdef.h index e1f8832edf..72d2c142d2 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -37,7 +37,8 @@ typedef enum { TSDB_STREAM_TABLE = 4, // table created from stream computing TSDB_TEMP_TABLE = 5, // temp table created by nest query TSDB_SYSTEM_TABLE = 6, - TSDB_TABLE_MAX = 7 + TSDB_TSMA_TABLE = 7, // time-range-wise sma + TSDB_TABLE_MAX = 8 } ETableType; typedef enum { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 72ec017980..46dbb6ba6e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2160,26 +2160,23 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; // MACRO: TIME_UNIT_XXX - int8_t slidingUnit; // MACRO: TIME_UNIT_XXX - int8_t timezoneInt; // sma data expired if timezone changes. - char indexName[TSDB_INDEX_NAME_LEN]; - int32_t exprLen; - int32_t tagsFilterLen; - int64_t indexUid; - tb_uid_t tableUid; // super/child/common table uid - int64_t interval; - int64_t offset; // use unit by precision of DB - int64_t sliding; - char* expr; // sma expression - char* tagsFilter; + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. + char indexName[TSDB_INDEX_NAME_LEN]; + int32_t exprLen; + int32_t tagsFilterLen; + int64_t indexUid; + tb_uid_t tableUid; // super/child/common table uid + int64_t interval; + int64_t offset; // use unit by precision of DB + int64_t sliding; + const char* expr; // sma expression + const char* tagsFilter; } STSma; // Time-range-wise SMA -typedef struct { - int64_t ver; // use a general definition - STSma tSma; -} SVCreateTSmaReq; +typedef STSma SVCreateTSmaReq; typedef struct { int8_t type; // 0 status report, 1 update data @@ -2188,7 +2185,6 @@ typedef struct { } STSmaMsg; typedef struct { - int64_t ver; // use a general definition int64_t indexUid; char indexName[TSDB_INDEX_NAME_LEN]; } SVDropTSmaReq; @@ -2197,28 +2193,21 @@ typedef struct { int tmp; // TODO: to avoid compile error } SVCreateTSmaRsp, SVDropTSmaRsp; +#if 0 int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq); void* tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq); int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq); void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq); +#endif -// RSma: Rollup SMA -typedef struct { - int64_t interval; - int32_t retention; // unit: day - uint16_t days; // unit: day - int8_t intervalUnit; -} SSmaParams; +int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq); +int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq); +int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq); +int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq); typedef struct { - STSma tsma; - float xFilesFactor; - SArray* smaParams; // SSmaParams -} SRSma; - -typedef struct { - uint32_t number; - STSma* tSma; + int32_t number; + STSma* tSma; } STSmaWrapper; static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { @@ -2245,334 +2234,264 @@ static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) { return NULL; } -static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { - int32_t tlen = 0; +int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq); +int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq); - tlen += taosEncodeFixedI8(buf, pSma->version); - tlen += taosEncodeFixedI8(buf, pSma->intervalUnit); - tlen += taosEncodeFixedI8(buf, pSma->slidingUnit); - tlen += taosEncodeFixedI8(buf, pSma->timezoneInt); - tlen += taosEncodeString(buf, pSma->indexName); - tlen += taosEncodeFixedI32(buf, pSma->exprLen); - tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen); - tlen += taosEncodeFixedI64(buf, pSma->indexUid); - tlen += taosEncodeFixedI64(buf, pSma->tableUid); - tlen += taosEncodeFixedI64(buf, pSma->interval); - tlen += taosEncodeFixedI64(buf, pSma->offset); - tlen += taosEncodeFixedI64(buf, pSma->sliding); +int32_t tEncodeTSma(SEncoder* pCoder, const STSma* pSma); +int32_t tDecodeTSma(SDecoder* pCoder, STSma* pSma); - if (pSma->exprLen > 0) { - tlen += taosEncodeString(buf, pSma->expr); +static int32_t tEncodeTSmaWrapper(SEncoder* pEncoder, const STSmaWrapper* pReq) { + if (tEncodeI32(pEncoder, pReq->number) < 0) return -1; + for (int32_t i = 0; i < pReq->number; ++i) { + tEncodeTSma(pEncoder, pReq->tSma + i); } - - if (pSma->tagsFilterLen > 0) { - tlen += taosEncodeString(buf, pSma->tagsFilter); - } - - return tlen; + return 0; } -static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) { - int32_t tlen = 0; - - tlen += taosEncodeFixedU32(buf, pSW->number); - for (uint32_t i = 0; i < pSW->number; ++i) { - tlen += tEncodeTSma(buf, pSW->tSma + i); +static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) { + if (tDecodeI32(pDecoder, &pReq->number) < 0) return -1; + for (int32_t i = 0; i < pReq->number; ++i) { + tDecodeTSma(pDecoder, pReq->tSma + i); } - return tlen; + return 0; } -static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { - buf = taosDecodeFixedI8(buf, &pSma->version); - buf = taosDecodeFixedI8(buf, &pSma->intervalUnit); - buf = taosDecodeFixedI8(buf, &pSma->slidingUnit); - buf = taosDecodeFixedI8(buf, &pSma->timezoneInt); - buf = taosDecodeStringTo(buf, pSma->indexName); - buf = taosDecodeFixedI32(buf, &pSma->exprLen); - buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen); - buf = taosDecodeFixedI64(buf, &pSma->indexUid); - buf = taosDecodeFixedI64(buf, &pSma->tableUid); - buf = taosDecodeFixedI64(buf, &pSma->interval); - buf = taosDecodeFixedI64(buf, &pSma->offset); - buf = taosDecodeFixedI64(buf, &pSma->sliding); + typedef struct { + int idx; + } SMCreateFullTextReq; - if (pSma->exprLen > 0) { - if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) { - tdDestroyTSma(pSma); + int32_t tSerializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq); + int32_t tDeserializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq); + void tFreeSMCreateFullTextReq(SMCreateFullTextReq * pReq); + + typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + int8_t igNotExists; + } SMDropFullTextReq; + + int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); + int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); + + typedef struct { + char indexFName[TSDB_INDEX_FNAME_LEN]; + } SUserIndexReq; + + int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); + int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); + + typedef struct { + char dbFName[TSDB_DB_FNAME_LEN]; + char tblFName[TSDB_TABLE_FNAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; + char indexType[TSDB_INDEX_TYPE_LEN]; + char indexExts[TSDB_INDEX_EXTS_LEN]; + } SUserIndexRsp; + + int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp* pRsp); + int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp); + + typedef struct { + int8_t mqMsgType; + int32_t code; + int32_t epoch; + int64_t consumerId; + } SMqRspHead; + + typedef struct { + SMsgHead head; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int8_t withTbName; + int32_t epoch; + uint64_t reqId; + int64_t consumerId; + int64_t waitTime; + int64_t currentOffset; + } SMqPollReq; + + typedef struct { + int32_t vgId; + int64_t offset; + SEpSet epSet; + } SMqSubVgEp; + + static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeFixedI64(buf, pVgEp->offset); + tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); + return tlen; + } + + static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { + buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeFixedI64(buf, &pVgEp->offset); + buf = taosDecodeSEpSet(buf, &pVgEp->epSet); + return buf; + } + + typedef struct { + char topic[TSDB_TOPIC_FNAME_LEN]; + int8_t isSchemaAdaptive; + SArray* vgs; // SArray + SSchemaWrapper schema; + } SMqSubTopicEp; + + static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pTopicEp->topic); + tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive); + int32_t sz = taosArrayGetSize(pTopicEp->vgs); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); + tlen += tEncodeSMqSubVgEp(buf, pVgEp); + } + tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); + return tlen; + } + + static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { + buf = taosDecodeStringTo(buf, pTopicEp->topic); + buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); + if (pTopicEp->vgs == NULL) { return NULL; } - } else { - pSma->expr = NULL; + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp vgEp; + buf = tDecodeSMqSubVgEp(buf, &vgEp); + taosArrayPush(pTopicEp->vgs, &vgEp); + } + buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); + return buf; } - if (pSma->tagsFilterLen > 0) { - if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) { - tdDestroyTSma(pSma); + static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp * pSubTopicEp) { + // taosMemoryFree(pSubTopicEp->schema.pSchema); + taosArrayDestroy(pSubTopicEp->vgs); + } + + typedef struct { + SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t blockNum; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + SArray* blockDataLen; // SArray + SArray* blockData; // SArray + SArray* blockTbName; // SArray + SArray* blockSchema; // SArray + SArray* blockTags; // SArray + SArray* blockTagSchema; // SArray + } SMqDataBlkRsp; + + static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); + tlen += taosEncodeFixedI32(buf, pRsp->blockNum); + if (pRsp->blockNum != 0) { + tlen += taosEncodeFixedI8(buf, pRsp->withTbName); + tlen += taosEncodeFixedI8(buf, pRsp->withSchema); + tlen += taosEncodeFixedI8(buf, pRsp->withTag); + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); + void* data = taosArrayGetP(pRsp->blockData, i); + tlen += taosEncodeFixedI32(buf, bLen); + tlen += taosEncodeBinary(buf, data, bLen); + if (pRsp->withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i); + tlen += taosEncodeSSchemaWrapper(buf, pSW); + } + if (pRsp->withTbName) { + char* tbName = (char*)taosArrayGetP(pRsp->blockTbName, i); + tlen += taosEncodeString(buf, tbName); + } + } + } + return tlen; + } + + static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) { + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); + buf = taosDecodeFixedI32(buf, &pRsp->blockNum); + pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); + if (pRsp->blockNum != 0) { + buf = taosDecodeFixedI8(buf, &pRsp->withTbName); + buf = taosDecodeFixedI8(buf, &pRsp->withSchema); + buf = taosDecodeFixedI8(buf, &pRsp->withTag); + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = 0; + void* data = NULL; + buf = taosDecodeFixedI32(buf, &bLen); + buf = taosDecodeBinary(buf, &data, bLen); + taosArrayPush(pRsp->blockDataLen, &bLen); + taosArrayPush(pRsp->blockData, &data); + if (pRsp->withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); + buf = taosDecodeSSchemaWrapper(buf, pSW); + taosArrayPush(pRsp->blockSchema, &pSW); + } + if (pRsp->withTbName) { + char* name = NULL; + buf = taosDecodeString(buf, &name); + taosArrayPush(pRsp->blockTbName, &name); + } + } + } + return (void*)buf; + } + + typedef struct { + SMqRspHead head; + char cgroup[TSDB_CGROUP_LEN]; + SArray* topics; // SArray + } SMqAskEpRsp; + + static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pRsp) { + int32_t tlen = 0; + // tlen += taosEncodeString(buf, pRsp->cgroup); + int32_t sz = taosArrayGetSize(pRsp->topics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubTopicEp* pVgEp = (SMqSubTopicEp*)taosArrayGet(pRsp->topics, i); + tlen += tEncodeSMqSubTopicEp(buf, pVgEp); + } + return tlen; + } + + static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) { + // buf = taosDecodeStringTo(buf, pRsp->cgroup); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); + if (pRsp->topics == NULL) { return NULL; } - } else { - pSma->tagsFilter = NULL; - } - - return buf; -} - -static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { - buf = taosDecodeFixedU32(buf, &pSW->number); - - pSW->tSma = (STSma*)taosMemoryCalloc(pSW->number, sizeof(STSma)); - if (pSW->tSma == NULL) { - return NULL; - } - - for (uint32_t i = 0; i < pSW->number; ++i) { - if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) { - for (uint32_t j = i; j >= 0; --i) { - tdDestroyTSma(pSW->tSma + j); - } - taosMemoryFree(pSW->tSma); - return NULL; + for (int32_t i = 0; i < sz; i++) { + SMqSubTopicEp topicEp; + buf = tDecodeSMqSubTopicEp(buf, &topicEp); + taosArrayPush(pRsp->topics, &topicEp); } + return buf; } - return buf; -} -typedef struct { - int idx; -} SMCreateFullTextReq; - -int32_t tSerializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq); -int32_t tDeserializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq); -void tFreeSMCreateFullTextReq(SMCreateFullTextReq* pReq); - -typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; - int8_t igNotExists; -} SMDropFullTextReq; - -int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); -int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); - -typedef struct { - char indexFName[TSDB_INDEX_FNAME_LEN]; -} SUserIndexReq; - -int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); -int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); - -typedef struct { - char dbFName[TSDB_DB_FNAME_LEN]; - char tblFName[TSDB_TABLE_FNAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; - char indexType[TSDB_INDEX_TYPE_LEN]; - char indexExts[TSDB_INDEX_EXTS_LEN]; -} SUserIndexRsp; - -int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp* pRsp); -int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp); - -typedef struct { - int8_t mqMsgType; - int32_t code; - int32_t epoch; - int64_t consumerId; -} SMqRspHead; - -typedef struct { - SMsgHead head; - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; - int8_t withTbName; - int32_t epoch; - uint64_t reqId; - int64_t consumerId; - int64_t waitTime; - int64_t currentOffset; -} SMqPollReq; - -typedef struct { - int32_t vgId; - int64_t offset; - SEpSet epSet; -} SMqSubVgEp; - -static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pVgEp->vgId); - tlen += taosEncodeFixedI64(buf, pVgEp->offset); - tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { - buf = taosDecodeFixedI32(buf, &pVgEp->vgId); - buf = taosDecodeFixedI64(buf, &pVgEp->offset); - buf = taosDecodeSEpSet(buf, &pVgEp->epSet); - return buf; -} - -typedef struct { - char topic[TSDB_TOPIC_FNAME_LEN]; - int8_t isSchemaAdaptive; - SArray* vgs; // SArray - SSchemaWrapper schema; -} SMqSubTopicEp; - -static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pTopicEp->topic); - tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive); - int32_t sz = taosArrayGetSize(pTopicEp->vgs); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); - tlen += tEncodeSMqSubVgEp(buf, pVgEp); + static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp * pRsp) { + taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); } - tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { - buf = taosDecodeStringTo(buf, pTopicEp->topic); - buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); - if (pTopicEp->vgs == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp vgEp; - buf = tDecodeSMqSubVgEp(buf, &vgEp); - taosArrayPush(pTopicEp->vgs, &vgEp); - } - buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); - return buf; -} - -static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { - // taosMemoryFree(pSubTopicEp->schema.pSchema); - taosArrayDestroy(pSubTopicEp->vgs); -} - -typedef struct { - SMqRspHead head; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t blockNum; - int8_t withTbName; - int8_t withSchema; - int8_t withTag; - SArray* blockDataLen; // SArray - SArray* blockData; // SArray - SArray* blockTbName; // SArray - SArray* blockSchema; // SArray - SArray* blockTags; // SArray - SArray* blockTagSchema; // SArray -} SMqDataBlkRsp; - -static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); - tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); - tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); - tlen += taosEncodeFixedI32(buf, pRsp->blockNum); - if (pRsp->blockNum != 0) { - tlen += taosEncodeFixedI8(buf, pRsp->withTbName); - tlen += taosEncodeFixedI8(buf, pRsp->withSchema); - tlen += taosEncodeFixedI8(buf, pRsp->withTag); - - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); - void* data = taosArrayGetP(pRsp->blockData, i); - tlen += taosEncodeFixedI32(buf, bLen); - tlen += taosEncodeBinary(buf, data, bLen); - if (pRsp->withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i); - tlen += taosEncodeSSchemaWrapper(buf, pSW); - } - if (pRsp->withTbName) { - char* tbName = (char*)taosArrayGetP(pRsp->blockTbName, i); - tlen += taosEncodeString(buf, tbName); - } - } - } - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); - buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); - buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); - buf = taosDecodeFixedI32(buf, &pRsp->blockNum); - pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); - if (pRsp->blockNum != 0) { - buf = taosDecodeFixedI8(buf, &pRsp->withTbName); - buf = taosDecodeFixedI8(buf, &pRsp->withSchema); - buf = taosDecodeFixedI8(buf, &pRsp->withTag); - - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = 0; - void* data = NULL; - buf = taosDecodeFixedI32(buf, &bLen); - buf = taosDecodeBinary(buf, &data, bLen); - taosArrayPush(pRsp->blockDataLen, &bLen); - taosArrayPush(pRsp->blockData, &data); - if (pRsp->withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); - buf = taosDecodeSSchemaWrapper(buf, pSW); - taosArrayPush(pRsp->blockSchema, &pSW); - } - if (pRsp->withTbName) { - char* name = NULL; - buf = taosDecodeString(buf, &name); - taosArrayPush(pRsp->blockTbName, &name); - } - } - } - return (void*)buf; -} - -typedef struct { - SMqRspHead head; - char cgroup[TSDB_CGROUP_LEN]; - SArray* topics; // SArray -} SMqAskEpRsp; - -static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pRsp) { - int32_t tlen = 0; - // tlen += taosEncodeString(buf, pRsp->cgroup); - int32_t sz = taosArrayGetSize(pRsp->topics); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubTopicEp* pVgEp = (SMqSubTopicEp*)taosArrayGet(pRsp->topics, i); - tlen += tEncodeSMqSubTopicEp(buf, pVgEp); - } - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) { - // buf = taosDecodeStringTo(buf, pRsp->cgroup); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); - if (pRsp->topics == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubTopicEp topicEp; - buf = tDecodeSMqSubTopicEp(buf, &topicEp); - taosArrayPush(pRsp->topics, &topicEp); - } - return buf; -} - -static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { - taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); -} typedef struct { void* data; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 03f445b498..96fb210e26 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -354,7 +354,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061A) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x061C) -#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x062D) +#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x061D) +#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x061E) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) diff --git a/include/util/tlog.h b/include/util/tlog.h index dead25a4a8..be31aa8115 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -61,6 +61,7 @@ extern int32_t tqDebugFlag; extern int32_t fsDebugFlag; extern int32_t metaDebugFlag; extern int32_t fnDebugFlag; +extern int32_t smaDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 025ad0ca0d..aaad606feb 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -299,6 +299,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1; return 0; } @@ -480,6 +481,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32; + smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; } static int32_t taosSetClientCfg(SConfig *pCfg) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 021ee8455e..2b164bf6b2 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3562,39 +3562,93 @@ int tDecodeSVCreateTbBatchRsp(SDecoder *pCoder, SVCreateTbBatchRsp *pRsp) { return 0; } -int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) { - int32_t tlen = 0; - - tlen += taosEncodeFixedI64(buf, pReq->ver); - tlen += tEncodeTSma(buf, &pReq->tSma); - - return tlen; -} - -void *tDeserializeSVCreateTSmaReq(void *buf, SVCreateTSmaReq *pReq) { - buf = taosDecodeFixedI64(buf, &(pReq->ver)); - - if ((buf = tDecodeTSma(buf, &pReq->tSma)) == NULL) { - tdDestroyTSma(&pReq->tSma); +int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { + if (tEncodeI8(pCoder, pSma->version) < 0) return -1; + if (tEncodeI8(pCoder, pSma->intervalUnit) < 0) return -1; + if (tEncodeI8(pCoder, pSma->slidingUnit) < 0) return -1; + if (tEncodeI8(pCoder, pSma->timezoneInt) < 0) return -1; + if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1; + if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1; + if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1; + if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1; + if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1; + if (tEncodeI64(pCoder, pSma->interval) < 0) return -1; + if (tEncodeI64(pCoder, pSma->offset) < 0) return -1; + if (tEncodeI64(pCoder, pSma->sliding) < 0) return -1; + if (pSma->exprLen > 0) { + if (tEncodeCStr(pCoder, pSma->expr) < 0) return -1; } - return buf; + if (pSma->tagsFilterLen > 0) { + if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1; + } + + return 0; } -int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) { - int32_t tlen = 0; +int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { + if (tDecodeI8(pCoder, &pSma->version) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->version) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1; + if (tDecodeCStr(pCoder, (const char **)&pSma->indexName) < 0) return -1; + if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1; + if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->offset) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->sliding) < 0) return -1; + if (pSma->exprLen > 0) { + if (tDecodeCStr(pCoder, &pSma->expr) < 0) return -1; + } else { + pSma->expr = NULL; + } + if (pSma->tagsFilterLen > 0) { + if (tDecodeCStr(pCoder, &pSma->tagsFilter) < 0) return -1; + } else { + pSma->tagsFilter = NULL; + } - tlen += taosEncodeFixedI64(buf, pReq->ver); - tlen += taosEncodeFixedI64(buf, pReq->indexUid); - tlen += taosEncodeString(buf, pReq->indexName); - - return tlen; + return 0; } -void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) { - buf = taosDecodeFixedI64(buf, &(pReq->ver)); - buf = taosDecodeFixedI64(buf, &(pReq->indexUid)); - buf = taosDecodeStringTo(buf, pReq->indexName); - return buf; +int32_t tEncodeSVCreateTSmaReq(SEncoder *pCoder, const SVCreateTSmaReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + tEncodeTSma(pCoder, pReq); + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVCreateTSmaReq(SDecoder *pCoder, SVCreateTSmaReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + tDecodeTSma(pCoder, pReq); + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->indexName) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; + if (tDecodeCStr(pCoder, (const char**)&pReq->indexName) < 0) return -1; + + tEndDecode(pCoder); + return 0; } int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 02a8eaa832..869bf0c0a9 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -242,26 +242,35 @@ SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *smaName) { } static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { - SName name = {0}; + SEncoder encoder = {0}; + int32_t contLen = 0; + SName name = {0}; tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); SVCreateTSmaReq req = {0}; - req.tSma.version = 0; - req.tSma.intervalUnit = pSma->intervalUnit; - req.tSma.slidingUnit = pSma->slidingUnit; - req.tSma.timezoneInt = pSma->timezone; - tstrncpy(req.tSma.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); - req.tSma.exprLen = pSma->exprLen; - req.tSma.tagsFilterLen = pSma->tagsFilterLen; - req.tSma.indexUid = pSma->uid; - req.tSma.tableUid = pSma->stbUid; - req.tSma.interval = pSma->interval; - req.tSma.offset = pSma->offset; - req.tSma.sliding = pSma->sliding; - req.tSma.expr = pSma->expr; - req.tSma.tagsFilter = pSma->tagsFilter; + req.version = 0; + req.intervalUnit = pSma->intervalUnit; + req.slidingUnit = pSma->slidingUnit; + req.timezoneInt = pSma->timezone; + tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); + req.exprLen = pSma->exprLen; + req.tagsFilterLen = pSma->tagsFilterLen; + req.indexUid = pSma->uid; + req.tableUid = pSma->stbUid; + req.interval = pSma->interval; + req.offset = pSma->offset; + req.sliding = pSma->sliding; + req.expr = pSma->expr; + req.tagsFilter = pSma->tagsFilter; + + // get length + int32_t ret = 0; + tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret); + if (ret < 0) { + return NULL; + } + contLen += sizeof(SMsgHead); - int32_t contLen = tSerializeSVCreateTSmaReq(NULL, &req) + sizeof(SMsgHead); SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -272,22 +281,38 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm pHead->vgId = htonl(pVgroup->vgId); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); - tSerializeSVCreateTSmaReq(&pBuf, &req); + tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); + if (tEncodeSVCreateTSmaReq(&encoder, &req) < 0) { + taosMemoryFreeClear(pHead); + tEncoderClear(&encoder); + return NULL; + } + + tEncoderClear(&encoder); *pContLen = contLen; return pHead; } static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { + SEncoder encoder = {0}; + int32_t contLen; SName name = {0}; tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); SVDropTSmaReq req = {0}; - req.ver = 0; req.indexUid = pSma->uid; tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); - int32_t contLen = tSerializeSVDropTSmaReq(NULL, &req) + sizeof(SMsgHead); + // get length + int32_t ret = 0; + tEncodeSize(tEncodeSVDropTSmaReq, &req, contLen, ret); + if (ret < 0) { + return NULL; + } + + contLen += sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -298,7 +323,14 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, pHead->vgId = htonl(pVgroup->vgId); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); - tDeserializeSVDropTSmaReq(&pBuf, &req); + tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); + + if (tEncodeSVDropTSmaReq(&encoder, &req) < 0) { + taosMemoryFreeClear(pHead); + tEncoderClear(&encoder); + return NULL; + } + tEncoderClear(&encoder); *pContLen = contLen; return pHead; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 12e89277f4..e02d629c12 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -425,6 +425,10 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); if (tEncodeSVCreateStbReq(&encoder, &req) < 0) { + taosMemoryFreeClear(pHead); + taosMemoryFreeClear(req.pRSmaParam.qmsg1); + taosMemoryFreeClear(req.pRSmaParam.qmsg2); + tEncoderClear(&encoder); return NULL; } tEncoderClear(&encoder); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 58e00ee34a..a8e3860ed1 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -18,12 +18,21 @@ target_sources( "src/meta/metaOpen.c" "src/meta/metaIdx.c" "src/meta/metaTable.c" + "src/meta/metaSma.c" "src/meta/metaQuery.c" "src/meta/metaCommit.c" "src/meta/metaEntry.c" + # sma + "src/sma/sma.c" + "src/sma/smaTDBImpl.c" + "src/sma/smaEnv.c" + "src/sma/smaOpen.c" + "src/sma/smaRollup.c" + "src/sma/smaTimeRange.c" + # tsdb - "src/tsdb/tsdbTDBImpl.c" + # "src/tsdb/tsdbTDBImpl.c" "src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit2.c" "src/tsdb/tsdbFile.c" @@ -33,7 +42,7 @@ target_sources( "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" - "src/tsdb/tsdbSma.c" + # "src/tsdb/tsdbSma.c" "src/tsdb/tsdbWrite.c" # tq diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 130cebf0b1..be131148fe 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -191,6 +191,9 @@ struct SMetaEntry { int32_t ttlDays; SSchemaWrapper schema; } ntbEntry; + struct { + STSmaWrapper tsma; + } smaEntry; }; }; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 60b3a889ef..9b52e3d854 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -108,6 +108,11 @@ typedef struct { tb_uid_t uid; } STtlIdxKey; +typedef struct { + tb_uid_t uid; + int64_t smaUid; +} SSmaIdxKey; + #if 1 SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); @@ -118,7 +123,7 @@ int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); // SMetaDB int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); -// int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); +int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h new file mode 100644 index 0000000000..76a30f58dd --- /dev/null +++ b/source/dnode/vnode/src/inc/sma.h @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_SMA_H_ +#define _TD_VNODE_SMA_H_ + +#include "vnodeInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// smaDebug ================ +// clang-format off +#define smaFatal(...) do { if (smaDebugFlag & DEBUG_FATAL) { taosPrintLog("SMA FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define smaError(...) do { if (smaDebugFlag & DEBUG_ERROR) { taosPrintLog("SMA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define smaWarn(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define smaInfo(...) do { if (smaDebugFlag & DEBUG_INFO) { taosPrintLog("SMA ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) +// clang-format on + +typedef struct SSmaEnv SSmaEnv; +typedef struct SSmaStat SSmaStat; +typedef struct SSmaStatItem SSmaStatItem; +typedef struct SSmaKey SSmaKey; +typedef struct SRSmaInfo SRSmaInfo; + +#define SMA_IVLD_FID INT_MIN + +struct SSmaEnv { + TdThreadRwlock lock; + int8_t type; + TXN txn; + void *pPool; // SPoolMem + SDiskID did; + TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + char *path; // relative path + SSmaStat *pStat; +}; + +#define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_TYPE(env) ((env)->type) +#define SMA_ENV_DID(env) ((env)->did) +#define SMA_ENV_ENV(env) ((env)->dbEnv) +#define SMA_ENV_PATH(env) ((env)->path) +#define SMA_ENV_STAT(env) ((env)->pStat) +#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) + +struct SSmaStatItem { + /** + * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. + * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from + * Streaming Module or TSDB local persistence. + * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, + * without information about its previous state. + * - TSDB_SMA_STAT_DROPPED: 1)sma dropped + * N.B. only applicable to tsma + */ + int8_t state; // ETsdbSmaStat + SHashObj *expiredWindows; // key: skey of time window, value: N/A + STSma *pTSma; // cache schema +}; + +struct SSmaStat { + union { + SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem for tsma + SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + }; + T_REF_DECLARE() +}; +#define SMA_STAT_ITEMS(s) ((s)->smaStatItems) +#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash) + +struct SSmaKey { + TSKEY skey; + int64_t groupId; +}; + +typedef struct SDBFile SDBFile; + +struct SDBFile { + int32_t fid; + TDB *pDB; + char *path; +}; + +int32_t tdSmaBeginCommit(SSmaEnv *pEnv); +int32_t tdSmaEndCommit(SSmaEnv *pEnv); + +int32_t smaOpenDBEnv(TENV **ppEnv, const char *path); +int32_t smaCloseDBEnv(TENV *pEnv); +int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF); +int32_t smaCloseDBF(SDBFile *pDBF); +int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); +void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); + +void tdDestroySmaEnv(SSmaEnv *pSmaEnv); +void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); +#if 0 +int32_t tbGetTSmaStatus(SSma *pSma, STSma *param, void *result); +int32_t tbRemoveTSmaData(SSma *pSma, STSma *param, STimeWindow *pWin); +#endif + +static FORCE_INLINE int32_t tdEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { + int32_t len = 0; + len += taosEncodeFixedI64(pData, tsKey); + len += taosEncodeFixedI64(pData, groupId); + return len; +} + +int32_t tdInitSma(SSma *pSma); +int32_t tdDropTSma(SSma *pSma, char *pMsg); +int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid); +int32_t tdInsertRSmaData(SSma *pSma, char *msg); + +int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); +int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); +int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); + +int32_t tdLockSma(SSma *pSma); +int32_t tdUnLockSma(SSma *pSma); + +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); + +static FORCE_INLINE int16_t tdTSmaAdd(SSma *pSma, int16_t n) { return atomic_add_fetch_16(&SMA_TSMA_NUM(pSma), n); } +static FORCE_INLINE int16_t tdTSmaSub(SSma *pSma, int16_t n) { return atomic_sub_fetch_16(&SMA_TSMA_NUM(pSma), n); } + +static FORCE_INLINE int32_t tdRLockSmaEnv(SSmaEnv *pEnv) { + int code = taosThreadRwlockRdlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tdWLockSmaEnv(SSmaEnv *pEnv) { + int code = taosThreadRwlockWrlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) { + int code = taosThreadRwlockUnlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int8_t tdSmaStat(SSmaStatItem *pStatItem) { + if (pStatItem) { + return atomic_load_8(&pStatItem->state); + } + return TSDB_SMA_STAT_UNKNOWN; +} + +static FORCE_INLINE bool tdSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) { + if (!pStatItem) { + return false; + } + + if (state) { + *state = atomic_load_8(&pStatItem->state); + return *state == TSDB_SMA_STAT_OK; + } + return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK; +} + +static FORCE_INLINE bool tdSmaStatIsExpired(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true; +} + +static FORCE_INLINE bool tdSmaStatIsDropped(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true; +} + +static FORCE_INLINE void tdSmaStatSetOK(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK); + } +} + +static FORCE_INLINE void tdSmaStatSetExpired(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED); + } +} + +static FORCE_INLINE void tdSmaStatSetDropped(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED); + } +} + +static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType); +void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem); +static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); +static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did); +static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv); + +void *tdFreeRSmaInfo(SRSmaInfo *pInfo); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_SMA_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 102c40337d..93a25da0a8 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -60,31 +60,22 @@ typedef struct { TSKEY minKey; } SRtn; -struct SSmaEnvs { - int16_t nTSma; - int16_t nRSma; - SSmaEnv *pTSmaEnv; - SSmaEnv *pRSmaEnv; -}; - +#define TSDB_DATA_DIR_LEN 6 struct STsdb { char *path; SVnode *pVnode; TdThreadMutex mutex; + char dir[TSDB_DATA_DIR_LEN]; bool repoLocked; - int8_t level; // retention level STsdbKeepCfg keepCfg; STsdbMemTable *mem; STsdbMemTable *imem; SRtn rtn; STsdbFS *fs; - SSmaEnvs smaEnvs; }; #if 1 // ====================================== -typedef struct SSmaStat SSmaStat; - struct STable { uint64_t tid; uint64_t uid; @@ -95,10 +86,6 @@ struct STable { #define TABLE_UID(t) (t)->uid int tsdbPrepareCommit(STsdb *pTsdb); -int32_t tsdbInitSma(STsdb *pTsdb); -int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg); -int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); -int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); typedef enum { TSDB_FILE_HEAD = 0, // .head TSDB_FILE_DATA, // .data @@ -107,8 +94,6 @@ typedef enum { TSDB_FILE_SMAL, // .smal(Block-wise SMA) TSDB_FILE_MAX, // TSDB_FILE_META, // meta - TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA - TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA } E_TSDB_FILE_T; typedef struct { @@ -186,15 +171,10 @@ struct STsdbFS { #define REPO_ID(r) TD_VID((r)->pVnode) #define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg) #define REPO_KEEP_CFG(r) (&(r)->keepCfg) -#define REPO_LEVEL(r) ((r)->level) #define REPO_FS(r) ((r)->fs) #define REPO_META(r) ((r)->pVnode->pMeta) #define REPO_TFS(r) ((r)->pVnode->pTfs) #define IS_REPO_LOCKED(r) ((r)->repoLocked) -#define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma) -#define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma) -#define REPO_TSMA_ENV(r) ((r)->smaEnvs.pTSmaEnv) -#define REPO_RSMA_ENV(r) ((r)->smaEnvs.pRSmaEnv) int tsdbLockRepo(STsdb *pTsdb); int tsdbUnlockRepo(STsdb *pTsdb); @@ -794,25 +774,6 @@ typedef struct { } SFSHeader; // ================== TSDB File System Meta - -/** - * @brief Directory structure of .tsma data files. - * - * /vnode2/tsdb $ tree tsma/ - * tsma/ - * ├── v2f100.index_name_1 - * ├── v2f101.index_name_1 - * ├── v2f102.index_name_1 - * ├── v2f1900.index_name_3 - * ├── v2f1901.index_name_3 - * ├── v2f1902.index_name_3 - * ├── v2f200.index_name_2 - * ├── v2f201.index_name_2 - * └── v2f202.index_name_2 - * - * 0 directories, 9 files - */ - #define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus) #define FS_NEW_STATUS(pfs) ((pfs)->nstatus) #define FS_IN_TXN(pfs) (pfs)->intxn @@ -874,43 +835,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { return 0; } -typedef struct SSmaKey SSmaKey; - -struct SSmaKey { - TSKEY skey; - int64_t groupId; -}; - -typedef struct SDBFile SDBFile; - -struct SDBFile { - int32_t fid; - TDB *pDB; - char *path; -}; - -int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path); -int32_t tsdbCloseDBEnv(TENV *pEnv); -int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF); -int32_t tsdbCloseDBF(SDBFile *pDBF); -int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); -void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); - -void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); -void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); -#if 0 -int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); -int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); -#endif - -// internal func -static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { - int32_t len = 0; - len += taosEncodeFixedI64(pData, tsKey); - len += taosEncodeFixedI64(pData, groupId); - return len; -} - #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9a36fc6eae..c9d1a0e06e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -47,13 +47,15 @@ extern "C" { #endif -typedef struct SVnodeInfo SVnodeInfo; -typedef struct SMeta SMeta; -typedef struct STsdb STsdb; -typedef struct STQ STQ; -typedef struct SVState SVState; -typedef struct SVBufPool SVBufPool; -typedef struct SQWorker SQHandle; +typedef struct SVnodeInfo SVnodeInfo; +typedef struct SMeta SMeta; +typedef struct SSma SSma; +typedef struct STsdb STsdb; +typedef struct STQ STQ; +typedef struct SVState SVState; +typedef struct SVBufPool SVBufPool; +typedef struct SQWorker SQHandle; +typedef struct STsdbKeepCfg STsdbKeepCfg; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -90,17 +92,14 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup); void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode); STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid); -int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); +int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); // tsdb -int tsdbOpen(SVnode* pVnode, int8_t type); -int tsdbClose(STsdb* pTsdb); +int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg *pKeepCfg); +int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb); -int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version); -int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); -int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, @@ -121,13 +120,31 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); // sma +int32_t smaOpen(SVnode* pVnode); +int32_t smaClose(SSma* pSma); +int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version); +int32_t tdProcessTSmaCreate(SSma* pSma, char* pMsg); +int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); + +int32_t tdProcessRSmaCreate(SSma* pSma, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb); +int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); +int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); +int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore); +void tdUidStoreDestory(STbUidStore* pStore); +void* tdUidStoreFree(STbUidStore* pStore); + +#if 0 +int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version); +int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); +int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb); int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); +#endif typedef struct { int8_t streamType; // sma or other @@ -164,13 +181,13 @@ typedef enum { TSDB_TYPE_RSMA_L2 = 4, // RSMA Level 2 } ETsdbType; -typedef struct { +struct STsdbKeepCfg{ int8_t precision; // precision always be used with below keep cfgs int32_t days; int32_t keep0; int32_t keep1; int32_t keep2; -} STsdbKeepCfg; +}; struct SVnode { char* path; @@ -183,9 +200,8 @@ struct SVnode { SVBufPool* onCommit; SVBufPool* onRecycle; SMeta* pMeta; + SSma* pSma; STsdb* pTsdb; - STsdb* pRSma1; - STsdb* pRSma2; SWal* pWal; STQ* pTq; SSink* pSink; @@ -194,10 +210,12 @@ struct SVnode { SQHandle* pQuery; }; +#define TD_VID(PVNODE) (PVNODE)->config.vgId + #define VND_TSDB(vnd) ((vnd)->pTsdb) #define VND_RSMA0(vnd) ((vnd)->pTsdb) -#define VND_RSMA1(vnd) ((vnd)->pRSma1) -#define VND_RSMA2(vnd) ((vnd)->pRSma2) +#define VND_RSMA1(vnd) ((vnd)->pSma->pRSmaTsdb1) +#define VND_RSMA2(vnd) ((vnd)->pSma->pRSmaTsdb2) #define VND_RETENTIONS(vnd) (&(vnd)->config.tsdbCfg.retentions) struct STbUidStore { @@ -207,7 +225,29 @@ struct STbUidStore { SHashObj* uidHash; }; -#define TD_VID(PVNODE) (PVNODE)->config.vgId +struct SSma { + int16_t nTSma; + bool locked; + TdThreadMutex mutex; + SVnode* pVnode; + STsdb* pRSmaTsdb1; + STsdb* pRSmaTsdb2; + void* pTSmaEnv; + void* pRSmaEnv; +}; + +#define SMA_CFG(s) (&(s)->pVnode->config) +#define SMA_TSDB_CFG(s) (&(s)->pVnode->config.tsdbCfg) +#define SMA_LOCKED(s) ((s)->locked) +#define SMA_META(s) ((s)->pVnode->pMeta) +#define SMA_VID(s) TD_VID((s)->pVnode) +#define SMA_TFS(s) ((s)->pVnode->pTfs) +#define SMA_TSMA_NUM(s) ((s)->nTSma) +#define SMA_TSMA_ENV(s) ((s)->pTSmaEnv) +#define SMA_RSMA_ENV(s) ((s)->pRSmaEnv) +#define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb) +#define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb1) +#define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb2) static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) { SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 581b876e84..8ddf3419b5 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -35,6 +35,8 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; + } else if (pME->type == TSDB_TSMA_TABLE) { + if (tEncodeTSmaWrapper(pCoder, &pME->smaEntry.tsma) < 0) return -1; } else { ASSERT(0); } diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 853b2ecefb..3f52071315 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -112,35 +112,4 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) { #endif // TODO return 0; -} - -int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) { - // TODO: Validate the cfg - // The table uid should exists and be super table or common table. - // Check other cfg value - - // TODO: add atomicity - -#ifdef META_REFACT -#else - if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) { - // TODO: handle error - return -1; - } -#endif - return TSDB_CODE_SUCCESS; -} - -int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { - // TODO: Validate the cfg - // TODO: add atomicity - -#ifdef META_REFACT -#else - if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) { - // TODO: handle error - return -1; - } -#endif - return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 07422e3193..db47e794ba 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -21,6 +21,7 @@ static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); } static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); } @@ -121,7 +122,7 @@ _err: if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); - if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx); + if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); @@ -137,7 +138,7 @@ int metaClose(SMeta *pMeta) { if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); - if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx); + if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); @@ -295,3 +296,22 @@ static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return 0; } + +static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + SSmaIdxKey *pSmaIdxKey1 = (SSmaIdxKey *)pKey1; + SSmaIdxKey *pSmaIdxKey2 = (SSmaIdxKey *)pKey2; + + if (pSmaIdxKey1->uid > pSmaIdxKey2->uid) { + return 1; + } else if (pSmaIdxKey1->uid < pSmaIdxKey2->uid) { + return -1; + } + + if (pSmaIdxKey1->smaUid > pSmaIdxKey2->smaUid) { + return 1; + } else if (pSmaIdxKey1->smaUid < pSmaIdxKey2->smaUid) { + return -1; + } + + return 0; +} diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c new file mode 100644 index 0000000000..e7d4d3e117 --- /dev/null +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "meta.h" + +static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME); +static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME); + +int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { + // TODO: Validate the cfg + // The table uid should exists and be super table or normal table. + // Check other cfg value + + // TODO: add atomicity + + SMetaEntry me = {0}; + int kLen = 0; + int vLen = 0; + const void *pKey = NULL; + const void *pVal = NULL; + void *pBuf = NULL; + int32_t szBuf = 0; + void *p = NULL; + SMetaReader mr = {0}; + + // validate req + metaReaderInit(&mr, pMeta, 0); + if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) { +// TODO: just for pass case +#if 1 + terrno = TSDB_CODE_TDB_TSMA_ALREADY_EXIST; + metaReaderClear(&mr); + return -1; +#else + metaReaderClear(&mr); + return 0; +#endif + } + metaReaderClear(&mr); + + // set structs + me.version = version; + me.type = TSDB_TSMA_TABLE; + me.uid = pCfg->indexUid; + me.name = pCfg->indexName; + // me.smaEntry = xx; + + if (metaHandleSmaEntry(pMeta, &me) < 0) goto _err; + + metaDebug("vgId:%d tsma is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid); + + return 0; + +_err: + metaError("vgId:%d failed to create tsma: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName, + pCfg->indexUid, tstrerror(terrno)); + return -1; +} + +int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { + // TODO: Validate the cfg + // TODO: add atomicity + +#ifdef META_REFACT +#else + if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) { + // TODO: handle error + return -1; + } +#endif + return TSDB_CODE_SUCCESS; +} + +// static int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { +// int32_t ret = 0; +// void *pBuf = NULL, *qBuf = NULL; +// void *key = {0}, *val = {0}; + +// // save sma info +// int32_t len = tEncodeTSma(NULL, pSmaCfg); +// pBuf = taosMemoryCalloc(1, len); +// if (pBuf == NULL) { +// terrno = TSDB_CODE_OUT_OF_MEMORY; +// return -1; +// } + +// key = (void *)&pSmaCfg->indexUid; +// qBuf = pBuf; +// tEncodeTSma(&qBuf, pSmaCfg); +// val = pBuf; + +// int32_t kLen = sizeof(pSmaCfg->indexUid); +// int32_t vLen = POINTER_DISTANCE(qBuf, pBuf); + +// ret = tdbDbInsert(pMeta->pTbDb, key, kLen, val, vLen, &pMeta->txn); +// if (ret < 0) { +// taosMemoryFreeClear(pBuf); +// return -1; +// } + +// // add sma idx +// SSmaIdxKey smaIdxKey; +// smaIdxKey.uid = pSmaCfg->tableUid; +// smaIdxKey.smaUid = pSmaCfg->indexUid; +// key = &smaIdxKey; +// kLen = sizeof(smaIdxKey); +// val = NULL; +// vLen = 0; + +// ret = tdbDbInsert(pMeta->pSmaIdx, key, kLen, val, vLen, &pMeta->txn); +// if (ret < 0) { +// taosMemoryFreeClear(pBuf); +// return -1; +// } + +// // release +// taosMemoryFreeClear(pBuf); + +// return 0; +// } + + +static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { + STbDbKey tbDbKey; + void *pKey = NULL; + void *pVal = NULL; + int kLen = 0; + int vLen = 0; + SEncoder coder = {0}; + + // set key and value + tbDbKey.version = pME->version; + tbDbKey.uid = pME->uid; + + pKey = &tbDbKey; + kLen = sizeof(tbDbKey); + + int32_t ret = 0; + tEncodeSize(metaEncodeEntry, pME, vLen, ret); + if (ret < 0) { + goto _err; + } + + pVal = taosMemoryMalloc(vLen); + if (pVal == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tEncoderInit(&coder, pVal, vLen); + + if (metaEncodeEntry(&coder, pME) < 0) { + goto _err; + } + + tEncoderClear(&coder); + + // write to table.db + if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + goto _err; + } + + taosMemoryFree(pVal); + return 0; + +_err: + taosMemoryFree(pVal); + return -1; +} + + + + +static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { + metaWLock(pMeta); + + // save to table.db + if (metaSaveSmaToDB(pMeta, pME) < 0) goto _err; + + // // update uid.idx + // if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err; + + metaULock(pMeta); + return 0; + +_err: + metaULock(pMeta); + return -1; +} diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/sma/sma.c similarity index 50% rename from source/dnode/vnode/src/inc/tsdbSma.h rename to source/dnode/vnode/src/sma/sma.c index 5215812ac5..2c54e10087 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/sma/sma.c @@ -13,35 +13,18 @@ * along with this program. If not, see . */ -#ifndef _TD_VNODE_TSDB_SMA_H_ -#define _TD_VNODE_TSDB_SMA_H_ +#include "sma.h" -#include "tsdb.h" -#ifdef __cplusplus -extern "C" { -#endif +// TODO: Who is responsible for resource allocate and release? +int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) { + int32_t code = TSDB_CODE_SUCCESS; -// typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2); - -// struct STbDdlH { -// void *ahandle; -// void *result; -// __tb_ddl_fn_t fp; -// }; - -static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) { - ASSERT(*pStore == NULL); - *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); - if (*pStore == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { + smaWarn("vgId:%d insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); } - return TSDB_CODE_SUCCESS; + // TODO: destroy SSDataBlocks(msg) + return code; } -#ifdef __cplusplus -} -#endif -#endif /*_TD_VNODE_TSDB_SMA_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c new file mode 100644 index 0000000000..c02276f5fe --- /dev/null +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -0,0 +1,463 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" + +typedef struct SSmaStat SSmaStat; + +static const char *TSDB_SMA_DNAME[] = { + "", // TSDB_SMA_TYPE_BLOCK + "tsma", // TSDB_SMA_TYPE_TIME_RANGE + "rsma", // TSDB_SMA_TYPE_ROLLUP +}; + +#define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test +#define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test +#define SMA_STATE_HASH_SLOT 4 + +#define RSMA_TASK_INFO_HASH_SLOT 8 + +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +// declaration of static functions + +// insert data + +static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]); + +// Pool Memory +static SPoolMem *openPool(); +static void clearPool(SPoolMem *pPool); +static void closePool(SPoolMem *pPool); +static void *poolMalloc(void *arg, size_t size); +static void poolFree(void *arg, void *ptr); + +// implementation + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + if (!pPool) return; + + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + if (pPool) { + clearPool(pPool); + taosMemoryFree(pPool); + } +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (!pMem) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); +} + +int32_t tdInitSma(SSma *pSma) { + // tSma + int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(SMA_META(pSma), false)); + if (numOfTSma > 0) { + atomic_store_16(&SMA_TSMA_NUM(pSma), (int16_t)numOfTSma); + } + // TODO: rSma + return TSDB_CODE_SUCCESS; +} + +static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TSDB_SMA_DNAME[smaType]); +} + +static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did) { + SSmaEnv *pEnv = NULL; + + pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv)); + if (!pEnv) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMA_ENV_TYPE(pEnv) = smaType; + + int code = taosThreadRwlockInit(&(pEnv->lock), NULL); + if (code) { + terrno = TAOS_SYSTEM_ERROR(code); + taosMemoryFree(pEnv); + return NULL; + } + + ASSERT(path && (strlen(path) > 0)); + SMA_ENV_PATH(pEnv) = strdup(path); + if (!SMA_ENV_PATH(pEnv)) { + tdFreeSmaEnv(pEnv); + return NULL; + } + + SMA_ENV_DID(pEnv) = did; + + if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) { + tdFreeSmaEnv(pEnv); + return NULL; + } + + char aname[TSDB_FILENAME_LEN] = {0}; + tfsAbsoluteName(SMA_TFS(pSma), did, path, aname); + if (smaOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { + tdFreeSmaEnv(pEnv); + return NULL; + } + + if (!(pEnv->pPool = openPool())) { + tdFreeSmaEnv(pEnv); + return NULL; + } + + return pEnv; +} + +static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv) { + if (!pEnv) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + if (!(*pEnv)) { + if (!(*pEnv = tdNewSmaEnv(pSma, smaType, path, did))) { + return TSDB_CODE_FAILED; + } + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Release resources allocated for its member fields, not including itself. + * + * @param pSmaEnv + * @return int32_t + */ +void tdDestroySmaEnv(SSmaEnv *pSmaEnv) { + if (pSmaEnv) { + tdDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv)); + taosMemoryFreeClear(pSmaEnv->pStat); + taosMemoryFreeClear(pSmaEnv->path); + taosThreadRwlockDestroy(&(pSmaEnv->lock)); + smaCloseDBEnv(pSmaEnv->dbEnv); + closePool(pSmaEnv->pPool); + } +} + +void *tdFreeSmaEnv(SSmaEnv *pSmaEnv) { + tdDestroySmaEnv(pSmaEnv); + taosMemoryFreeClear(pSmaEnv); + return NULL; +} + +int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat) { + if (!pStat) return 0; + + int ref = T_REF_INC(pStat); + smaDebug("vgId:%d ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); + return 0; +} + +int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { + if (!pStat) return 0; + + int ref = T_REF_DEC(pStat); + smaDebug("vgId:%d unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); + return 0; +} + +static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { + ASSERT(pSmaStat != NULL); + + if (*pSmaStat) { // no lock + return TSDB_CODE_SUCCESS; + } + + /** + * 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tdNew). + * 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if + * tdInitSmaStat invoked in other multithread environment later. + */ + if (!(*pSmaStat)) { + *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat)); + if (!(*pSmaStat)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + if (smaType == TSDB_SMA_TYPE_ROLLUP) { + SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit( + RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + + if (!SMA_STAT_INFO_HASH(*pSmaStat)) { + taosMemoryFreeClear(*pSmaStat); + return TSDB_CODE_FAILED; + } + } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { + SMA_STAT_ITEMS(*pSmaStat) = + taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + if (!SMA_STAT_ITEMS(*pSmaStat)) { + taosMemoryFreeClear(*pSmaStat); + return TSDB_CODE_FAILED; + } + } else { + ASSERT(0); + } + } + return TSDB_CODE_SUCCESS; +} + +void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { + if (pSmaStatItem) { + tdDestroyTSma(pSmaStatItem->pTSma); + taosMemoryFreeClear(pSmaStatItem->pTSma); + taosHashCleanup(pSmaStatItem->expiredWindows); + taosMemoryFreeClear(pSmaStatItem); + } + return NULL; +} + +/** + * @brief Release resources allocated for its member fields, not including itself. + * + * @param pSmaStat + * @return int32_t + */ +int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { + if (pSmaStat) { + // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. + if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { + void *item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), NULL); + while (item) { + SSmaStatItem *pItem = *(SSmaStatItem **)item; + tdFreeSmaStatItem(pItem); + item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), item); + } + taosHashCleanup(SMA_STAT_ITEMS(pSmaStat)); + } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { + void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL); + while (infoHash) { + SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; + tdFreeRSmaInfo(pInfoHash); + infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash); + } + taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat)); + } else { + ASSERT(0); + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdLockSma(SSma *pSma) { + int code = taosThreadMutexLock(&pSma->mutex); + if (code != 0) { + smaError("vgId:%d failed to lock td since %s", SMA_VID(pSma), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + pSma->locked = true; + return 0; +} + +int32_t tdUnLockSma(SSma *pSma) { + ASSERT(SMA_LOCKED(pSma)); + pSma->locked = false; + int code = taosThreadMutexUnlock(&pSma->mutex); + if (code != 0) { + smaError("vgId:%d failed to unlock td since %s", SMA_VID(pSma), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { + SSmaEnv *pEnv = NULL; + + // return if already init + switch (smaType) { + case TSDB_SMA_TYPE_TIME_RANGE: + if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_TSMA_ENV(pSma)))) { + return TSDB_CODE_SUCCESS; + } + break; + case TSDB_SMA_TYPE_ROLLUP: + if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_RSMA_ENV(pSma)))) { + return TSDB_CODE_SUCCESS; + } + break; + default: + TASSERT(0); + return TSDB_CODE_FAILED; + } + + // init sma env + tdLockSma(pSma); + pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&SMA_TSMA_ENV(pSma)) + : atomic_load_ptr(&SMA_RSMA_ENV(pSma)); + if (!pEnv) { + char rname[TSDB_FILENAME_LEN] = {0}; + + SDiskID did = {0}; + if (tfsAllocDisk(SMA_TFS(pSma), TFS_PRIMARY_LEVEL, &did) < 0) { + tdUnLockSma(pSma); + return TSDB_CODE_FAILED; + } + + if (did.level < 0 || did.id < 0) { + tdUnLockSma(pSma); + smaError("vgId:%d init sma env failed since invalid did(%d,%d)", SMA_VID(pSma), did.level, did.id); + return TSDB_CODE_FAILED; + } + + tdGetSmaDir(SMA_VID(pSma), smaType, rname); + + if (tfsMkdirRecurAt(SMA_TFS(pSma), rname, did) < 0) { + tdUnLockSma(pSma); + return TSDB_CODE_FAILED; + } + + if (tdInitSmaEnv(pSma, smaType, rname, did, &pEnv) < 0) { + tdUnLockSma(pSma); + return TSDB_CODE_FAILED; + } + + (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), pEnv) + : atomic_store_ptr(&SMA_RSMA_ENV(pSma), pEnv); + } + tdUnLockSma(pSma); + + return TSDB_CODE_SUCCESS; +}; + +int32_t tdSmaBeginCommit(SSmaEnv *pEnv) { + TXN *pTxn = &pEnv->txn; + // start a new txn + tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + if (tdbBegin(pEnv->dbEnv, pTxn) != 0) { + smaWarn("tdSma tdb begin commit fail"); + return -1; + } + return 0; +} + +int32_t tdSmaEndCommit(SSmaEnv *pEnv) { + TXN *pTxn = &pEnv->txn; + + // Commit current txn + if (tdbCommit(pEnv->dbEnv, pTxn) != 0) { + smaWarn("tdSma tdb end commit fail"); + return -1; + } + tdbTxnClose(pTxn); + clearPool(pEnv->pPool); + return 0; +} + +#if 0 +/** + * @brief Get the start TS key of the last data block of one interval/sliding. + * + * @param pSma + * @param param + * @param result + * @return int32_t + * 1) Return 0 and fill the result if the check procedure is normal; + * 2) Return -1 if error occurs during the check procedure. + */ +int32_t tdGetTSmaStatus(SSma *pSma, void *smaIndex, void *result) { + const char *procedure = ""; + if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) { + return -1; + } + // fill the result + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Remove the tSma data files related to param between pWin. + * + * @param pSma + * @param param + * @param pWin + * @return int32_t + */ +int32_t tdRemoveTSmaData(SSma *pSma, void *smaIndex, STimeWindow *pWin) { + // for ("tSmaFiles of param-interval-sliding between pWin") { + // // remove the tSmaFile + // } + return TSDB_CODE_SUCCESS; +} +#endif diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c new file mode 100644 index 0000000000..1c7db28e18 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +static int32_t smaEvalDays(SRetention *r, int8_t precision); +static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type); + +#define SMA_SET_KEEP_CFG(l) \ + do { \ + SRetention *r = &pCfg->retentions[l]; \ + pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \ + pKeepCfg->keep0 = pKeepCfg->keep2; \ + pKeepCfg->keep1 = pKeepCfg->keep2; \ + pKeepCfg->days = smaEvalDays(r, pCfg->precision); \ + } while (0) + +#define SMA_OPEN_RSMA_IMPL(v, l) \ + do { \ + SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \ + if (!RETENTION_VALID(r)) { \ + if (l == 0) { \ + goto _err; \ + } \ + break; \ + } \ + smaSetKeepCfg(&keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ + if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg) < 0) { \ + goto _err; \ + } \ + } while (0) + +#define RETENTION_DAYS_SPLIT_RATIO 10 +#define RETENTION_DAYS_SPLIT_MIN 1 +#define RETENTION_DAYS_SPLIT_MAX 30 + +static int32_t smaEvalDays(SRetention *r, int8_t precision) { + int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY); + int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY); + + int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO; + if (days <= RETENTION_DAYS_SPLIT_MIN) { + days = RETENTION_DAYS_SPLIT_MIN; + if (days < freqDays) { + days = freqDays + 1; + } + } else { + if (days > RETENTION_DAYS_SPLIT_MAX) { + days = RETENTION_DAYS_SPLIT_MAX; + } + if (days < freqDays) { + days = freqDays + 1; + } + } + return days * 1440; +} + +int smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) { + pKeepCfg->precision = pCfg->precision; + switch (type) { + case TSDB_TYPE_TSMA: + ASSERT(0); + break; + case TSDB_TYPE_RSMA_L0: + SMA_SET_KEEP_CFG(0); + break; + case TSDB_TYPE_RSMA_L1: + SMA_SET_KEEP_CFG(1); + break; + case TSDB_TYPE_RSMA_L2: + SMA_SET_KEEP_CFG(2); + break; + default: + ASSERT(0); + break; + } + return 0; +} + +int32_t smaOpen(SVnode *pVnode) { + STsdbCfg *pCfg = &pVnode->config.tsdbCfg; + + ASSERT(!pVnode->pSma); + + SSma *pSma = taosMemoryCalloc(1, sizeof(SSma)); + if (!pSma) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pSma->pVnode = pVnode; + taosThreadMutexInit(&pSma->mutex, NULL); + pSma->locked = false; + + if (vnodeIsRollup(pVnode)) { + STsdbKeepCfg keepCfg = {0}; + for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { + if (i == TSDB_RETENTION_L0) { + SMA_OPEN_RSMA_IMPL(pVnode, 0); + } else if (i == TSDB_RETENTION_L1) { + SMA_OPEN_RSMA_IMPL(pVnode, 1); + } else if (i == TSDB_RETENTION_L2) { + SMA_OPEN_RSMA_IMPL(pVnode, 2); + } else { + ASSERT(0); + } + } + } + + pVnode->pSma = pSma; + return 0; +_err: + taosMemoryFreeClear(pSma); + return -1; +} + +int32_t smaClose(SSma *pSma) { + if (pSma) { + taosThreadMutexDestroy(&pSma->mutex); + if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma)); + if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma)); + if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma)); + } + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c new file mode 100644 index 0000000000..f9cb5a1a09 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" + +static FORCE_INLINE int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); +static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); +static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo, + STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level); + +struct SRSmaInfo { + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t +}; + +static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) { + // Note: free/kill may in RC + qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); + if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { + qDestroyTask(otaskHandle); + } +} + +void *tdFreeRSmaInfo(SRSmaInfo *pInfo) { + for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { + if (pInfo->taskInfo[i]) { + tdFreeTaskHandle(pInfo->taskInfo[i]); + } + } + return NULL; +} + +static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { + ASSERT(*pStore == NULL); + *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); + if (*pStore == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaInfo *pRSmaInfo = NULL; + + if (!suid || !tbUids) { + terrno = TSDB_CODE_INVALID_PTR; + smaError("vgId:%d failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } + + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + smaError("vgId:%d failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + return TSDB_CODE_FAILED; + } + + if (pRSmaInfo->taskInfo[0] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[0], tbUids, true) != 0)) { + smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } + + if (pRSmaInfo->taskInfo[1] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[1], tbUids, true) != 0)) { + smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->taskInfo[1], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore) { + if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) { + return TSDB_CODE_SUCCESS; + } + + if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } + + void *pIter = taosHashIterate(pStore->uidHash, NULL); + while (pIter) { + tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + SArray *pTbUids = *(SArray **)pIter; + + if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) { + taosHashCancelIterate(pStore->uidHash, pIter); + return TSDB_CODE_FAILED; + } + + pIter = taosHashIterate(pStore->uidHash, pIter); + } + return TSDB_CODE_SUCCESS; +} + +/** + * @brief fetch suid/uids when create child tables of rollup SMA + * + * @param pTsdb + * @param ppStore + * @param suid + * @param uid + * @return int32_t + */ +int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_uid_t uid) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + + // only applicable to rollup SMA ctables + if (!pEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SHashObj *infoHash = NULL; + if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + return TSDB_CODE_FAILED; + } + + // info cached when create rsma stable and return directly for non-rsma ctables + if (!taosHashGet(infoHash, &suid, sizeof(tb_uid_t))) { + return TSDB_CODE_SUCCESS; + } + + ASSERT(ppStore != NULL); + + if (!(*ppStore)) { + if (tdUidStoreInit(ppStore) != 0) { + return TSDB_CODE_FAILED; + } + } + + if (tdUidStorePut(*ppStore, suid, &uid) != 0) { + *ppStore = tdUidStoreFree(*ppStore); + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * + * @param pTsdb + * @param pMeta + * @param pReq + * @return int32_t + */ +int32_t tdProcessRSmaCreate(SSma *pSma, SMeta *pMeta, SVCreateStbReq *pReq, SMsgCb *pMsgCb) { + if (!pReq->rollup) { + smaTrace("vgId:%d return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + return TSDB_CODE_SUCCESS; + } + + SRSmaParam *param = &pReq->pRSmaParam; + + if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) { + smaWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + return TSDB_CODE_SUCCESS; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaInfo *pRSmaInfo = NULL; + + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); + if (pRSmaInfo) { + smaWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + return TSDB_CODE_SUCCESS; + } + + pRSmaInfo = (SRSmaInfo *)taosMemoryCalloc(1, sizeof(SRSmaInfo)); + if (!pRSmaInfo) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta); + if (!pReadHandle) { + taosMemoryFree(pRSmaInfo); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + SReadHandle handle = { + .reader = pReadHandle, + .meta = pMeta, + .pMsgCb = pMsgCb, + }; + + if (param->qmsg1) { + pRSmaInfo->taskInfo[0] = qCreateStreamExecTaskInfo(param->qmsg1, &handle); + if (!pRSmaInfo->taskInfo[0]) { + taosMemoryFree(pRSmaInfo); + taosMemoryFree(pReadHandle); + return TSDB_CODE_FAILED; + } + } + + if (param->qmsg2) { + pRSmaInfo->taskInfo[1] = qCreateStreamExecTaskInfo(param->qmsg2, &handle); + if (!pRSmaInfo->taskInfo[1]) { + taosMemoryFree(pRSmaInfo); + taosMemoryFree(pReadHandle); + return TSDB_CODE_FAILED; + } + } + + if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != + TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief store suid/[uids], prefer to use array and then hash + * + * @param pStore + * @param suid + * @param uid + * @return int32_t + */ +static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) { + // prefer to store suid/uids in array + if ((suid == pStore->suid) || (pStore->suid == 0)) { + if (pStore->suid == 0) { + pStore->suid = suid; + } + if (uid) { + if (!pStore->tbUids) { + if (!(pStore->tbUids = taosArrayInit(1, sizeof(tb_uid_t)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + if (!taosArrayPush(pStore->tbUids, uid)) { + return TSDB_CODE_FAILED; + } + } + } else { + // store other suid/uids in hash when multiple stable/table included in 1 batch of request + if (!pStore->uidHash) { + pStore->uidHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + if (!pStore->uidHash) { + return TSDB_CODE_FAILED; + } + } + if (uid) { + SArray *uidArray = taosHashGet(pStore->uidHash, &suid, sizeof(tb_uid_t)); + if (uidArray && ((uidArray = *(SArray **)uidArray))) { + taosArrayPush(uidArray, uid); + } else { + SArray *pUidArray = taosArrayInit(1, sizeof(tb_uid_t)); + if (!pUidArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + if (!taosArrayPush(pUidArray, uid)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) { + return TSDB_CODE_FAILED; + } + } + } else { + if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) { + return TSDB_CODE_FAILED; + } + } + } + return TSDB_CODE_SUCCESS; +} + +void tdUidStoreDestory(STbUidStore *pStore) { + if (pStore) { + if (pStore->uidHash) { + if (pStore->tbUids) { + // When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys. + void *pIter = taosHashIterate(pStore->uidHash, NULL); + while (pIter) { + SArray *arr = *(SArray **)pIter; + taosArrayDestroy(arr); + pIter = taosHashIterate(pStore->uidHash, pIter); + } + } + taosHashCleanup(pStore->uidHash); + } + taosArrayDestroy(pStore->tbUids); + } +} + +void *tdUidStoreFree(STbUidStore *pStore) { + if (pStore) { + tdUidStoreDestory(pStore); + taosMemoryFree(pStore); + } + return NULL; +} + +static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { + if (!pReq) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { + ASSERT(pMsg != NULL); + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SSubmitBlkIter blkIter = {0}; + STSRow *row = NULL; + + terrno = TSDB_CODE_SUCCESS; + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; + while (true) { + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + + if (!pBlock) break; + tdUidStorePut(pStore, msgIter.suid, NULL); + pStore->uid = msgIter.uid; // TODO: remove, just for debugging + } + + if (terrno != TSDB_CODE_SUCCESS) return -1; + return 0; +} + +static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo, + STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level) { + SArray *pResult = NULL; + + if (!taskInfo) { + smaDebug("vgId:%d no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); + return TSDB_CODE_SUCCESS; + } + + smaDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, taskInfo, suid); + + qSetStreamInput(taskInfo, pMsg, inputType); + while (1) { + SSDataBlock *output = NULL; + uint64_t ts; + if (qExecTask(taskInfo, &output, &ts) < 0) { + ASSERT(false); + } + if (!output) { + break; + } + if (!pResult) { + pResult = taosArrayInit(0, sizeof(SSDataBlock)); + if (!pResult) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + + taosArrayPush(pResult, output); + } + + if (taosArrayGetSize(pResult) > 0) { + blockDebugShowData(pResult); + STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); + SSubmitReq *pReq = NULL; + if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), uid, suid) != 0) { + taosArrayDestroy(pResult); + return TSDB_CODE_FAILED; + } + if (tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) { + taosArrayDestroy(pResult); + taosMemoryFreeClear(pReq); + return TSDB_CODE_FAILED; + } + taosMemoryFreeClear(pReq); + } else { + smaWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno)); + } + + taosArrayDestroy(pResult); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + if (!pEnv) { + // only applicable when rsma env exists + return TSDB_CODE_SUCCESS; + } + + ASSERT(uid != 0); // TODO: remove later + + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaInfo *pRSmaInfo = NULL; + + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + smaDebug("vgId:%d no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); + return TSDB_CODE_SUCCESS; + } + if (!pRSmaInfo->taskInfo[0]) { + smaDebug("vgId:%d no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); + return TSDB_CODE_SUCCESS; + } + + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + // TODO: use the proper schema instead of 0, and cache STSchema in cache + STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, 0); + if (!pTSchema) { + terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; + return TSDB_CODE_FAILED; + } + tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1); + tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2); + taosMemoryFree(pTSchema); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + if (!pEnv) { + // only applicable when rsma env exists + return TSDB_CODE_SUCCESS; + } + + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + STbUidStore uidStore = {0}; + tdFetchSubmitReqSuids(pMsg, &uidStore); + + if (uidStore.suid != 0) { + tdExecuteRSma(pSma, pMsg, inputType, uidStore.suid, uidStore.uid); + + void *pIter = taosHashIterate(uidStore.uidHash, NULL); + while (pIter) { + tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + tdExecuteRSma(pSma, pMsg, inputType, *pTbSuid, 0); + pIter = taosHashIterate(uidStore.uidHash, pIter); + } + + tdUidStoreDestory(&uidStore); + } + } + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/vnode/src/sma/smaTDBImpl.c b/source/dnode/vnode/src/sma/smaTDBImpl.c new file mode 100644 index 0000000000..821ec44aa5 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTDBImpl.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include "sma.h" + +int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { + int ret = 0; + + if (path == NULL) return -1; + + ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + + if (ret != 0) { + smaError("failed to create tsdb db env, ret = %d", ret); + return -1; + } + + return 0; +} + +int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } + +static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { + const SSmaKey *pKey1 = (const SSmaKey *)arg1; + const SSmaKey *pKey2 = (const SSmaKey *)arg2; + + ASSERT(len1 == len2 && len1 == sizeof(SSmaKey)); + + if (pKey1->skey < pKey2->skey) { + return -1; + } else if (pKey1->skey > pKey2->skey) { + return 1; + } + if (pKey1->groupId < pKey2->groupId) { + return -1; + } else if (pKey1->groupId > pKey2->groupId) { + return 1; + } + + return 0; +} + +static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { + int ret; + tdb_cmpr_fn_t compFunc; + + // Create a database + compFunc = tdSmaKeyCmpr; + ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); + + return 0; +} + +static int32_t smaCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); } + +int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) { + // TEnv is shared by a group of SDBFile + if (!pEnv || !pDBF) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } + + // Open DBF + if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + smaCloseDBDb(pDBF->pDB); + return -1; + } + + return 0; +} + +int32_t smaCloseDBF(SDBFile *pDBF) { + int32_t ret = 0; + if (pDBF->pDB) { + ret = smaCloseDBDb(pDBF->pDB); + pDBF->pDB = NULL; + } + taosMemoryFreeClear(pDBF->path); + return ret; +} + +int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { + int32_t ret; + + ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + if (ret < 0) { + smaError("failed to create insert sma data into db, ret = %d", ret); + return -1; + } + + return 0; +} + +void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) { + void *pVal = NULL; + int ret; + + ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + + if (ret < 0) { + smaError("failed to get sma data from db, ret = %d", ret); + return NULL; + } + + ASSERT(*valLen >= 0); + + // TODO: lock? + // TODO: Would the key/value be destoryed during return the data? + // TODO: How about the key is updated while value length is changed? The original value buffer would be freed + // automatically? + + return pVal; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c new file mode 100644 index 0000000000..b04885c5f0 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -0,0 +1,1103 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +#undef _TEST_SMA_PRINT_DEBUG_LOG_ +#define SMA_STORAGE_TSDB_DAYS 30 +#define SMA_STORAGE_TSDB_TIMES 10 +#define SMA_STORAGE_SPLIT_HOURS 24 +#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8 +#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds + +#define SMA_STATE_ITEM_HASH_SLOT 32 + + +typedef struct { + SSma *pSma; + SDBFile dFile; + const SArray *pDataBlocks; // sma data + int32_t interval; // interval with the precision of DB +} STSmaWriteH; + +typedef struct { + int32_t iter; + int32_t fid; +} SmaFsIter; + +typedef struct { + STsdb *pTsdb; + SSma *pSma; + SDBFile dFile; + int32_t interval; // interval with the precision of DB + int32_t blockSize; // size of SMA block item + int8_t storageLevel; + int8_t days; + SmaFsIter smaFsIter; +} STSmaReadH; + +typedef enum { + SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma + SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma +} ESmaStorageLevel; + + +// static func + +static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); +static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); +static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, + int8_t intervalUnit); +static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit); +static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH); +static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel); +static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); +static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); +static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); +static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn); +// expired window +static int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version); +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, + int64_t version); +static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); +static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); + +// read data +// TODO: This is the basic params, and should wrap the params to a queryHandle. +static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); + +// implementation + +/** + * @brief + * + * @param pSmaH + * @param pSma + * @param interval + * @param intervalUnit + * @return int32_t + */ +static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) { + pSmaH->pSma = pSma; + pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); + pSmaH->storageLevel = tdGetSmaStorageLevel(interval, intervalUnit); + pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Init of tSma FS + * + * @param pReadH + * @param indexUid + * @param skey + * @return int32_t + */ +static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { + SSma *pSma = pSmaH->pSma; + + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision)); + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); + pSmaH->dFile.path = strdup(tSmaFile); + pSmaH->smaFsIter.iter = 0; + pSmaH->smaFsIter.fid = fid; + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Set and open tSma file if it has key locates in queryWin. + * + * @param pReadH + * @param param + * @param queryWin + * @return true + * @return false + */ +static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { + // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; + // int32_t nSmaFs = taosArrayGetSize(smaFs); + + smaCloseDBF(&pReadH->dFile); + +#if 0 + while (pReadH->smaFsIter.iter < nSmaFs) { + void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); + if (pSmaFile) { // match(indexName, queryWindow) + // TODO: select the file by index_name ... + pReadH->dFile = pSmaFile; + ++pReadH->smaFsIter.iter; + break; + } + ++pReadH->smaFsIter.iter; + } + + if (pReadH->pDFile) { + tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); + return true; + } +#endif + + return false; +} + + +/** + * @brief Approximate value for week/month/year. + * + * @param interval + * @param intervalUnit + * @param precision + * @param adjusted Interval already adjusted according to DB precision + * @return int64_t + */ +static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { + if (adjusted) { + return interval; + } + + switch (intervalUnit) { + case TIME_UNIT_YEAR: // approximate value + interval *= 365 * 86400 * 1e3; + break; + case TIME_UNIT_MONTH: // approximate value + interval *= 30 * 86400 * 1e3; + break; + case TIME_UNIT_WEEK: // approximate value + interval *= 7 * 86400 * 1e3; + break; + case TIME_UNIT_DAY: // the interval for tSma calculation must <= day + interval *= 86400 * 1e3; + break; + case TIME_UNIT_HOUR: + interval *= 3600 * 1e3; + break; + case TIME_UNIT_MINUTE: + interval *= 60 * 1e3; + break; + case TIME_UNIT_SECOND: + interval *= 1e3; + break; + default: + break; + } + + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval / 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second + return interval / 1e6; + } else { // ms + return interval; + } + break; + case TSDB_TIME_PRECISION_MICRO: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval / 1e3; + } else { // ms + return interval * 1e3; + } + break; + case TSDB_TIME_PRECISION_NANO: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval * 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval; + } else { // ms + return interval * 1e6; + } + break; + default: // ms + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval / 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval / 1e6; + } else { // ms + return interval; + } + break; + } + return interval; +} + + +static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, + int8_t intervalUnit) { + pSmaH->pSma = pSma; + pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); + pSmaH->pDataBlocks = pDataBlocks; + pSmaH->dFile.fid = SMA_IVLD_FID; + return TSDB_CODE_SUCCESS; +} + +static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) { + if (pSmaH) { + smaCloseDBF(&pSmaH->dFile); + } +} + +static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { + SSma *pSma = pSmaH->pSma; + ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); + + pSmaH->dFile.fid = fid; + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); + pSmaH->dFile.path = strdup(tSmaFile); + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief + * + * @param pSma + * @param interval Interval calculated by DB's precision + * @param storageLevel + * @return int32_t + */ +static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) { + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + int32_t daysPerFile = pCfg->days; + + if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { + int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); + daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS; + } + + return daysPerFile; +} + +/** + * @brief Judge the tSma storage level + * + * @param interval + * @param intervalUnit + * @return int32_t + */ +static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { + // TODO: configurable for SMA_STORAGE_SPLIT_HOURS? + switch (intervalUnit) { + case TIME_UNIT_HOUR: + if (interval < SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_MINUTE: + if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_SECOND: + if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_MILLISECOND: + if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_MICROSECOND: + if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_NANOSECOND: + if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + default: + break; + } + return SMA_STORAGE_LEVEL_TSDB; +} + +/** + * @brief Insert/Update Time-range-wise SMA data. + * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. + * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files. + * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The + * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d). + * - The destination file of one data block for some interval is determined by its start TS key. + * + * @param pSma + * @param msg + * @return int32_t + */ +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + const SArray *pDataBlocks = (const SArray *)msg; + + // TODO: destroy SSDataBlocks(msg) + + // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus + // the sma data would arrive ahead of the update-expired-window msg. + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + if (!pDataBlocks) { + terrno = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + return terrno; + } + + if (taosArrayGetSize(pDataBlocks) <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + smaWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SSmaStatItem *pItem = NULL; + + tdRefSmaStat(pSma, pStat); + + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); + } + + if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + + STSma *pTSma = pItem->pTSma; + STSmaWriteH tSmaH = {0}; + + if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) { + return TSDB_CODE_FAILED; + } + + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + } + + // Step 1: Judge the storage level and days + int32_t storageLevel = tdGetSmaStorageLevel(pTSma->interval, pTSma->intervalUnit); + int32_t daysPerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel); + + char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId + char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? + void *pDataBuf = NULL; + int32_t sz = taosArrayGetSize(pDataBlocks); + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + int32_t rowSize = pDataBlock->info.rowSize; + int64_t groupId = pDataBlock->info.groupId; + for (int32_t j = 0; j < rows; ++j) { + printf("|"); + TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval + void *pSmaKey = &smaKey; + bool isStartKey = false; + + int32_t tlen = 0; // reset the len + pDataBuf = &dataBuf; // reset the buf + for (int32_t k = 0; k < colNum; ++k) { + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + if (!isStartKey) { + isStartKey = true; + skey = *(TSKEY *)var; + printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); + tdEncodeTSmaKey(groupId, skey, &pSmaKey); + } else { + printf(" %" PRIi64 " |", *(int64_t *)var); + tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); + break; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + printf(" %15d |", *(uint8_t *)var); + tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); + break; + case TSDB_DATA_TYPE_TINYINT: + printf(" %15d |", *(int8_t *)var); + tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf(" %15d |", *(int16_t *)var); + tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf(" %15d |", *(uint16_t *)var); + tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); + break; + case TSDB_DATA_TYPE_INT: + printf(" %15d |", *(int32_t *)var); + tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); + break; + case TSDB_DATA_TYPE_FLOAT: + printf(" %15f |", *(float *)var); + tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); + break; + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t *)var); + tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); + break; + case TSDB_DATA_TYPE_BIGINT: + printf(" %15ld |", *(int64_t *)var); + tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf(" %15lf |", *(double *)var); + tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); + case TSDB_DATA_TYPE_UBIGINT: + printf(" %15lu |", *(uint64_t *)var); + tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); + break; + case TSDB_DATA_TYPE_NCHAR: { + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARBINARY: + // TODO: add binary/varbinary + TASSERT(0); + default: + printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + TASSERT(0); + break; + } + } + // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { + if (tlen > 0) { + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision)); + + // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index + // file + // - Set and open the DFile or the B+Tree file + // TODO: tsdbStartTSmaCommit(); + if (fid != tSmaH.dFile.fid) { + if (tSmaH.dFile.fid != SMA_IVLD_FID) { + tdSmaEndCommit(pEnv); + smaCloseDBF(&tSmaH.dFile); + } + tdSetTSmaDataFile(&tSmaH, indexUid, fid); + if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { + smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + tdSmaBeginCommit(pEnv); + } + + if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { + smaWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 + " since %s", + SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno)); + tdSmaEndCommit(pEnv); + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, + SMA_VID(pSma), indexUid, skey, groupId); + // TODO:tsdbEndTSmaCommit(); + + // Step 3: reset the SSmaStat + tdResetExpiredWindow(pSma, pStat, indexUid, skey); + } else { + smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, + SMA_VID(pSma), skey, tlen, indexUid); + } + + printf("\n"); + } + } + tdSmaEndCommit(pEnv); // TODO: not commit for every insert + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} + +int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) { + smaWarn("vgId:%d drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + +/** + * @brief Insert TSma data blocks to DB File build by B+Tree + * + * @param pSmaH + * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) + * @param keyLen + * @param pData + * @param dataLen + * @return int32_t + */ +static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn) { + SDBFile *pDBFile = &pSmaH->dFile; + + // TODO: insert sma data blocks into B+Tree(TDB) + if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { + smaWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", + SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", + SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); + +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ + uint32_t valueSize = 0; + void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); + ASSERT(data != NULL); + for (uint32_t v = 0; v < valueSize; v += 8) { + smaWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); + } +#endif + return TSDB_CODE_SUCCESS; +} + +/** + * @brief When sma data received from stream computing, make the relative expired window valid. + * + * @param pSma + * @param pStat + * @param indexUid + * @param skey + * @return int32_t + */ +static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { + SSmaStatItem *pItem = NULL; + + tdRefSmaStat(pSma, pStat); + + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); + } + if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { + // pItem resides in hash buffer all the time unless drop sma index + // TODO: multithread protect + if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { + // error handling + tdUnRefSmaStat(pSma, pStat); + smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), + skey, indexUid); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), + skey, indexUid); + // TODO: use a standalone interface to received state upate notification from stream computing module. + /** + * @brief state + * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. + * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. + * when batch data caculation not finised) + * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. + */ + pItem->state = TSDB_SMA_STAT_OK; + } else { + // error handling + tdUnRefSmaStat(pSma, pStat); + smaWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); + return TSDB_CODE_FAILED; + } + + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Drop tSma data and local cache + * - insert/query reference + * @param pSma + * @param msg + * @return int32_t + */ +static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { + SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); + + // clear local cache + if (pEnv) { + smaDebug("vgId:%d drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); + + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { + if (tdSmaStatIsDropped(pItem)) { + smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + + tdWLockSmaEnv(pEnv); + if (tdSmaStatIsDropped(pItem)) { + tdUnLockSmaEnv(pEnv); + smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + tdSmaStatSetDropped(pItem); + tdUnLockSmaEnv(pEnv); + + int32_t nSleep = 0; + int32_t refVal = INT32_MAX; + while (true) { + if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { + smaDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + break; + } + smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + taosSsleep(1); + if (++nSleep > SMA_DROP_EXPIRED_TIME) { + smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, + refVal); + break; + }; + } + + tdFreeSmaStatItem(pItem); + smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); + } + } + // clear sma data files + // TODO: + return TSDB_CODE_SUCCESS; +} + +/** + * @brief + * + * @param pSma Return the data between queryWin and fill the pData. + * @param pData + * @param indexUid + * @param pQuerySKey + * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. + * @return int32_t + */ +static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { + SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); + SSmaStat *pStat = NULL; + + if (!pEnv) { + terrno = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + pStat = SMA_ENV_STAT(pEnv); + + tdRefSmaStat(pSma, pStat); + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { + // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if + // it's NULL. + tdUnRefSmaStat(pSma, pStat); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_FAILED; + } + +#if 0 + int32_t nQueryWin = taosArrayGetSize(pQuerySKey); + for (int32_t n = 0; n < nQueryWin; ++n) { + TSKEY skey = taosArrayGet(pQuerySKey, n); + if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { + // TODO: mark this window as expired. + } + } +#endif + +#if 1 + int8_t smaStat = 0; + if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query + tdUnRefSmaStat(pSma, pStat); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, + tstrerror(terrno), smaStat); + return TSDB_CODE_FAILED; + } + + if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { + // TODO: mark this window as expired. + smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), + querySKey, indexUid); + } else { + smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + indexUid); + } + + STSma *pTSma = pItem->pTSma; +#endif + +#if 1 + STSmaReadH tReadH = {0}; + tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit); + smaCloseDBF(&tReadH.dFile); + + tdUnRefSmaStat(pSma, pStat); + + tdInitTSmaFile(&tReadH, indexUid, querySKey); + if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { + smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + + char smaKey[SMA_KEY_LEN] = {0}; + void *pSmaKey = &smaKey; + int64_t queryGroupId = 1; + tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); + + smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), + tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); + + void *result = NULL; + int32_t valueSize = 0; + if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { + smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", + SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); + smaCloseDBF(&tReadH.dFile); + return TSDB_CODE_FAILED; + } + #endif + +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ + for (uint32_t v = 0; v < valueSize; v += 8) { + smaWarn("vgId:%d get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); + } +#endif + taosMemoryFreeClear(result); // TODO: fill the result to output + +#if 0 + int32_t nResult = 0; + int64_t lastKey = 0; + + while (true) { + if (nResult >= nMaxResult) { + break; + } + + // set and open the file according to the STSma param + if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) { + char bTree[100] = "\0"; + while (strncmp(bTree, "has more nodes", 100) == 0) { + if (nResult >= nMaxResult) { + break; + } + // tdGetDataFromBTree(bTree, queryWin, lastKey) + // fill the pData + ++nResult; + } + } + } +#endif + // read data from file and fill the result + smaCloseDBF(&tReadH.dFile); + return TSDB_CODE_SUCCESS; +} + +int32_t tdProcessTSmaCreate(SSma *pSma, char *pMsg) { + #if 0 + SSmaCfg vCreateSmaReq = {0}; + if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + smaWarn("vgId:%d tsma create msg received but deserialize failed since %s", SMA_VID(pSma), terrstr(terrno)); + return -1; + } + + smaDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", SMA_VID(pSma), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid); + + // record current timezone of server side + vCreateSmaReq.tSma.timezoneInt = tsTimezone; + + if (metaCreateTSma(SMA_META(pSma), &vCreateSmaReq) < 0) { + // TODO: handle error + smaWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", SMA_VID(pSma), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno)); + tdDestroyTSma(&vCreateSmaReq.tSma); + return -1; + } + + tdTSmaAdd(pSma, 1); + + tdDestroyTSma(&vCreateSmaReq.tSma); + // TODO: return directly or go on follow steps? +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdDropTSma(SSma *pSma, char *pMsg) { +#if 0 + SVDropTSmaReq vDropSmaReq = {0}; + if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + // TODO: send msg to stream computing to drop tSma + // if ((send msg to stream computing) < 0) { + // tdDestroyTSma(&vCreateSmaReq); + // return -1; + // } + // + + if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + tdTSmaSub(pSma, 1); +#endif + + // TODO: return directly or go on follow steps? + return TSDB_CODE_SUCCESS; +} + +static SSmaStatItem *tdNewSmaStatItem(int8_t state) { + SSmaStatItem *pItem = NULL; + + pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); + if (!pItem) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pItem->state = state; + pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), + true, HASH_ENTRY_LOCK); + if (!pItem->expiredWindows) { + taosMemoryFreeClear(pItem); + return NULL; + } + + return pItem; +} + +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, + int64_t version) { + SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); + if (!pItem) { + // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later + pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state + if (!pItem) { + // Response to stream computing: OOM + // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. + return TSDB_CODE_FAILED; + } + + // cache smaMeta + STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid, true); + if (!pTSma) { + terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + smaWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, + tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pTSma = pTSma; + + if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { + // If error occurs during put smaStatItem, free the resources of pItem + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + return TSDB_CODE_FAILED; + } + } else if (!(pItem = *(SSmaStatItem **)pItem)) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { + // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would + // tell query module to query raw TS data. + // N.B. + // 1) It is assumed to be extemely little probability event of fail to taosHashPut. + // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired + // windows failed to put into hash table. + taosHashCleanup(pItem->expiredWindows); + taosMemoryFreeClear(pItem->pTSma); + taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); + smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, + winSKey); + return TSDB_CODE_FAILED; + } + + smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, + winSKey); + return TSDB_CODE_SUCCESS; +} + + + +/** + * @brief Update expired window according to msg from stream computing module. + * + * @param pSma + * @param msg SSubmitReq + * @return int32_t + */ +int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) { + // no time-range-sma, just return success + if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { + smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma)); + return TSDB_CODE_SUCCESS; + } + + if (!SMA_META(pSma)) { + terrno = TSDB_CODE_INVALID_PTR; + smaError("vgId:%d update expire window failed since no meta ptr", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) { + smaError("vgId:%d init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + // Firstly, assume that tSma can only be created on super table/normal table. + // getActiveTimeWindow + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); + + TASSERT(pEnv && pStat && pItemsHash); + + // basic procedure + // TODO: optimization + tdRefSmaStat(pSma, pStat); + + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SInterval interval = {0}; + TSKEY lastWinSKey = INT64_MIN; + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { + return TSDB_CODE_FAILED; + } + + while (true) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (!pBlock) break; + + STSmaWrapper *pSW = NULL; + STSma *pTSma = NULL; + + SSubmitBlkIter blkIter = {0}; + if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { + pSW = tdFreeTSmaWrapper(pSW); + break; + } + + while (true) { + STSRow *row = tGetSubmitBlkNext(&blkIter); + if (!row) { + tdFreeTSmaWrapper(pSW); + break; + } + if (!pSW || (pTSma->tableUid != pBlock->suid)) { + if (pSW) { + pSW = tdFreeTSmaWrapper(pSW); + } + if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), pBlock->suid))) { + break; + } + if ((pSW->number) <= 0 || !pSW->tSma) { + pSW = tdFreeTSmaWrapper(pSW); + break; + } + + pTSma = pSW->tSma; + + interval.interval = pTSma->interval; + interval.intervalUnit = pTSma->intervalUnit; + interval.offset = pTSma->offset; + interval.precision = SMA_TSDB_CFG(pSma)->precision; + interval.sliding = pTSma->sliding; + interval.slidingUnit = pTSma->slidingUnit; + } + + TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); + + if (lastWinSKey != winSKey) { + lastWinSKey = winSKey; + if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + } else { + smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", + SMA_VID(pSma), pTSma->indexUid, winSKey); + } + } + } + + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} + + +int32_t tdUpdateExpireWindow(SSma *pSma, SSubmitReq *pMsg, int64_t version) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) { + smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + +int32_t tdGetTSmaData(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) { + smaWarn("vgId:%d get tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + + diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ac8a271a4f..c69b3de05d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -234,7 +234,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (msgType != TDMT_VND_SUBMIT) return 0; // make sure msgType == TDMT_VND_SUBMIT - if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) { + if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) { return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 585ef63531..844cfc094b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -16,6 +16,8 @@ #include "tsdb.h" int tsdbBegin(STsdb *pTsdb) { + if (!pTsdb) return 0; + STsdbMemTable *pMem; if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 52b466d0f6..6dfd73158e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -37,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); // static int tsdbProcessExpiredFS(STsdb *pRepo); // static int tsdbCreateMeta(STsdb *pRepo); -static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]); +static void tsdbGetRootDir(int repoid, const char* dir, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, dir); } -static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]); +static void tsdbGetDataDir(int repoid, const char* dir, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, dir); } // For backward compatibility @@ -591,7 +591,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), - TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]); + pRepo->dir, tsdbTxnFname[ftype]); } static int tsdbOpenFSFromCurrent(STsdb *pRepo) { @@ -721,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir); + tsdbGetRootDir(REPO_ID(pRepo), pRepo->dir, rootDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); @@ -755,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); @@ -803,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { regex_t regex; STsdbFS *pfs = REPO_FS(pRepo); - tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir); // Resource allocation and init regcomp(®ex, pattern, REG_EXTENDED); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 7f024786de..04be2a48de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -23,14 +23,6 @@ static const char *TSDB_FNAME_SUFFIX[] = { "smal", // TSDB_FILE_SMAL "", // TSDB_FILE_MAX "meta", // TSDB_FILE_META - "tsma", // TSDB_FILE_TSMA - "rsma", // TSDB_FILE_RSMA -}; - -const char *TSDB_LEVEL_DNAME[] = { - "tsdb", - "rsma1", - "rsma2", }; static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname); @@ -51,7 +43,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.fver = tsdbGetDFSVersion(ftype); - tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname); + tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, pRepo->dir, fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 807ee95b03..8e689fc185 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -15,100 +15,17 @@ #include "tsdb.h" -#define TSDB_OPEN_RSMA_IMPL(v, l) \ - do { \ - SRetention *r = VND_RETENTIONS(v)[0]; \ - if (RETENTION_VALID(r)) { \ - return tsdbOpenImpl((v), type, &VND_RSMA##l(v), VNODE_RSMA##l##_DIR, TSDB_RETENTION_L##l); \ - } \ - } while (0) +static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg); -#define TSDB_SET_KEEP_CFG(l) \ - do { \ - SRetention *r = &pCfg->retentions[l]; \ - pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \ - pKeepCfg->keep0 = pKeepCfg->keep2; \ - pKeepCfg->keep1 = pKeepCfg->keep2; \ - pKeepCfg->days = tsdbEvalDays(r, pCfg->precision); \ - } while (0) -#define RETENTION_DAYS_SPLIT_RATIO 10 -#define RETENTION_DAYS_SPLIT_MIN 1 -#define RETENTION_DAYS_SPLIT_MAX 30 +// implementation -static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type); -static int32_t tsdbEvalDays(SRetention *r, int8_t precision); -static int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level); - -int tsdbOpen(SVnode *pVnode, int8_t type) { - switch (type) { - case TSDB_TYPE_TSDB: - return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); - case TSDB_TYPE_TSMA: - ASSERT(0); - break; - case TSDB_TYPE_RSMA_L0: - TSDB_OPEN_RSMA_IMPL(pVnode, 0); - break; - case TSDB_TYPE_RSMA_L1: - TSDB_OPEN_RSMA_IMPL(pVnode, 1); - break; - case TSDB_TYPE_RSMA_L2: - TSDB_OPEN_RSMA_IMPL(pVnode, 2); - break; - default: - ASSERT(0); - break; - } - return 0; -} - -static int32_t tsdbEvalDays(SRetention *r, int8_t precision) { - int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY); - int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY); - - int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO; - if (days <= RETENTION_DAYS_SPLIT_MIN) { - days = RETENTION_DAYS_SPLIT_MIN; - if (days < freqDays) { - days = freqDays + 1; - } - } else { - if (days > RETENTION_DAYS_SPLIT_MAX) { - days = RETENTION_DAYS_SPLIT_MAX; - } - if (days < freqDays) { - days = freqDays + 1; - } - } - return days * 1440; -} - -static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type) { +static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg) { pKeepCfg->precision = pCfg->precision; - switch (type) { - case TSDB_TYPE_TSDB: - pKeepCfg->days = pCfg->days; - pKeepCfg->keep0 = pCfg->keep0; - pKeepCfg->keep1 = pCfg->keep1; - pKeepCfg->keep2 = pCfg->keep2; - break; - case TSDB_TYPE_TSMA: - ASSERT(0); - break; - case TSDB_TYPE_RSMA_L0: - TSDB_SET_KEEP_CFG(0); - break; - case TSDB_TYPE_RSMA_L1: - TSDB_SET_KEEP_CFG(1); - break; - case TSDB_TYPE_RSMA_L2: - TSDB_SET_KEEP_CFG(2); - break; - default: - ASSERT(0); - break; - } + pKeepCfg->days = pCfg->days; + pKeepCfg->keep0 = pCfg->keep0; + pKeepCfg->keep1 = pCfg->keep1; + pKeepCfg->keep2 = pCfg->keep2; return 0; } @@ -116,18 +33,16 @@ static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t typ * @brief * * @param pVnode - * @param type * @param ppTsdb * @param dir - * @param level retention level * @return int */ -int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) { +int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg) { STsdb *pTsdb = NULL; int slen = 0; *ppTsdb = NULL; - slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + 3; + slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + TSDB_DATA_DIR_LEN + 3; // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); @@ -136,13 +51,18 @@ int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *di return -1; } + ASSERT(strlen(dir) < TSDB_DATA_DIR_LEN); + memcpy(pTsdb->dir, dir, strlen(dir)); pTsdb->path = (char *)&pTsdb[1]; sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir); pTsdb->pVnode = pVnode; - pTsdb->level = level; pTsdb->repoLocked = false; taosThreadMutexInit(&pTsdb->mutex, NULL); - tsdbSetKeepCfg(REPO_KEEP_CFG(pTsdb), REPO_CFG(pTsdb), type); + if (!pKeepCfg) { + tsdbSetKeepCfg(&pTsdb->keepCfg, &pVnode->config.tsdbCfg); + } else { + memcpy(&pTsdb->keepCfg, pKeepCfg, sizeof(STsdbKeepCfg)); + } pTsdb->fs = tsdbNewFS(REPO_KEEP_CFG(pTsdb)); // create dir (TODO: use tfsMkdir) @@ -163,12 +83,13 @@ _err: return -1; } -int tsdbClose(STsdb *pTsdb) { - if (pTsdb) { +int tsdbClose(STsdb **pTsdb) { + if (*pTsdb) { // TODO: destroy mem/imem - tsdbCloseFS(pTsdb); - tsdbFreeFS(pTsdb->fs); - taosMemoryFree(pTsdb); + taosThreadMutexDestroy(&(*pTsdb)->mutex); + tsdbCloseFS(*pTsdb); + tsdbFreeFS((*pTsdb)->fs); + taosMemoryFreeClear(*pTsdb); } return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index e7bee3342a..b4fbd01c63 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -47,7 +47,7 @@ int vnodeBegin(SVnode *pVnode) { } // begin tsdb - if (vnodeIsRollup(pVnode)) { + if (pVnode->pSma) { if (tsdbBegin(VND_RSMA0(pVnode)) < 0) { vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 7476da2a0f..d44e30988d 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -96,26 +96,15 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { } // open tsdb - if (vnodeIsRollup(pVnode)) { - if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L0) < 0) { - vError("vgId:%d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + if (!vnodeIsRollup(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_TYPE_TSDB) < 0) { + vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } - if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L1) < 0) { - vError("vgId:%d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } - - if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L2) < 0) { - vError("vgId:%d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } - } else { - if (tsdbOpen(pVnode, TSDB_TYPE_TSDB) < 0) { - vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + // open sma + if (smaOpen(pVnode)) { + vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; } // open wal @@ -161,10 +150,10 @@ _err: if (pVnode->pQuery) vnodeQueryClose(pVnode); if (pVnode->pTq) tqClose(pVnode->pTq); if (pVnode->pWal) walClose(pVnode->pWal); - if (pVnode->pTsdb) tsdbClose(pVnode->pTsdb); + if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pMeta) metaClose(pVnode->pMeta); - tsdbClose(VND_RSMA1(pVnode)); - tsdbClose(VND_RSMA2(pVnode)); + if (pVnode->pSma) smaClose(pVnode->pSma); + tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); return NULL; @@ -177,9 +166,8 @@ void vnodeClose(SVnode *pVnode) { vnodeQueryClose(pVnode); walClose(pVnode->pWal); tqClose(pVnode->pTq); - tsdbClose(VND_TSDB(pVnode)); - tsdbClose(VND_RSMA1(pVnode)); - tsdbClose(VND_RSMA2(pVnode)); + if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); + smaClose(pVnode->pSma); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); // destroy handle diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4f76bc5386..6e3b46bf91 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -22,6 +22,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { #if 0 @@ -86,10 +87,8 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg case TDMT_VND_DROP_TABLE: if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; - case TDMT_VND_CREATE_SMA: { // timeRangeSMA - if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - // TODO - } + case TDMT_VND_CREATE_SMA: { + if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; } break; /* TSDB */ case TDMT_VND_SUBMIT: @@ -195,7 +194,7 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO // blockDebugShowData(data); - tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); + tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { @@ -305,7 +304,7 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, goto _err; } - tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req, &pVnode->msgCb); + tdProcessRSmaCreate(pVnode->pSma, pVnode->pMeta, &req, &pVnode->msgCb); tDecoderClear(&coder); return 0; @@ -366,7 +365,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } } else { cRsp.code = TSDB_CODE_SUCCESS; - tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); + tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); } taosArrayPush(rsp.pArray, &cRsp); @@ -374,8 +373,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, tDecoderClear(&decoder); - tsdbUpdateTbUidList(pVnode->pTsdb, pStore); - tsdbUidStoreFree(pStore); + tdUpdateTbUidList(pVnode->pSma, pStore); + tdUidStoreFree(pStore); // prepare rsp SEncoder encoder = {0}; @@ -649,8 +648,39 @@ _exit: // TODO: refactor if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) && (pRsp->code == TSDB_CODE_SUCCESS)) { - tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); } return 0; } + +static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { + SVCreateTSmaReq req = {0}; + SDecoder coder; + + pRsp->msgType = TDMT_VND_CREATE_SMA_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + // decode and process req + tDecoderInit(&coder, pReq, len); + + if (tDecodeSVCreateTSmaReq(&coder, &req) < 0) { + pRsp->code = terrno; + goto _err; + } +#if 0 + if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) { + pRsp->code = terrno; + goto _err; + } +#endif + + tDecoderClear(&coder); + return 0; + +_err: + tDecoderClear(&coder); + return -1; +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6427e13ae9..1639abceba 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2949,6 +2949,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p } static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { + SEncoder encoder = {0}; + int32_t contLen = 0; SVDropTSmaReq dropSmaReq = {0}; strcpy(dropSmaReq.indexName, pStmt->indexName); @@ -2956,16 +2958,26 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } + + int32_t ret = 0; + tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret); + if (ret < 0) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA; - pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq); + pCxt->pCmdMsg->msgLen = contLen; pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } void* pBuf = pCxt->pCmdMsg->pMsg; - tSerializeSVDropTSmaReq(&pBuf, &dropSmaReq); - + if (tEncodeSVDropTSmaReq(&encoder, &dropSmaReq) < 0) { + tEncoderClear(&encoder); + return TSDB_CODE_OUT_OF_MEMORY; + } + tEncoderClear(&encoder); return TSDB_CODE_SUCCESS; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a2b0648a4b..8a2a60d3e2 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -354,6 +354,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST, "Tsma already exists") // query diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 8531f2c8ea..9970ac24d7 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -94,6 +94,7 @@ int32_t tqDebugFlag = 135; int32_t fsDebugFlag = 135; int32_t metaDebugFlag = 135; int32_t fnDebugFlag = 135; +int32_t smaDebugFlag = 135; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; @@ -755,6 +756,7 @@ void taosSetAllDebugFlag(int32_t flag) { tqDebugFlag = flag; fsDebugFlag = flag; fnDebugFlag = flag; + smaDebugFlag = flag; uInfo("all debug flag are set to %d", flag); } From 9e7b43c836bb27f7e2c5a40e0d9373040caa8d4a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 15 May 2022 22:03:02 +0800 Subject: [PATCH 56/71] feat: sma code refactor --- source/common/src/tmsg.c | 5 ++--- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/inc/meta.h | 1 + source/dnode/vnode/src/meta/metaEntry.c | 6 ++++-- source/dnode/vnode/src/meta/metaOpen.c | 9 +++++++++ source/dnode/vnode/src/meta/metaSma.c | 15 +++++++++++---- source/dnode/vnode/src/vnd/vnodeSvr.c | 5 ++--- 7 files changed, 30 insertions(+), 13 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2b164bf6b2..76b3dc0078 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3586,12 +3586,11 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { } int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { - if (tDecodeI8(pCoder, &pSma->version) < 0) return -1; if (tDecodeI8(pCoder, &pSma->version) < 0) return -1; if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1; if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1; if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1; - if (tDecodeCStr(pCoder, (const char **)&pSma->indexName) < 0) return -1; + if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1; if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1; if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1; if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1; @@ -3645,7 +3644,7 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; - if (tDecodeCStr(pCoder, (const char**)&pReq->indexName) < 0) return -1; + if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1; tEndDecode(pCoder); return 0; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index be131148fe..a2be393eb2 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -192,7 +192,7 @@ struct SMetaEntry { SSchemaWrapper schema; } ntbEntry; struct { - STSmaWrapper tsma; + STSma *tsma; } smaEntry; }; }; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 9b52e3d854..c5ca806829 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -73,6 +73,7 @@ struct SMeta { TDB* pCtbIdx; TDB* pTagIdx; TDB* pTtlIdx; + TDB* pSmaIdx; SMetaIdx* pIdx; }; diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 8ddf3419b5..dd36906b19 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -36,7 +36,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; } else if (pME->type == TSDB_TSMA_TABLE) { - if (tEncodeTSmaWrapper(pCoder, &pME->smaEntry.tsma) < 0) return -1; + if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; } else { ASSERT(0); } @@ -66,7 +66,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; - } else { + } else if (pME->type == TSDB_TSMA_TABLE) { + if (tDecodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; + } else { ASSERT(0); } diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index db47e794ba..3ce146904c 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -105,6 +105,13 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { goto _err; } + // open pSmaIdx + ret = tdbDbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); + if (ret < 0) { + metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + // open index if (metaOpenIdx(pMeta) < 0) { metaError("vgId:%d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); @@ -118,6 +125,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { _err: if (pMeta->pIdx) metaCloseIdx(pMeta); + if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); @@ -134,6 +142,7 @@ _err: int metaClose(SMeta *pMeta) { if (pMeta) { if (pMeta->pIdx) metaCloseIdx(pMeta); + if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index e7d4d3e117..8ce7ea5895 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -23,8 +23,6 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { // The table uid should exists and be super table or normal table. // Check other cfg value - // TODO: add atomicity - SMetaEntry me = {0}; int kLen = 0; int vLen = 0; @@ -55,7 +53,7 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { me.type = TSDB_TSMA_TABLE; me.uid = pCfg->indexUid; me.name = pCfg->indexName; - // me.smaEntry = xx; + me.smaEntry.tsma = pCfg; if (metaHandleSmaEntry(pMeta, &me) < 0) goto _err; @@ -180,8 +178,15 @@ _err: return -1; } +static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); +} +static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { + SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; + return tdbDbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); +} static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { metaWLock(pMeta); @@ -190,7 +195,9 @@ static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { if (metaSaveSmaToDB(pMeta, pME) < 0) goto _err; // // update uid.idx - // if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err; + if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err; + + if (metaUpdateSmaIdx(pMeta, pME) < 0) goto _err; metaULock(pMeta); return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 6e3b46bf91..f638ac056a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -670,12 +670,11 @@ static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq pRsp->code = terrno; goto _err; } -#if 0 - if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) { + + if (metaCreateTSma(pVnode->pMeta, version, &req) < 0) { pRsp->code = terrno; goto _err; } -#endif tDecoderClear(&coder); return 0; From fa77a6d840f17d2821f3d3a5b7c7cf44d54c4c77 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 15 May 2022 22:08:30 +0800 Subject: [PATCH 57/71] feat: code optimization --- include/common/tmsg.h | 406 +++++++++++++++++++-------------------- source/common/src/tmsg.c | 2 +- 2 files changed, 204 insertions(+), 204 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 46dbb6ba6e..b54ea58c4e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2256,242 +2256,242 @@ static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) { return 0; } - typedef struct { - int idx; - } SMCreateFullTextReq; +typedef struct { + int idx; +} SMCreateFullTextReq; - int32_t tSerializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq); - int32_t tDeserializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq); - void tFreeSMCreateFullTextReq(SMCreateFullTextReq * pReq); +int32_t tSerializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq); +int32_t tDeserializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq); +void tFreeSMCreateFullTextReq(SMCreateFullTextReq* pReq); - typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; - int8_t igNotExists; - } SMDropFullTextReq; +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + int8_t igNotExists; +} SMDropFullTextReq; - int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); - int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); - typedef struct { - char indexFName[TSDB_INDEX_FNAME_LEN]; - } SUserIndexReq; +typedef struct { + char indexFName[TSDB_INDEX_FNAME_LEN]; +} SUserIndexReq; - int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); - int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); +int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); +int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); - typedef struct { - char dbFName[TSDB_DB_FNAME_LEN]; - char tblFName[TSDB_TABLE_FNAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; - char indexType[TSDB_INDEX_TYPE_LEN]; - char indexExts[TSDB_INDEX_EXTS_LEN]; - } SUserIndexRsp; +typedef struct { + char dbFName[TSDB_DB_FNAME_LEN]; + char tblFName[TSDB_TABLE_FNAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; + char indexType[TSDB_INDEX_TYPE_LEN]; + char indexExts[TSDB_INDEX_EXTS_LEN]; +} SUserIndexRsp; - int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp* pRsp); - int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp); +int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp* pRsp); +int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp); - typedef struct { - int8_t mqMsgType; - int32_t code; - int32_t epoch; - int64_t consumerId; - } SMqRspHead; +typedef struct { + int8_t mqMsgType; + int32_t code; + int32_t epoch; + int64_t consumerId; +} SMqRspHead; - typedef struct { - SMsgHead head; - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; - int8_t withTbName; - int32_t epoch; - uint64_t reqId; - int64_t consumerId; - int64_t waitTime; - int64_t currentOffset; - } SMqPollReq; +typedef struct { + SMsgHead head; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int8_t withTbName; + int32_t epoch; + uint64_t reqId; + int64_t consumerId; + int64_t waitTime; + int64_t currentOffset; +} SMqPollReq; - typedef struct { - int32_t vgId; - int64_t offset; - SEpSet epSet; - } SMqSubVgEp; +typedef struct { + int32_t vgId; + int64_t offset; + SEpSet epSet; +} SMqSubVgEp; - static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pVgEp->vgId); - tlen += taosEncodeFixedI64(buf, pVgEp->offset); - tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); - return tlen; +static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeFixedI64(buf, pVgEp->offset); + tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { + buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeFixedI64(buf, &pVgEp->offset); + buf = taosDecodeSEpSet(buf, &pVgEp->epSet); + return buf; +} + +typedef struct { + char topic[TSDB_TOPIC_FNAME_LEN]; + int8_t isSchemaAdaptive; + SArray* vgs; // SArray + SSchemaWrapper schema; +} SMqSubTopicEp; + +static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pTopicEp->topic); + tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive); + int32_t sz = taosArrayGetSize(pTopicEp->vgs); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); + tlen += tEncodeSMqSubVgEp(buf, pVgEp); } + tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); + return tlen; +} - static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { - buf = taosDecodeFixedI32(buf, &pVgEp->vgId); - buf = taosDecodeFixedI64(buf, &pVgEp->offset); - buf = taosDecodeSEpSet(buf, &pVgEp->epSet); - return buf; +static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { + buf = taosDecodeStringTo(buf, pTopicEp->topic); + buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); + if (pTopicEp->vgs == NULL) { + return NULL; } - - typedef struct { - char topic[TSDB_TOPIC_FNAME_LEN]; - int8_t isSchemaAdaptive; - SArray* vgs; // SArray - SSchemaWrapper schema; - } SMqSubTopicEp; - - static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pTopicEp->topic); - tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive); - int32_t sz = taosArrayGetSize(pTopicEp->vgs); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); - tlen += tEncodeSMqSubVgEp(buf, pVgEp); - } - tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); - return tlen; + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp vgEp; + buf = tDecodeSMqSubVgEp(buf, &vgEp); + taosArrayPush(pTopicEp->vgs, &vgEp); } + buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); + return buf; +} - static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { - buf = taosDecodeStringTo(buf, pTopicEp->topic); - buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); - if (pTopicEp->vgs == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp vgEp; - buf = tDecodeSMqSubVgEp(buf, &vgEp); - taosArrayPush(pTopicEp->vgs, &vgEp); - } - buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); - return buf; - } +static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { + // taosMemoryFree(pSubTopicEp->schema.pSchema); + taosArrayDestroy(pSubTopicEp->vgs); +} - static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp * pSubTopicEp) { - // taosMemoryFree(pSubTopicEp->schema.pSchema); - taosArrayDestroy(pSubTopicEp->vgs); - } +typedef struct { + SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t blockNum; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + SArray* blockDataLen; // SArray + SArray* blockData; // SArray + SArray* blockTbName; // SArray + SArray* blockSchema; // SArray + SArray* blockTags; // SArray + SArray* blockTagSchema; // SArray +} SMqDataBlkRsp; - typedef struct { - SMqRspHead head; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t blockNum; - int8_t withTbName; - int8_t withSchema; - int8_t withTag; - SArray* blockDataLen; // SArray - SArray* blockData; // SArray - SArray* blockTbName; // SArray - SArray* blockSchema; // SArray - SArray* blockTags; // SArray - SArray* blockTagSchema; // SArray - } SMqDataBlkRsp; +static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); + tlen += taosEncodeFixedI32(buf, pRsp->blockNum); + if (pRsp->blockNum != 0) { + tlen += taosEncodeFixedI8(buf, pRsp->withTbName); + tlen += taosEncodeFixedI8(buf, pRsp->withSchema); + tlen += taosEncodeFixedI8(buf, pRsp->withTag); - static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); - tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); - tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); - tlen += taosEncodeFixedI32(buf, pRsp->blockNum); - if (pRsp->blockNum != 0) { - tlen += taosEncodeFixedI8(buf, pRsp->withTbName); - tlen += taosEncodeFixedI8(buf, pRsp->withSchema); - tlen += taosEncodeFixedI8(buf, pRsp->withTag); - - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); - void* data = taosArrayGetP(pRsp->blockData, i); - tlen += taosEncodeFixedI32(buf, bLen); - tlen += taosEncodeBinary(buf, data, bLen); - if (pRsp->withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i); - tlen += taosEncodeSSchemaWrapper(buf, pSW); - } - if (pRsp->withTbName) { - char* tbName = (char*)taosArrayGetP(pRsp->blockTbName, i); - tlen += taosEncodeString(buf, tbName); - } + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); + void* data = taosArrayGetP(pRsp->blockData, i); + tlen += taosEncodeFixedI32(buf, bLen); + tlen += taosEncodeBinary(buf, data, bLen); + if (pRsp->withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i); + tlen += taosEncodeSSchemaWrapper(buf, pSW); + } + if (pRsp->withTbName) { + char* tbName = (char*)taosArrayGetP(pRsp->blockTbName, i); + tlen += taosEncodeString(buf, tbName); } } - return tlen; } + return tlen; +} - static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); - buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); - buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); - buf = taosDecodeFixedI32(buf, &pRsp->blockNum); - pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); - if (pRsp->blockNum != 0) { - buf = taosDecodeFixedI8(buf, &pRsp->withTbName); - buf = taosDecodeFixedI8(buf, &pRsp->withSchema); - buf = taosDecodeFixedI8(buf, &pRsp->withTag); +static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) { + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); + buf = taosDecodeFixedI32(buf, &pRsp->blockNum); + pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); + if (pRsp->blockNum != 0) { + buf = taosDecodeFixedI8(buf, &pRsp->withTbName); + buf = taosDecodeFixedI8(buf, &pRsp->withSchema); + buf = taosDecodeFixedI8(buf, &pRsp->withTag); - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = 0; - void* data = NULL; - buf = taosDecodeFixedI32(buf, &bLen); - buf = taosDecodeBinary(buf, &data, bLen); - taosArrayPush(pRsp->blockDataLen, &bLen); - taosArrayPush(pRsp->blockData, &data); - if (pRsp->withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); - buf = taosDecodeSSchemaWrapper(buf, pSW); - taosArrayPush(pRsp->blockSchema, &pSW); - } - if (pRsp->withTbName) { - char* name = NULL; - buf = taosDecodeString(buf, &name); - taosArrayPush(pRsp->blockTbName, &name); - } + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = 0; + void* data = NULL; + buf = taosDecodeFixedI32(buf, &bLen); + buf = taosDecodeBinary(buf, &data, bLen); + taosArrayPush(pRsp->blockDataLen, &bLen); + taosArrayPush(pRsp->blockData, &data); + if (pRsp->withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); + buf = taosDecodeSSchemaWrapper(buf, pSW); + taosArrayPush(pRsp->blockSchema, &pSW); + } + if (pRsp->withTbName) { + char* name = NULL; + buf = taosDecodeString(buf, &name); + taosArrayPush(pRsp->blockTbName, &name); } } - return (void*)buf; } + return (void*)buf; +} - typedef struct { - SMqRspHead head; - char cgroup[TSDB_CGROUP_LEN]; - SArray* topics; // SArray - } SMqAskEpRsp; +typedef struct { + SMqRspHead head; + char cgroup[TSDB_CGROUP_LEN]; + SArray* topics; // SArray +} SMqAskEpRsp; - static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pRsp) { - int32_t tlen = 0; - // tlen += taosEncodeString(buf, pRsp->cgroup); - int32_t sz = taosArrayGetSize(pRsp->topics); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubTopicEp* pVgEp = (SMqSubTopicEp*)taosArrayGet(pRsp->topics, i); - tlen += tEncodeSMqSubTopicEp(buf, pVgEp); - } - return tlen; +static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pRsp) { + int32_t tlen = 0; + // tlen += taosEncodeString(buf, pRsp->cgroup); + int32_t sz = taosArrayGetSize(pRsp->topics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubTopicEp* pVgEp = (SMqSubTopicEp*)taosArrayGet(pRsp->topics, i); + tlen += tEncodeSMqSubTopicEp(buf, pVgEp); } + return tlen; +} - static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) { - // buf = taosDecodeStringTo(buf, pRsp->cgroup); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); - if (pRsp->topics == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubTopicEp topicEp; - buf = tDecodeSMqSubTopicEp(buf, &topicEp); - taosArrayPush(pRsp->topics, &topicEp); - } - return buf; +static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) { + // buf = taosDecodeStringTo(buf, pRsp->cgroup); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); + if (pRsp->topics == NULL) { + return NULL; } + for (int32_t i = 0; i < sz; i++) { + SMqSubTopicEp topicEp; + buf = tDecodeSMqSubTopicEp(buf, &topicEp); + taosArrayPush(pRsp->topics, &topicEp); + } + return buf; +} - static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp * pRsp) { - taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); - } +static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { + taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); +} typedef struct { void* data; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 76b3dc0078..b954edcbfa 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3581,7 +3581,7 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (pSma->tagsFilterLen > 0) { if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1; } - + return 0; } From f77f91026492ea75124691337ee4383bd60e44a7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 16 May 2022 01:48:49 +0800 Subject: [PATCH 58/71] refactor(stream) --- include/libs/stream/tstream.h | 5 +- source/dnode/mnode/impl/src/mndScheduler.c | 5 + source/dnode/vnode/src/tq/tq.c | 56 +++++++- source/libs/stream/src/tstream.c | 152 ++++++++++++++++++++- 4 files changed, 212 insertions(+), 6 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 796fee4f89..4460327b88 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -40,6 +40,7 @@ enum { TASK_INPUT_STATUS__BLOCKED, TASK_INPUT_STATUS__RECOVER, TASK_INPUT_STATUS__STOP, + TASK_INPUT_STATUS__FAILED, }; enum { @@ -234,7 +235,9 @@ struct SStreamTask { int8_t outputStatus; STaosQueue* inputQ; + STaosQall* inputQAll; STaosQueue* outputQ; + STaosQall* outputQAll; // application storage void* ahandle; @@ -282,7 +285,7 @@ int32_t streamDequeueOutput(SStreamTask* pTask, void** output); int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId); -int32_t streamTaskExecNew(SStreamTask* pTask); +int32_t streamTaskRun(SStreamTask* pTask); int32_t streamTaskHandleInput(SStreamTask* pTask, void* data); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 824f031004..22a5f37334 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -194,6 +194,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p // source pTask->sourceType = TASK_SOURCE__MERGE; + pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; // exec pTask->execType = TASK_EXEC__NONE; @@ -235,6 +236,7 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); // source pTask->sourceType = TASK_SOURCE__MERGE; + pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; // exec pTask->execType = TASK_EXEC__NONE; @@ -309,6 +311,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SStreamTask* pTask = tNewSStreamTask(pStream->uid); // source part pTask->sourceType = TASK_SOURCE__SCAN; + pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; // sink part if (level == 0) { @@ -372,6 +375,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // source part, currently only support multi source pTask->sourceType = TASK_SOURCE__PIPE; + pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; // sink part pTask->sinkType = TASK_SINK__NONE; @@ -459,6 +463,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // source part pTask->sourceType = TASK_SOURCE__MERGE; + pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; // sink part pTask->sinkType = TASK_SINK__NONE; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c69b3de05d..873db62dd8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -14,6 +14,7 @@ */ #include "tq.h" +#include "tqueue.h" int32_t tqInit() { // @@ -1032,6 +1033,59 @@ int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId) return 0; } +int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) { + void* pIter = NULL; + bool failed = false; + + SStreamDataSubmit* pSubmit = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); + if (pSubmit == NULL) { + failed = true; + } + pSubmit->dataRef = taosMemoryMalloc(sizeof(int32_t)); + if (pSubmit->dataRef == NULL) { + failed = true; + } + + pSubmit->type = STREAM_DATA_TYPE_SUBMIT_BLOCK; + pSubmit->sourceVer = ver; + pSubmit->sourceVg = pTq->pVnode->config.vgId; + pSubmit->data = pReq; + *pSubmit->dataRef = 1; + + while (1) { + pIter = taosHashIterate(pTq->pStreamTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = (SStreamTask*)pIter; + if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue; + + int8_t inputStatus = atomic_load_8(&pTask->inputStatus); + if (inputStatus == TASK_INPUT_STATUS__NORMAL) { + if (failed) { + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + continue; + } + + streamDataSubmitRefInc(pSubmit); + taosWriteQitem(pTask->inputQ, pSubmit); + + int8_t execStatus = atomic_load_8(&pTask->status); + if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { + // TODO dispatch task launch msg to fetch queue + } + + } else { + // blocked or stopped, do nothing + } + } + + if (!failed) { + streamDataSubmitRefDec(pSubmit); + return 0; + } else { + return -1; + } +} + int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) { SStreamTaskExecReq req = {0}; tDecodeSStreamTaskExecReq(msg, &req); @@ -1051,7 +1105,7 @@ int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) { // try exec int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); if (execStatus == TASK_STATUS__IDLE) { - if (streamTaskExecNew(pTask) < 0) { + if (streamTaskRun(pTask) < 0) { atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); goto FAIL; diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index 33147a0c0a..812874dafb 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -133,7 +133,144 @@ int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) { return inputStatus; } -int32_t streamTaskProcessTriggerReq(SStreamTask* pTask, SMsgCb* pMsgCb, char* msg, int32_t msgLen) { +int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { + void* exec = pTask->exec.runners[0].executor; + + // set input + if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT); + + qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK); + } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) { + SStreamDataBlock* pBlock = (SStreamDataBlock*)data; + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); + + SArray* blocks = pBlock->blocks; + qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK); + } + + // exec + while (1) { + SSDataBlock* output; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) break; + taosArrayPush(pRes, &output); + } + + // destroy + if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { + streamDataSubmitRefDec((SStreamDataSubmit*)data); + } else { + taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); + } + return 0; +} + +// TODO: handle version +int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) { + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) return -1; + while (1) { + int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); + void* exec = pTask->exec.runners[0].executor; + if (execStatus == TASK_STATUS__IDLE) { + // first run, from qall, handle failure from last exec + while (1) { + void* data = NULL; + taosGetQitem(pTask->inputQAll, &data); + if (data == NULL) break; + + streamTaskExecImpl(pTask, data, pRes); + + taosFreeQitem(data); + + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + resQ->type = STREAM_INPUT__DATA_BLOCK; + resQ->blocks = pRes; + taosWriteQitem(pTask->outputQ, resQ); + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) goto FAIL; + } + } + // second run, from inputQ + taosReadAllQitems(pTask->inputQ, pTask->inputQAll); + while (1) { + void* data = NULL; + taosGetQitem(pTask->inputQAll, &data); + if (data == NULL) break; + + streamTaskExecImpl(pTask, data, pRes); + + taosFreeQitem(data); + + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + resQ->type = STREAM_INPUT__DATA_BLOCK; + resQ->blocks = pRes; + taosWriteQitem(pTask->outputQ, resQ); + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) goto FAIL; + } + } + // set status closing + atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); + // third run, make sure all inputQ is cleared + taosReadAllQitems(pTask->inputQ, pTask->inputQAll); + while (1) { + void* data = NULL; + taosGetQitem(pTask->inputQAll, &data); + if (data == NULL) break; + + streamTaskExecImpl(pTask, data, pRes); + + taosFreeQitem(data); + + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + resQ->type = STREAM_INPUT__DATA_BLOCK; + resQ->blocks = pRes; + taosWriteQitem(pTask->outputQ, resQ); + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) goto FAIL; + } + } + // set status closing + atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); + // third run, make sure all inputQ is cleared + taosReadAllQitems(pTask->inputQ, pTask->inputQAll); + while (1) { + void* data = NULL; + taosGetQitem(pTask->inputQAll, &data); + if (data == NULL) break; + } + + atomic_store_8(&pTask->status, TASK_STATUS__IDLE); + break; + } else if (execStatus == TASK_STATUS__CLOSING) { + continue; + } else if (execStatus == TASK_STATUS__EXECUTING) { + break; + } else { + ASSERT(0); + } + } + return 0; +FAIL: + atomic_store_8(&pTask->status, TASK_STATUS__IDLE); + return -1; +} + +int32_t streamTaskDispatchDown(SStreamTask* pTask, SMsgCb* pMsgCb) { + // + return 0; +} + +int32_t streamTaskSink(SStreamTask* pTask) { // return 0; } @@ -156,8 +293,8 @@ int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDat // 2.3. closing: keep trying while (1) { int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); - void* exec = pTask->exec.runners[0].executor; if (execStatus == TASK_STATUS__IDLE) { + void* exec = pTask->exec.runners[0].executor; SArray* pRes = taosArrayInit(0, sizeof(void*)); const SArray* blocks = pBlock->blocks; qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK); @@ -278,7 +415,7 @@ int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, char* msg) { return 0; } -int32_t streamTaskExecNew(SStreamTask* pTask) { +int32_t streamTaskRun(SStreamTask* pTask) { SArray* pRes = NULL; if (pTask->execType == TASK_EXEC__PIPE || pTask->execType == TASK_EXEC__MERGE) { // TODO remove multi runner @@ -494,11 +631,16 @@ SStreamTask* tNewSStreamTask(int64_t streamId) { pTask->inputQ = taosOpenQueue(); pTask->outputQ = taosOpenQueue(); - if (pTask->inputQ == NULL || pTask->outputQ == NULL) goto FAIL; + pTask->inputQAll = taosAllocateQall(); + pTask->outputQAll = taosAllocateQall(); + if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL) + goto FAIL; return pTask; FAIL: if (pTask->inputQ) taosCloseQueue(pTask->inputQ); if (pTask->outputQ) taosCloseQueue(pTask->outputQ); + if (pTask->inputQAll) taosFreeQall(pTask->inputQAll); + if (pTask->outputQAll) taosFreeQall(pTask->outputQAll); if (pTask) taosMemoryFree(pTask); return NULL; } @@ -507,6 +649,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { /*if (tStartEncode(pEncoder) < 0) return -1;*/ if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; @@ -552,6 +695,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { /*if (tStartDecode(pDecoder) < 0) return -1;*/ if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; From 5b35fcacd6c91033e44f2babeb0c81d17bf84555 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 16 May 2022 07:47:56 +0800 Subject: [PATCH 59/71] fix: teardown udf functions handles --- include/libs/function/tudf.h | 2 + source/libs/executor/src/executorMain.c | 8 ++- source/libs/function/src/tudf.c | 75 ++++++++++++++++++------- 3 files changed, 61 insertions(+), 24 deletions(-) diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index b37dcd2b61..ca5079a0a8 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -121,6 +121,8 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output); + +int32_t teardownUdfs(); // end API to taosd and qworker //============================================================================================================================= // begin API to UDF writer. diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 3cc75a815d..0c464d9b43 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -21,13 +21,14 @@ #include "tcache.h" #include "tglobal.h" #include "tmsg.h" +#include "tudf.h" -#include "thash.h" -#include "executorimpl.h" #include "executor.h" +#include "executorimpl.h" +#include "query.h" +#include "thash.h" #include "tlosertree.h" #include "ttypes.h" -#include "query.h" typedef struct STaskMgmt { TdThreadMutex lock; @@ -156,6 +157,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0; pTaskInfo->totalRows += current; + teardownUdfs(); qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 35702eb71f..b5058a2d60 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -313,6 +313,7 @@ int64_t gUdfTaskSeqNum = 0; typedef struct SUdfcFuncStub { char udfName[TSDB_FUNC_NAME_LEN]; UdfcFuncHandle handle; + int32_t refCount; } SUdfcFuncStub; typedef struct SUdfcProxy { @@ -338,7 +339,7 @@ typedef struct SUdfcProxy { SUdfcProxy gUdfdProxy = {0}; -typedef struct SClientUdfUvSession { +typedef struct SUdfcUvSession { SUdfcProxy *udfc; int64_t severHandle; uv_pipe_t *udfUvPipe; @@ -346,7 +347,9 @@ typedef struct SClientUdfUvSession { int8_t outputType; int32_t outputLen; int32_t bufSize; -} SClientUdfUvSession; + + char udfName[TSDB_FUNC_NAME_LEN]; +} SUdfcUvSession; typedef struct SClientUvTaskNode { SUdfcProxy *udfc; @@ -369,7 +372,7 @@ typedef struct SClientUvTaskNode { typedef struct SClientUdfTask { int8_t type; - SClientUdfUvSession *session; + SUdfcUvSession *session; int32_t errCode; @@ -401,7 +404,7 @@ typedef struct SClientUvConn { uv_pipe_t *pipe; QUEUE taskQueue; SClientConnBuf readBuf; - SClientUdfUvSession *session; + SUdfcUvSession *session; } SClientUvConn; enum { @@ -825,11 +828,6 @@ void onUdfcPipeClose(uv_handle_t *handle) { taosMemoryFree(conn->readBuf.buf); taosMemoryFree(conn); taosMemoryFree((uv_pipe_t *) handle); - - //clear the udf handles cache TODO move to other thread - uv_mutex_lock(&gUdfdProxy.udfStubsMutex); - taosArrayClear(gUdfdProxy.udfStubs); - uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); } int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *uvTask) { @@ -1283,7 +1281,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { } SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask)); task->errCode = 0; - task->session = taosMemoryCalloc(1, sizeof(SClientUdfUvSession)); + task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession)); task->session->udfc = &gUdfdProxy; task->type = UDF_TASK_SETUP; @@ -1303,6 +1301,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { task->session->outputType = rsp->outputType; task->session->outputLen = rsp->outputLen; task->session->bufSize = rsp->bufSize; + strcpy(task->session->udfName, udfName); if (task->errCode != 0) { fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode) } else { @@ -1317,14 +1316,14 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, SSDataBlock* output, SUdfInterBuf *newState) { fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle); - SClientUdfUvSession *session = (SClientUdfUvSession *) handle; + SUdfcUvSession *session = (SUdfcUvSession *) handle; if (session->udfUvPipe == NULL) { fnError("No pipe to udfd"); return TSDB_CODE_UDF_PIPE_NO_PIPE; } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); task->errCode = 0; - task->session = (SClientUdfUvSession *) handle; + task->session = (SUdfcUvSession *) handle; task->type = UDF_TASK_CALL; SUdfCallRequest *req = &task->_call.req; @@ -1440,7 +1439,7 @@ int compareUdfcFuncSub(const void* elem1, const void* elem2) { return strcmp(stub1->udfName, stub2->udfName); } -int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) { +int32_t accquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) { int32_t code = 0; uv_mutex_lock(&gUdfdProxy.udfStubsMutex); SUdfcFuncStub key = {0}; @@ -1449,6 +1448,7 @@ int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) { if (foundStub != NULL) { uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); *pHandle = foundStub->handle; + ++foundStub->refCount; return 0; } *pHandle = NULL; @@ -1457,6 +1457,7 @@ int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) { SUdfcFuncStub stub = {0}; strcpy(stub.udfName, udfName); stub.handle = *pHandle; + ++stub.refCount; taosArrayPush(gUdfdProxy.udfStubs, &stub); taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub); } else { @@ -1467,13 +1468,25 @@ int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) { return code; } +void releaseUdfFuncHandle(char* udfName) { + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + SUdfcFuncStub key = {0}; + strcpy(key.udfName, udfName); + SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); + ASSERT(foundStub); + --foundStub->refCount; + ASSERT(foundStub->refCount>=0); + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); +} + int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) { UdfcFuncHandle handle = NULL; - int32_t code = setupUdf(udfName, &handle); + int32_t code = accquireUdfFuncHandle(udfName, &handle); if (code != 0) { return code; } code = doCallUdfScalarFunc(handle, input, numOfCols, output); + releaseUdfFuncHandle(udfName); return code; } @@ -1481,7 +1494,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, int32_t doTeardownUdf(UdfcFuncHandle handle) { fnInfo("tear down udf. udf func handle: %p", handle); - SClientUdfUvSession *session = (SClientUdfUvSession *) handle; + SUdfcUvSession *session = (SUdfcUvSession *) handle; if (session->udfUvPipe == NULL) { fnError("pipe to udfd does not exist"); return TSDB_CODE_UDF_PIPE_NO_PIPE; @@ -1511,7 +1524,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { //memory layout |---SUdfAggRes----|-----final result-----|---inter result----| typedef struct SUdfAggRes { - SClientUdfUvSession *session; + SUdfcUvSession *session; int8_t finalResNum; int8_t interResNum; char* finalResBuf; @@ -1532,11 +1545,11 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } UdfcFuncHandle handle; int32_t udfCode = 0; - if ((udfCode = setupUdf((char *)pCtx->udfName, &handle)) != 0) { + if ((udfCode = accquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode); return false; } - SClientUdfUvSession *session = (SClientUdfUvSession *)handle; + SUdfcUvSession *session = (SUdfcUvSession *)handle; SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; memset(udfRes, 0, envSize); @@ -1544,7 +1557,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - udfRes->session = (SClientUdfUvSession *)handle; + udfRes->session = (SUdfcUvSession *)handle; SUdfInterBuf buf = {0}; if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); @@ -1560,7 +1573,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t numOfCols = pInput->numOfInputCols; SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SClientUdfUvSession *session = udfRes->session; + SUdfcUvSession *session = udfRes->session; if (session == NULL) { return TSDB_CODE_UDF_NO_FUNC_HANDLE; } @@ -1615,7 +1628,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SClientUdfUvSession *session = udfRes->session; + SUdfcUvSession *session = udfRes->session; if (session == NULL) { return TSDB_CODE_UDF_NO_FUNC_HANDLE; } @@ -1644,5 +1657,25 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { // } int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); + releaseUdfFuncHandle(pCtx->udfName); return udfCallCode == 0 ? numOfResults : udfCallCode; +} + +int32_t teardownUdfs() { + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + int32_t i = 0; + SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); + while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) { + SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i); + if (stub->refCount == 0) { + doTeardownUdf(stub->handle); + } else { + taosArrayPush(udfStubs, stub); + } + ++i; + } + taosArrayDestroy(gUdfdProxy.udfStubs); + gUdfdProxy.udfStubs = udfStubs; + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); + return 0; } \ No newline at end of file From 8ce04a94f92cf8c47d0d0dd7c9ef3205d5172b3f Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 16 May 2022 09:06:08 +0800 Subject: [PATCH 60/71] fix: change function name for tearing down udf handles --- include/libs/function/tudf.h | 2 +- source/libs/executor/src/executorMain.c | 2 +- source/libs/function/src/tudf.c | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index ca5079a0a8..6a98138c6c 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -122,7 +122,7 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output); -int32_t teardownUdfs(); +int32_t cleanUpUdfs(); // end API to taosd and qworker //============================================================================================================================= // begin API to UDF writer. diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 0c464d9b43..354f4d8752 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -157,7 +157,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0; pTaskInfo->totalRows += current; - teardownUdfs(); + cleanUpUdfs(); qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index b5058a2d60..0414841ba4 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1661,7 +1661,8 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { return udfCallCode == 0 ? numOfResults : udfCallCode; } -int32_t teardownUdfs() { +int32_t cleanUpUdfs() { + fnInfo("clean up udf unused function handles"); uv_mutex_lock(&gUdfdProxy.udfStubsMutex); int32_t i = 0; SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); From 4ae7a167494af94b30704e506461e2344ea6840a Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 16 May 2022 09:24:35 +0800 Subject: [PATCH 61/71] fix: teardown udf optimization --- source/libs/function/src/tudf.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 0414841ba4..4841e05267 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -25,8 +25,6 @@ #include "functionMgt.h" //TODO: add unit test -//TODO: include all global variable under context struct - typedef struct SUdfdData { bool startCalled; bool needCleanUp; @@ -1275,7 +1273,6 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { } int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { - fnInfo("udfc setup udf. udfName: %s", udfName); if (gUdfdProxy.udfcState != UDFC_STATE_READY) { return TSDB_CODE_UDF_INVALID_STATE; } @@ -1305,7 +1302,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { if (task->errCode != 0) { fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode) } else { - fnInfo("sucessfully setup udf func handle. handle: %p", task->session); + fnInfo("sucessfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session); *funcHandle = task->session; } int32_t err = task->errCode; @@ -1490,13 +1487,11 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, return code; } -//TODO: when to teardown udf. teardown udf is not called int32_t doTeardownUdf(UdfcFuncHandle handle) { - fnInfo("tear down udf. udf func handle: %p", handle); - SUdfcUvSession *session = (SUdfcUvSession *) handle; + if (session->udfUvPipe == NULL) { - fnError("pipe to udfd does not exist"); + fnError("tear down udf. pipe to udfd does not exist. udf name: %s", session->udfName); return TSDB_CODE_UDF_PIPE_NO_PIPE; } @@ -1511,7 +1506,6 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { udfcRunUdfUvTask(task, UV_TASK_REQ_RSP); SUdfTeardownResponse *rsp = &task->_teardown.rsp; - int32_t err = task->errCode; udfcRunUdfUvTask(task, UV_TASK_DISCONNECT); @@ -1519,6 +1513,8 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { taosMemoryFree(task->session); taosMemoryFree(task); + fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle); + return err; } @@ -1651,26 +1647,22 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; } -// int32_t code = doTeardownUdf(session); -// if (code != 0) { -// fnError("udfAggFinalize error. doTeardownUdf step. udf code: %d", code); -// } - int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); releaseUdfFuncHandle(pCtx->udfName); return udfCallCode == 0 ? numOfResults : udfCallCode; } int32_t cleanUpUdfs() { - fnInfo("clean up udf unused function handles"); uv_mutex_lock(&gUdfdProxy.udfStubsMutex); int32_t i = 0; SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) { SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i); if (stub->refCount == 0) { + fnInfo("tear down udf. udf name: %s, handle: %p", stub->udfName, stub->handle); doTeardownUdf(stub->handle); } else { + fnInfo("udf still in use. udf name: %s, ref count: %d, handle: %p", stub->udfName, stub->refCount, stub->handle); taosArrayPush(udfStubs, stub); } ++i; From 5c1cfd2cf9b5849cdc051f2f136b5f360f2194a5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 16 May 2022 09:45:14 +0800 Subject: [PATCH 62/71] fix(rpc): fix regArg mem leak --- source/libs/transport/src/transSrv.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index fc840691b6..72ab250709 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -137,7 +137,9 @@ static void destroySmsg(SSrvMsg* smsg); // check whether already read complete packet static SSrvConn* createConn(void* hThrd); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); -static int reallocConnRefHandle(SSrvConn* conn); +static void destroyConnRegArg(SSrvConn* conn); + +static int reallocConnRefHandle(SSrvConn* conn); static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd); static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd); @@ -429,6 +431,8 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { if (smsg->type == Release) { pHead->msgType = 0; pConn->status = ConnNormal; + + destroyConnRegArg(pConn); transUnrefSrvHandle(pConn); } else { pHead->msgType = pMsg->msgType; @@ -800,6 +804,12 @@ static void destroyConn(SSrvConn* conn, bool clear) { // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } } +static void destroyConnRegArg(SSrvConn* conn) { + if (conn->regArg.init == 1) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } +} static int reallocConnRefHandle(SSrvConn* conn) { uvReleaseExHandle(conn->refId); uvRemoveExHandle(conn->refId); @@ -827,16 +837,9 @@ static void uvDestroyConn(uv_handle_t* handle) { // uv_timer_stop(&conn->pTimer); transQueueDestroy(&conn->srvMsgs); - if (conn->regArg.init == 1) { - transFreeMsg(conn->regArg.msg.pCont); - conn->regArg.init = 0; - } QUEUE_REMOVE(&conn->queue); taosMemoryFree(conn->pTcp); - if (conn->regArg.init == 1) { - transFreeMsg(conn->regArg.msg.pCont); - conn->regArg.init = 0; - } + destroyConnRegArg(conn); taosMemoryFree(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { From f2858c76fa65564059e3290699111b2e74716827 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 16 May 2022 01:57:49 +0000 Subject: [PATCH 63/71] refact --- include/util/tencode.h | 1 - include/util/tfreelist.h | 60 ------------------------------- source/util/test/freelistTest.cpp | 17 --------- 3 files changed, 78 deletions(-) delete mode 100644 include/util/tfreelist.h delete mode 100644 source/util/test/freelistTest.cpp diff --git a/include/util/tencode.h b/include/util/tencode.h index 2a43d7934f..e49429f865 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -18,7 +18,6 @@ #include "tcoding.h" #include "tlist.h" -// #include "tfreelist.h" #ifdef __cplusplus extern "C" { diff --git a/include/util/tfreelist.h b/include/util/tfreelist.h deleted file mode 100644 index e9b5ca5fca..0000000000 --- a/include/util/tfreelist.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_FREELIST_H_ -#define _TD_UTIL_FREELIST_H_ - -#include "tlist.h" - -#ifdef __cplusplus -extern "C" { -#endif - -struct SFreeListNode { - TD_SLIST_NODE(SFreeListNode); - char payload[]; -}; - -typedef TD_SLIST(SFreeListNode) SFreeList; - -#define TFL_MALLOC(PTR, TYPE, SIZE, LIST) \ - do { \ - void *ptr = taosMemoryMalloc((SIZE) + sizeof(struct SFreeListNode)); \ - if (ptr) { \ - TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \ - ptr = ((struct SFreeListNode *)ptr)->payload; \ - (PTR) = (TYPE)(ptr); \ - }else{ \ - (PTR) = NULL; \ - } \ - }while(0); - -#define tFreeListInit(pFL) TD_SLIST_INIT(pFL) - -static FORCE_INLINE void tFreeListClear(SFreeList *pFL) { - struct SFreeListNode *pNode; - for (;;) { - pNode = TD_SLIST_HEAD(pFL); - if (pNode == NULL) break; - TD_SLIST_POP(pFL); - taosMemoryFree(pNode); - } -} - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_FREELIST_H_*/ \ No newline at end of file diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp deleted file mode 100644 index a445a16ad3..0000000000 --- a/source/util/test/freelistTest.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include - -#include "tfreelist.h" - -TEST(TD_UTIL_FREELIST_TEST, simple_test) { - SFreeList fl; - - tFreeListInit(&fl); - - for (size_t i = 0; i < 1000; i++) { - void *ptr = NULL; - TFL_MALLOC(ptr, void*, 1024, &fl); - GTEST_ASSERT_NE(ptr, nullptr); - } - - tFreeListClear(&fl); -} From 2df67d9133e00f867708845bcd6a710a6800ef37 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 16 May 2022 11:10:05 +0800 Subject: [PATCH 64/71] fix(query): extract correct join condition from physical plan. --- source/libs/executor/src/joinoperator.c | 31 ++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 82bb435a2c..d7d6d96346 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -26,6 +26,7 @@ static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); static void destroyMergeJoinOperator(void* param, int32_t numOfOutput); +static void extractTimeCondition(SJoinOperatorInfo *Info, SLogicConditionNode* pLogicConditionNode); SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, @@ -38,22 +39,22 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t initResultSizeInfo(pOperator, 4096); - pInfo->pRes = pResBlock; - pOperator->name = "MergeJoinOperator"; + pInfo->pRes = pResBlock; + pOperator->name = "MergeJoinOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; pOperator->numOfExprs = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; if (nodeType(pOnCondition) == QUERY_NODE_OPERATOR) { SOperatorNode* pNode = (SOperatorNode*)pOnCondition; setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft); setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight); } else if (nodeType(pOnCondition) == QUERY_NODE_LOGIC_CONDITION) { - ASSERT(0); + extractTimeCondition(pInfo, (SLogicConditionNode*) pOnCondition); } pOperator->fpSet = @@ -182,3 +183,17 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { return (pRes->info.rows > 0) ? pRes : NULL; } + +static void extractTimeCondition(SJoinOperatorInfo *pInfo, SLogicConditionNode* pLogicConditionNode) { + int32_t len = LIST_LENGTH(pLogicConditionNode->pParameterList); + + for(int32_t i = 0; i < len; ++i) { + SNode* pNode = nodesListGetNode(pLogicConditionNode->pParameterList, i); + if (nodeType(pNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pn1 = (SOperatorNode*)pNode; + setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pn1->pLeft); + setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pn1->pRight); + break; + } + } +} From d2eeaf86c78df02a417e11fab9be1bf77e2a5bc2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 16 May 2022 03:54:05 +0000 Subject: [PATCH 65/71] alter table --- include/common/tmsg.h | 40 +++++++++++++++ source/common/src/tmsg.c | 108 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b54ea58c4e..24ed795a02 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1670,6 +1670,7 @@ typedef struct SVDropStbReq { int32_t tEncodeSVDropStbReq(SEncoder* pCoder, const SVDropStbReq* pReq); int32_t tDecodeSVDropStbReq(SDecoder* pCoder, SVDropStbReq* pReq); +// TDMT_VND_CREATE_TABLE ============== #define TD_CREATE_IF_NOT_EXISTS 0x1 typedef struct SVCreateTbReq { int32_t flags; @@ -1759,6 +1760,45 @@ typedef struct { int32_t tEncodeSVDropTbBatchRsp(SEncoder* pCoder, const SVDropTbBatchRsp* pRsp); int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp); +// TDMT_VND_ALTER_TABLE ===================== +typedef struct { + const char* tbName; + int8_t action; + // TSDB_ALTER_TABLE_ADD_COLUMN + int8_t type; + int32_t bytes; + const char* colAddName; + // TSDB_ALTER_TABLE_DROP_COLUMN + const char* colDropName; + // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES + const char* colModName; + int32_t colModBytes; + // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME + const char* colOldName; + const char* colNewName; + // TSDB_ALTER_TABLE_UPDATE_TAG_VAL + const char* tagName; + int8_t isNull; + uint32_t nTagVal; + const uint8_t* pTagVal; + // TSDB_ALTER_TABLE_UPDATE_OPTIONS + int8_t updateTTL; + int32_t newTTL; + int8_t updateComment; + const char* newComment; +} SVAlterTbReq; + +int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); +int32_t tDecodeSVAlterTbReq(SDecoder* pDecoder, SVAlterTbReq* pReq); + +typedef struct { + int32_t code; +} SVAlterTbRsp; + +int32_t tEncodeSVAlterTbRsp(SEncoder* pEncoder, const SVAlterTbRsp* pRsp); +int32_t tDecodeSVAlterTbRsp(SDecoder* pDecoder, SVAlterTbRsp* pRsp); +// ====================== + typedef struct { SMsgHead head; int64_t uid; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index b954edcbfa..e539aa3467 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4163,3 +4163,111 @@ void tFreeSSubmitRsp(SSubmitRsp *pRsp) { taosMemoryFree(pRsp); } + +int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + + if (tEncodeCStr(pEncoder, pReq->tbName) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->action) < 0) return -1; + switch (pReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + if (tEncodeI8(pEncoder, pReq->type) < 0) return -1; + if (tEncodeI32v(pEncoder, pReq->bytes) < 0) return -1; + if (tEncodeCStr(pEncoder, pReq->colAddName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_DROP_COLUMN: + if (tEncodeCStr(pEncoder, pReq->colDropName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + if (tEncodeCStr(pEncoder, pReq->colModName) < 0) return -1; + if (tEncodeI32v(pEncoder, pReq->colModBytes) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + if (tEncodeCStr(pEncoder, pReq->colOldName) < 0) return -1; + if (tEncodeCStr(pEncoder, pReq->colNewName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: + if (tEncodeCStr(pEncoder, pReq->tagName) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->isNull) < 0) return -1; + if (!pReq->isNull) { + if (tEncodeBinary(pEncoder, pReq->pTagVal, pReq->nTagVal) < 0) return -1; + } + break; + case TSDB_ALTER_TABLE_UPDATE_OPTIONS: + if (tEncodeI8(pEncoder, pReq->updateTTL) < 0) return -1; + if (pReq->updateTTL) { + if (tEncodeI32v(pEncoder, pReq->newTTL) < 0) return -1; + } + if (tEncodeI8(pEncoder, pReq->updateComment) < 0) return -1; + if (pReq->updateComment) { + if (tEncodeCStr(pEncoder, pReq->newComment) < 0) return -1; + } + break; + default: + break; + } + + tEndEncode(pEncoder); + return 0; +} + +int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + + if (tDecodeCStr(pDecoder, &pReq->tbName) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->action) < 0) return -1; + switch (pReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + if (tDecodeI8(pDecoder, &pReq->type) < 0) return -1; + if (tDecodeI32v(pDecoder, &pReq->bytes) < 0) return -1; + if (tDecodeCStr(pDecoder, &pReq->colAddName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_DROP_COLUMN: + if (tDecodeCStr(pDecoder, &pReq->colDropName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + if (tDecodeCStr(pDecoder, &pReq->colModName) < 0) return -1; + if (tDecodeI32v(pDecoder, &pReq->colModBytes) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + if (tDecodeCStr(pDecoder, &pReq->colOldName) < 0) return -1; + if (tDecodeCStr(pDecoder, &pReq->colNewName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: + if (tDecodeCStr(pDecoder, &pReq->tagName) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->isNull) < 0) return -1; + if (!pReq->isNull) { + if (tDecodeBinary(pDecoder, &pReq->pTagVal, &pReq->nTagVal) < 0) return -1; + } + break; + case TSDB_ALTER_TABLE_UPDATE_OPTIONS: + if (tDecodeI8(pDecoder, &pReq->updateTTL) < 0) return -1; + if (pReq->updateTTL) { + if (tDecodeI32v(pDecoder, &pReq->newTTL) < 0) return -1; + } + if (tDecodeI8(pDecoder, &pReq->updateComment) < 0) return -1; + if (pReq->updateComment) { + if (tDecodeCStr(pDecoder, &pReq->newComment) < 0) return -1; + } + break; + default: + break; + } + + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSVAlterTbRsp(SEncoder *pEncoder, const SVAlterTbRsp *pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->code) < 0) return -1; + tEndEncode(pEncoder); + return 0; +} + +int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->code) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} From 545de8cbe47a5019a4b6843b3e53345e426f672d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 16 May 2022 04:00:41 +0000 Subject: [PATCH 66/71] make it compile --- source/util/test/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 0d16a2129a..c1053ce7de 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -33,13 +33,13 @@ ENDIF() INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) -# freelistTest -add_executable(freelistTest "") -target_sources(freelistTest - PRIVATE - "freelistTest.cpp" -) -target_link_libraries(freelistTest os util gtest gtest_main) +# # freelistTest +# add_executable(freelistTest "") +# target_sources(freelistTest +# PRIVATE +# "freelistTest.cpp" +# ) +# target_link_libraries(freelistTest os util gtest gtest_main) # # encodeTest # add_executable(encodeTest "encodeTest.cpp") From 619eae3c4952c6d647ed23fc43866a1acbc55e89 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 16 May 2022 13:51:21 +0800 Subject: [PATCH 67/71] fix(query): merge updated data from cache. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 85 +++++++++----------------- 1 file changed, 28 insertions(+), 57 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 927babc26c..65bc6a9cc1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1271,7 +1271,6 @@ _error: static int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo); static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); -static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols); static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle); static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); @@ -1301,7 +1300,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* if ((ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || (!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { // do not load file block into buffer - int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; + int32_t step = ascScan ? 1 : -1; TSKEY maxKey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? (binfo.window.skey - step) : (binfo.window.ekey - step); @@ -1790,22 +1789,6 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa #endif } -static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) { - if (numOfRows == 0 || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - return; - } - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (numOfRows < pTsdbReadHandle->outputCapacity) { - int32_t emptySize = pTsdbReadHandle->outputCapacity - numOfRows; - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, - numOfRows * pColInfo->info.bytes); - } - } -} - static void getQualifiedRowsPos(STsdbReadHandle* pTsdbReadHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, int32_t* start, int32_t* end) { *start = -1; @@ -1891,9 +1874,6 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa cur->lastKey = tsArray[endPos] + step; cur->blockCompleted = true; - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); - // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases. pos = endPos + step; updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); @@ -1944,18 +1924,18 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows - 1] == pBlock->keyLast); + bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order); + int32_t step = ascScan ? 1 : -1; + // for search the endPos, so the order needs to reverse - int32_t order = (pTsdbReadHandle->order == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int32_t order = ascScan ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; - int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); - - STable* pTable = NULL; int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); + STimeWindow* pWin = &blockInfo.window; tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64 - " rows:%d, start:%d, end:%d, %s", - pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, + " rows:%d, start:%d, end:%d, %s", pTsdbReadHandle, pCheckInfo->tableId, pWin->skey, pWin->ekey, blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->idStr); // compared with the data from in-memory buffer, to generate the correct timestamp array list @@ -1986,20 +1966,16 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } TSKEY key = TD_ROW_KEY(row1); - if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + if ((key > pTsdbReadHandle->window.ekey && ascScan) || (key < pTsdbReadHandle->window.ekey && !ascScan)) { break; } - if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && - ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && - !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ascScan) || + ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ascScan)) { break; } - if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + if ((key < tsArray[pos] && ascScan) || (key > tsArray[pos] && !ascScan)) { if (rv1 != TD_ROW_SVER(row1)) { // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = TD_ROW_SVER(row1); @@ -2054,11 +2030,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } #endif if (TD_SUPPORT_UPDATE(pCfg->update)) { - if (lastKeyAppend != key) { - lastKeyAppend = key; - ++curRow; - } numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos); + lastKeyAppend = key; if (rv1 != TD_ROW_SVER(row1)) { // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); @@ -2068,9 +2041,10 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = TD_ROW_SVER(row2); } - numOfRows += - mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend); + + // still assign data into current row + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend); if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -2081,12 +2055,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf cur->mixBlock = true; moveToNextRowInMem(pCheckInfo); + ++curRow; + pos += step; } else { moveToNextRowInMem(pCheckInfo); } - } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - (key < tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + } else if ((key > tsArray[pos] && ascScan) || (key < tsArray[pos] && !ascScan)) { if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } @@ -2112,17 +2087,17 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf int32_t qstart = 0, qend = 0; getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend); - if ((lastKeyAppend != TSKEY_INITIAL_VAL) && - (lastKeyAppend != (ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qstart] : tsArray[qend]))) { + if ((lastKeyAppend != TSKEY_INITIAL_VAL) && (lastKeyAppend != (ascScan ? tsArray[qstart] : tsArray[qend]))) { ++curRow; } + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, qstart, qend); pos += (qend - qstart + 1) * step; if (numOfRows > 0) { curRow = numOfRows - 1; } - cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qend] : tsArray[qstart]; + cur->win.ekey = ascScan ? tsArray[qend] : tsArray[qstart]; cur->lastKey = cur->win.ekey + step; lastKeyAppend = cur->win.ekey; } @@ -2134,10 +2109,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && - ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && - !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && ascScan) || + ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && !ascScan)) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; @@ -2149,22 +2122,20 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end); pos += (end - start + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[end] : tsArray[start]; + cur->win.ekey = ascScan ? tsArray[end] : tsArray[start]; cur->lastKey = cur->win.ekey + step; cur->mixBlock = true; } } } - cur->blockCompleted = - (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); + cur->blockCompleted = (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ascScan) || + ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ascScan)); - if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (!ascScan) { TSWAP(cur->win.skey, cur->win.ekey); } - moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pTsdbReadHandle); From 956ab45a3043bd0c06a4aacbece898256d4eb8ab Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 16 May 2022 14:10:18 +0800 Subject: [PATCH 68/71] enh(sync): raft config change --- source/libs/sync/inc/syncIndexMgr.h | 1 + source/libs/sync/src/syncAppendEntries.c | 5 +++++ source/libs/sync/src/syncCommit.c | 5 +++++ source/libs/sync/src/syncIndexMgr.c | 7 +++++++ source/libs/sync/src/syncMain.c | 5 +++++ source/libs/sync/test/syncConfigChangeTest.cpp | 2 +- 6 files changed, 24 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 63f24b104f..0a6e2428fe 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -35,6 +35,7 @@ typedef struct SSyncIndexMgr { } SSyncIndexMgr; SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode); +void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode); void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 34fc7d4eb4..1a5d418e75 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -342,6 +342,11 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { ASSERT(ret == 0); syncNodeUpdateConfig(ths, &newSyncCfg); + if (ths->state == TAOS_SYNC_STATE_LEADER) { + syncNodeBecomeLeader(ths); + } else { + syncNodeBecomeFollower(ths); + } } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 5d7b5c1b21..0f17cf267e 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -120,6 +120,11 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { ASSERT(ret == 0); syncNodeUpdateConfig(pSyncNode, &newSyncCfg); + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + syncNodeBecomeLeader(pSyncNode); + } else { + syncNodeBecomeFollower(pSyncNode); + } } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index d33075054a..5809cedb90 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -31,6 +31,13 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { return pSyncIndexMgr; } +void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode) { + pSyncIndexMgr->replicas = &(pSyncNode->replicasId); + pSyncIndexMgr->replicaNum = pSyncNode->replicaNum; + pSyncIndexMgr->pSyncNode = pSyncNode; + syncIndexMgrClear(pSyncIndexMgr); +} + void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { if (pSyncIndexMgr != NULL) { taosMemoryFree(pSyncIndexMgr); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 34313baaa9..469682a7b5 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -905,6 +905,11 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig) { for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } + + syncIndexMgrUpdate(pSyncNode->pNextIndex, pSyncNode); + syncIndexMgrUpdate(pSyncNode->pMatchIndex, pSyncNode); + + syncNodeLog2("==syncNodeUpdateConfig==", pSyncNode); } SSyncNode* syncNodeAcquire(int64_t rid) { diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index 4455d25c59..9a2d9a6b34 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -220,7 +220,7 @@ int main(int argc, char** argv) { assert(pSyncNode != NULL); if (isConfigChange) { - configChange(rid, replicaNum, myIndex); + configChange(rid, 3, myIndex); } //--------------------------- From a47228aadd425dfdb480ab2b6765e45a8304db23 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 16 May 2022 14:24:25 +0800 Subject: [PATCH 69/71] fix: some problems of planner --- source/libs/nodes/src/nodesCodeFuncs.c | 35 +++++++++++++++++++ source/libs/parser/src/parAuthenticator.c | 7 ++-- source/libs/parser/test/parSelectTest.cpp | 10 ++++-- source/libs/planner/src/planOptimizer.c | 3 +- source/libs/planner/src/planSpliter.c | 4 +-- source/libs/planner/test/planOtherTest.cpp | 8 ++++- source/libs/planner/test/planStateTest.cpp | 4 +-- source/libs/planner/test/planSubqueryTest.cpp | 12 +++++-- source/libs/planner/test/planSysTbTest.cpp | 4 +-- 9 files changed, 71 insertions(+), 16 deletions(-) diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 57dfcaeddd..763ccbf7a0 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -584,6 +584,37 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkExchangeLogicPlanSrcGroupId = "SrcGroupId"; +static const char* jkExchangeLogicPlanSrcPrecision = "Precision"; + +static int32_t logicExchangeNodeToJson(const void* pObj, SJson* pJson) { + const SExchangeLogicNode* pNode = (const SExchangeLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcGroupId, pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcPrecision, pNode->precision); + } + + return code; +} + +static int32_t jsonToLogicExchangeNode(const SJson* pJson, void* pObj) { + SExchangeLogicNode* pNode = (SExchangeLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkExchangeLogicPlanSrcGroupId, &pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkExchangeLogicPlanSrcPrecision, &pNode->precision); + } + + return code; +} + static const char* jkFillLogicPlanMode = "Mode"; static const char* jkFillLogicPlanWStartTs = "WStartTs"; static const char* jkFillLogicPlanValues = "Values"; @@ -2987,6 +3018,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicProjectNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: break; + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return logicExchangeNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_FILL: return logicFillNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SORT: @@ -3083,6 +3116,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicScanNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PROJECT: return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return jsonToLogicExchangeNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_FILL: return jsonToLogicFillNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SORT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index ad1dca6857..250e7910d6 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -14,6 +14,7 @@ */ #include "catalog.h" +#include "cmdnodes.h" #include "parInt.h" typedef struct SAuthCxt { @@ -65,8 +66,8 @@ static int32_t authSetOperator(SAuthCxt* pCxt, SSetOperator* pSetOper) { return code; } -static int32_t authDropUser(SAuthCxt* pCxt, SDropUserReq* pStmt) { - if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->user, TSDB_DEFAULT_USER)) { +static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) { + if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->useName, TSDB_DEFAULT_USER)) { return TSDB_CODE_PAR_PERMISSION_DENIED; } return TSDB_CODE_SUCCESS; @@ -92,7 +93,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_ALTER_USER_STMT: break; case QUERY_NODE_DROP_USER_STMT: { - return authDropUser(pCxt, (SDropUserReq*)pStmt); + return authDropUser(pCxt, (SDropUserStmt*)pStmt); } case QUERY_NODE_USE_DATABASE_STMT: case QUERY_NODE_CREATE_DNODE_STMT: diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 821f480b20..47424d3138 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -235,11 +235,17 @@ TEST_F(ParserSelectTest, semanticError) { TEST_F(ParserSelectTest, setOperator) { useDb("root", "test"); - // run("SELECT * FROM t1 UNION ALL SELECT * FROM t1"); + run("SELECT * FROM t1 UNION ALL SELECT * FROM t1"); - // run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)"); + run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)"); run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); } +TEST_F(ParserSelectTest, informationSchema) { + useDb("root", "test"); + + run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'"); +} + } // namespace ParserTest diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index edac2a879f..e38c180ac6 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -392,7 +392,8 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, } static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { - if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) { + if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) || + TSDB_SYSTEM_TABLE == pScan->pMeta->tableType) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 1266e8ae4b..54bc24e8bb 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -303,7 +303,7 @@ static SLogicNode* unMatchByNode(SLogicNode* pNode) { } SNode* pChild; FOREACH(pChild, pNode->pChildren) { - SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild); + SLogicNode* pSplitNode = unMatchByNode((SLogicNode*)pChild); if (NULL != pSplitNode) { return pSplitNode; } @@ -318,7 +318,7 @@ static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan } pExchange->srcGroupId = pCxt->groupId; // pExchange->precision = pScan->pMeta->tableInfo.precision; - pExchange->node.pTargets = nodesCloneList(pAgg->node.pTargets); + pExchange->node.pTargets = nodesCloneList(pAgg->pGroupKeys); if (NULL == pExchange->node.pTargets) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index b70cb4d19a..67c09d706e 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -47,4 +47,10 @@ TEST_F(PlanOtherTest, explain) { run("explain analyze SELECT * FROM t1"); run("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); -} \ No newline at end of file +} + +TEST_F(PlanOtherTest, show) { + useDb("root", "test"); + + run("SHOW DATABASES"); +} diff --git a/source/libs/planner/test/planStateTest.cpp b/source/libs/planner/test/planStateTest.cpp index 83c9621916..9ff035e148 100644 --- a/source/libs/planner/test/planStateTest.cpp +++ b/source/libs/planner/test/planStateTest.cpp @@ -23,13 +23,13 @@ class PlanStateTest : public PlannerTestBase {}; TEST_F(PlanStateTest, basic) { useDb("root", "test"); - run("select count(*) from t1 state_window(c1)"); + run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1)"); } TEST_F(PlanStateTest, stateExpr) { useDb("root", "test"); - run("select count(*) from t1 state_window(c1 + 10)"); + run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1 + 10)"); } TEST_F(PlanStateTest, selectFunc) { diff --git a/source/libs/planner/test/planSubqueryTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp index 11e5e98052..6a7cb91bb9 100644 --- a/source/libs/planner/test/planSubqueryTest.cpp +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -25,11 +25,9 @@ TEST_F(PlanSubqeuryTest, basic) { if (0 == g_skipSql) { run("SELECT * FROM (SELECT * FROM t1)"); - - run("SELECT LAST(c1) FROM (SELECT * FROM t1)"); } - run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); + run("SELECT LAST(c1) FROM (SELECT * FROM t1)"); } TEST_F(PlanSubqeuryTest, doubleGroupBy) { @@ -39,3 +37,11 @@ TEST_F(PlanSubqeuryTest, doubleGroupBy) { "SELECT c1 + c3 a, c1 + COUNT(*) b FROM t1 WHERE c2 = 'abc' GROUP BY c1, c3) " "WHERE a > 100 GROUP BY b"); } + +TEST_F(PlanSubqeuryTest, withSetOperator) { + useDb("root", "test"); + + run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); + + run("SELECT c1 FROM (SELECT c1 FROM t1 UNION SELECT c1 FROM t1)"); +} diff --git a/source/libs/planner/test/planSysTbTest.cpp b/source/libs/planner/test/planSysTbTest.cpp index fff6bfcca4..e5c30030b3 100644 --- a/source/libs/planner/test/planSysTbTest.cpp +++ b/source/libs/planner/test/planSysTbTest.cpp @@ -27,8 +27,8 @@ TEST_F(PlanSysTableTest, show) { run("show stables"); } -TEST_F(PlanSysTableTest, information) { +TEST_F(PlanSysTableTest, informationSchema) { useDb("root", "information_schema"); - run("show tables"); + run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'"); } From c1b332f34d610417d3a60314036c8d1ab7a2207f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 16 May 2022 14:38:56 +0800 Subject: [PATCH 70/71] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 -- source/libs/qworker/src/qworker.c | 10 ++++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 65bc6a9cc1..467751e21c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2034,11 +2034,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf lastKeyAppend = key; if (rv1 != TD_ROW_SVER(row1)) { - // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = TD_ROW_SVER(row1); } if (row2 && rv2 != TD_ROW_SVER(row2)) { - // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = TD_ROW_SVER(row2); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 717958c033..6a25b23c6b 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -812,13 +812,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } if (ctx->rspCode) { - QW_TASK_ELOG("task already failed at phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode, - tstrerror(ctx->rspCode)); + QW_TASK_ELOG("task already failed at phase %s, code:%s", qwPhaseStr(phase), tstrerror(ctx->rspCode)); QW_ERR_JRET(ctx->rspCode); } _return: - if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); @@ -836,7 +834,11 @@ _return: QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code)); } - QW_TASK_DLOG("end to handle event at phase %s, code:%x - %s", qwPhaseStr(phase), code, tstrerror(code)); + if (code != TSDB_CODE_SUCCESS) { + QW_TASK_ELOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code)); + } else { + QW_TASK_DLOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code)); + } QW_RET(code); } From 8bad3305d209efaeb0e4a165b4105584ae97b7b3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 16 May 2022 14:52:39 +0800 Subject: [PATCH 71/71] fix(query): return error code when connecting to database failed. --- source/client/src/clientImpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 666e9e0ffa..100ec42899 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -518,8 +518,9 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t if (pRequest->code != TSDB_CODE_SUCCESS) { const char* errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code); - printf("failed to connect to server, reason: %s\n\n", errorMsg); + fprintf(stderr,"failed to connect to server, reason: %s\n\n", errorMsg); + terrno = pRequest->code; destroyRequest(pRequest); taos_close(pTscObj); pTscObj = NULL;