From 74f3a3f6de28f861a14593fe61ac03b440ac8f98 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 27 May 2022 14:24:54 +0800 Subject: [PATCH 01/57] other: disable the tsma case from ci temporarily --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 198b9007d7..155b675488 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -114,7 +114,7 @@ #./test.sh -f tsim/mnode/basic1.sim -m # --- sma -./test.sh -f tsim/sma/tsmaCreateInsertData.sim +#./test.sh -f tsim/sma/tsmaCreateInsertData.sim ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim # --- valgrind From 496ae4ca893508ac78f7aef4b2165ad50339e0a0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 31 May 2022 14:03:47 +0800 Subject: [PATCH 02/57] schedule based on qnode's load --- include/common/tmsg.h | 39 ++++++++-- include/common/tmsgdef.h | 1 + include/dnode/qnode/qnode.h | 14 ---- include/libs/executor/dataSinkMgt.h | 8 ++ include/libs/monitor/monitor.h | 5 ++ include/libs/qworker/qworker.h | 22 ++++-- source/client/inc/clientInt.h | 5 +- source/client/src/clientHb.c | 4 + source/client/src/clientImpl.c | 76 +++++++++++++++---- source/common/src/tmsg.c | 72 +++++++++++++++--- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 3 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 + source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 1 + source/dnode/mgmt/mgmt_qnode/src/qmHandle.c | 8 ++ source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 1 + source/dnode/mgmt/node_mgmt/inc/dmNodes.h | 1 + source/dnode/mgmt/node_mgmt/src/dmEnv.c | 1 + source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 14 ++++ source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/dnode/mgmt/node_util/inc/dmUtil.h | 5 +- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/inc/mndQnode.h | 6 ++ source/dnode/mnode/impl/src/mndDnode.c | 7 ++ source/dnode/mnode/impl/src/mndProfile.c | 4 + source/dnode/mnode/impl/src/mndQnode.c | 73 ++++++++++-------- source/dnode/qnode/src/qnode.c | 22 ++++-- source/libs/catalog/inc/catalogInt.h | 14 ++-- source/libs/catalog/src/catalog.c | 6 +- source/libs/catalog/src/ctgCache.c | 38 +++++----- source/libs/catalog/src/ctgRemote.c | 2 +- source/libs/catalog/src/ctgUtil.c | 8 +- source/libs/executor/inc/dataSinkInt.h | 2 + source/libs/executor/src/dataDispatcher.c | 24 +++++- source/libs/executor/src/dataSinkMgt.c | 14 ++++ source/libs/executor/src/scanoperator.c | 2 + source/libs/monitor/src/monMsg.c | 48 +++++++++++- source/libs/planner/src/planPhysiCreater.c | 12 ++- source/libs/qcom/src/querymsg.c | 2 +- source/libs/qworker/inc/qwInt.h | 36 +++++++-- source/libs/qworker/src/qwMsg.c | 23 ++++-- source/libs/qworker/src/qwUtil.c | 21 ++--- source/libs/qworker/src/qworker.c | 27 ++++++- source/libs/scheduler/src/schJob.c | 50 +++++++----- source/libs/scheduler/src/scheduler.c | 8 +- 45 files changed, 552 insertions(+), 184 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index faf4addb4b..b78937e964 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -791,19 +791,24 @@ typedef struct { int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq); int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq); +typedef struct SQueryNodeAddr { + int32_t nodeId; // vgId or qnodeId + SEpSet epSet; +} SQueryNodeAddr; + typedef struct { - SArray* addrsList; // SArray + SQueryNodeAddr addr; + uint64_t load; +} SQueryNodeLoad; + +typedef struct { + SArray* qnodeList; // SArray } SQnodeListRsp; int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); void tFreeSQnodeListRsp(SQnodeListRsp* pRsp); -typedef struct SQueryNodeAddr { - int32_t nodeId; // vgId or qnodeId - SEpSet epSet; -} SQueryNodeAddr; - typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; @@ -926,6 +931,21 @@ typedef struct { int32_t syncState; } SMnodeLoad; +typedef struct { + int32_t dnodeId; + int64_t numOfProcessedQuery; + int64_t numOfProcessedCQuery; + int64_t numOfProcessedFetch; + int64_t numOfProcessedDrop; + int64_t numOfProcessedHb; + int64_t cacheDataSize; + int64_t numOfQueryInQueue; + int64_t numOfFetchInQueue; + int64_t timeInQueryQueue; + int64_t timeInFetchQueue; +} SQnodeLoad; + + typedef struct { int32_t sver; // software version int64_t dnodeVer; // dnode table version in sdb @@ -937,6 +957,7 @@ typedef struct { int32_t numOfSupportVnodes; char dnodeEp[TSDB_EP_LEN]; SMnodeLoad mload; + SQnodeLoad qload; SClusterCfg clusterCfg; SArray* pVloads; // array of SVnodeLoad } SStatusReq; @@ -1937,6 +1958,7 @@ typedef struct { int8_t killConnection; int8_t align[3]; SEpSet epSet; + SArray *pQnodeList; } SQueryHbRspBasic; typedef struct { @@ -2016,7 +2038,10 @@ static FORCE_INLINE void tFreeClientKv(void* pKv) { static FORCE_INLINE void tFreeClientHbRsp(void* pRsp) { SClientHbRsp* rsp = (SClientHbRsp*)pRsp; - taosMemoryFreeClear(rsp->query); + if (rsp->query) { + taosArrayDestroy(rsp->query->pQnodeList); + taosMemoryFreeClear(rsp->query); + } if (rsp->info) taosArrayDestroyEx(rsp->info, tFreeClientKv); } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 51a15c1489..7858bc2246 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -253,6 +253,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MON_BM_INFO, "monitor-binfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_MM_LOAD, "monitor-mload", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MON_QM_LOAD, "monitor-qload", NULL, NULL) #if defined(TD_MSG_NUMBER_) TDMT_MAX diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 90a9529395..7d342c4ba1 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -25,20 +25,6 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SQnode SQnode; -typedef struct { - int64_t numOfProcessedQuery; - int64_t numOfProcessedCQuery; - int64_t numOfProcessedFetch; - int64_t numOfProcessedDrop; - int64_t memSizeInCache; - int64_t dataSizeSend; - int64_t dataSizeRecv; - int64_t numOfQueryInQueue; - int64_t numOfFetchInQueue; - int64_t waitTimeInQueryQUeue; - int64_t waitTimeInFetchQUeue; -} SQnodeLoad; - typedef struct { SMsgCb msgCb; } SQnodeOpt; diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 339743f153..2cc9caca6f 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -32,6 +32,10 @@ extern "C" { struct SDataSink; struct SSDataBlock; +typedef struct SDataSinkStat { + uint64_t cachedSize; +} SDataSinkStat; + typedef struct SDataSinkMgtCfg { uint32_t maxDataBlockNum; // todo: this should be numOfRows? uint32_t maxDataBlockNumPerQuery; @@ -62,6 +66,8 @@ typedef struct SOutputData { */ int32_t dsCreateDataSinker(const SDataSinkNode* pDataSink, DataSinkHandle* pHandle); +int32_t dsDataSinkGetCacheSize(SDataSinkStat *pStat); + /** * Put the result set returned by the executor into datasinker. * @param handle @@ -88,6 +94,8 @@ void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd); */ int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput); +int32_t dsGetCacheSize(DataSinkHandle handle, uint64_t *pSize); + /** * After dsGetStatus returns DS_NEED_SCHEDULE, the caller need to put this into the work queue. * @param ahandle diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 9d8cf61b06..39e8042b93 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -171,6 +171,7 @@ void tFreeSMonVmInfo(SMonVmInfo *pInfo); typedef struct { SMonSysInfo sys; SMonLogs log; + SQnodeLoad load; } SMonQmInfo; int32_t tSerializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo); @@ -210,6 +211,10 @@ typedef struct { int32_t tSerializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo); int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo); +int32_t tSerializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo); +int32_t tDeserializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo); + + typedef struct { const char *server; uint16_t port; diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 5942d00cb2..91cf975a56 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -22,7 +22,7 @@ extern "C" { #include "tmsgcb.h" #include "trpc.h" - +#include "executor.h" enum { NODE_TYPE_VNODE = 1, @@ -40,13 +40,19 @@ typedef struct SQWorkerCfg { } SQWorkerCfg; typedef struct { - uint64_t numOfStartTask; - uint64_t numOfStopTask; - uint64_t numOfRecvedFetch; - uint64_t numOfSentHb; - uint64_t numOfSentFetch; - uint64_t numOfTaskInQueue; + uint64_t cacheDataSize; + + uint64_t queryProcessed; + uint64_t cqueryProcessed; + uint64_t fetchProcessed; + uint64_t dropProcessed; + uint64_t hbProcessed; + + uint64_t numOfQueryInQueue; uint64_t numOfFetchInQueue; + uint64_t timeInQueryQueue; + uint64_t timeInFetchQueue; + uint64_t numOfErrors; } SQWorkerStat; @@ -68,7 +74,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_ void qWorkerDestroy(void **qWorkerMgmt); -int64_t qWorkerGetWaitTimeInQueue(void *qWorkerMgmt, EQueueType type); +int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pStat); #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d9f3351008..c5fa377fea 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -119,6 +119,8 @@ typedef struct SHeartBeatInfo { struct SAppInstInfo { int64_t numOfConns; SCorEpSet mgmtEp; + TdThreadMutex qnodeMutex; + SArray* pQnodeList; SInstanceSummary summary; SList* pConnList; // STscObj linked list uint64_t clusterId; @@ -290,7 +292,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen); int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb); -int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); +int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray** pNodeList); int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); @@ -317,6 +319,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList); int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList, void** res); int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); +int32_t updateQnodeList(SAppInstInfo*pInfo, SArray* pNodeList); #ifdef __cplusplus } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index a9c5cd06f6..70b60195d2 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -160,6 +160,10 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { taos_close(pTscObj); } + if (pRsp->query->pQnodeList) { + updateQnodeList(pTscObj->pAppInfo, pRsp->query->pQnodeList); + } + releaseTscObj(pRsp->connKey.tscRid); } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index eb4c4cb59f..53ee592945 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -117,7 +117,8 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, SAppInstInfo* p = NULL; if (pInst == NULL) { p = taosMemoryCalloc(1, sizeof(struct SAppInstInfo)); - p->mgmtEp = epSet; + p->mgmtEp = epSet; + taosThreadMutexInit(&p->qnodeMutex, NULL); p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); p->pAppHbMgr = appHbMgrInit(p, key); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); @@ -228,7 +229,61 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } -int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) { +int compareQueryNodeLoad(const void* elem1, const void* elem2) { + SQueryNodeLoad *node1 = (SQueryNodeLoad *)elem1; + SQueryNodeLoad *node2 = (SQueryNodeLoad *)elem2; + + if (node1->load < node2->load) { + return -1; + } + + return node1->load > node2->load; +} + +int32_t updateQnodeList(SAppInstInfo*pInfo, SArray* pNodeList) { + taosThreadMutexLock(&pInfo->qnodeMutex); + if (pInfo->pQnodeList) { + taosArrayDestroy(pInfo->pQnodeList); + pInfo->pQnodeList = NULL; + } + + if (pNodeList) { + pInfo->pQnodeList = taosArrayDup(pNodeList); + taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad); + } + taosThreadMutexUnlock(&pInfo->qnodeMutex); + + return TSDB_CODE_SUCCESS; +} + +int32_t getQnodeList(SRequestObj* pRequest, SArray** pNodeList) { + SAppInstInfo*pInfo = pRequest->pTscObj->pAppInfo; + int32_t code = 0; + + taosThreadMutexLock(&pInfo->qnodeMutex); + if (pInfo->pQnodeList) { + *pNodeList = taosArrayDup(pInfo->pQnodeList); + } + taosThreadMutexUnlock(&pInfo->qnodeMutex); + + if (NULL == *pNodeList) { + SEpSet mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + SCatalog* pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (TSDB_CODE_SUCCESS == code) { + *pNodeList = taosArrayInit(5, sizeof(SQueryNodeLoad)); + code = catalogGetQnodeList(pCatalog, pRequest->pTscObj->pAppInfo->pTransporter, &mgmtEpSet, *pNodeList); + } + + if (TSDB_CODE_SUCCESS == code && *pNodeList) { + code = updateQnodeList(pInfo, *pNodeList); + } + } + + return code; +} + +int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray** pNodeList) { pRequest->type = pQuery->msgType; SPlanContext cxt = {.queryId = pRequest->requestId, .acctId = pRequest->pTscObj->acctId, @@ -237,14 +292,10 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .showRewrite = pQuery->showRewrite, .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; - SEpSet mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - SCatalog* pCatalog = NULL; - int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + + int32_t code = getQnodeList(pRequest, pNodeList); if (TSDB_CODE_SUCCESS == code) { - code = catalogGetQnodeList(pCatalog, pRequest->pTscObj->pAppInfo->pTransporter, &mgmtEpSet, pNodeList); - } - if (TSDB_CODE_SUCCESS == code) { - code = qCreateQueryPlan(&cxt, pPlan, pNodeList); + code = qCreateQueryPlan(&cxt, pPlan, *pNodeList); } return code; } @@ -369,8 +420,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) { - *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); - return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList); + return getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); } int32_t validateSversion(SRequestObj* pRequest, void* res) { @@ -456,8 +506,8 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code code = execDdlQuery(pRequest, pQuery); break; case QUERY_EXEC_MODE_SCHEDULE: { - SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); - code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); + SArray* pNodeList = NULL; + code = getPlan(pRequest, pQuery, &pRequest->body.pDag, &pNodeList); if (TSDB_CODE_SUCCESS == code) { code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, &pRes); if (NULL != pRes) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7615f7b070..6bed28cb89 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -147,12 +147,25 @@ int32_t tEncodeSQueryNodeAddr(SEncoder *pEncoder, SQueryNodeAddr *pAddr) { return 0; } +int32_t tEncodeSQueryNodeLoad(SEncoder *pEncoder, SQueryNodeLoad *pLoad) { + if (tEncodeSQueryNodeAddr(pEncoder, &pLoad->addr) < 0) return -1; + if (tEncodeU64(pEncoder, pLoad->load) < 0) return -1; + return 0; +} + int32_t tDecodeSQueryNodeAddr(SDecoder *pDecoder, SQueryNodeAddr *pAddr) { if (tDecodeI32(pDecoder, &pAddr->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pAddr->epSet) < 0) return -1; return 0; } +int32_t tDecodeSQueryNodeLoad(SDecoder *pDecoder, SQueryNodeLoad *pLoad) { + if (tDecodeSQueryNodeAddr(pDecoder, &pLoad->addr) < 0) return -1; + if (tDecodeU64(pDecoder, &pLoad->load) < 0) return -1; + return 0; +} + + int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pEp->inUse); @@ -304,6 +317,12 @@ static int32_t tSerializeSClientHbRsp(SEncoder *pEncoder, const SClientHbRsp *pR if (tEncodeI32(pEncoder, pRsp->query->onlineDnodes) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->query->killConnection) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pRsp->query->epSet) < 0) return -1; + int32_t num = taosArrayGetSize(pRsp->query->pQnodeList); + if (tEncodeI32(pEncoder, num) < 0) return -1; + for (int32_t i = 0; i < num; ++i) { + SQueryNodeLoad *pLoad = taosArrayGet(pRsp->query->pQnodeList, i); + if (tEncodeSQueryNodeLoad(pEncoder, pLoad) < 0) return -1; + } } else { if (tEncodeI32(pEncoder, queryNum) < 0) return -1; } @@ -333,6 +352,15 @@ static int32_t tDeserializeSClientHbRsp(SDecoder *pDecoder, SClientHbRsp *pRsp) if (tDecodeI32(pDecoder, &pRsp->query->onlineDnodes) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->query->killConnection) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pRsp->query->epSet) < 0) return -1; + int32_t pQnodeNum = 0; + if (tDecodeI32(pDecoder, &pQnodeNum) < 0) return -1; + if (pQnodeNum > 0) { + pRsp->query->pQnodeList = taosArrayInit(pQnodeNum, sizeof(SQueryNodeLoad)); + if (NULL == pRsp->query->pQnodeList) return -1; + SQueryNodeLoad load = {0}; + if (tDecodeSQueryNodeLoad(pDecoder, &load) < 0) return -1; + taosArrayPush(pRsp->query->pQnodeList, &load); + } } int32_t kvNum = 0; @@ -898,6 +926,18 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { // mnode loads if (tEncodeI32(&encoder, pReq->mload.syncState) < 0) return -1; + if (tEncodeI32(&encoder, pReq->qload.dnodeId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.numOfProcessedQuery) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.numOfProcessedCQuery) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.numOfProcessedFetch) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.numOfProcessedDrop) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.numOfProcessedHb) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.cacheDataSize) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.numOfQueryInQueue) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.numOfFetchInQueue) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.timeInQueryQueue) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.timeInFetchQueue) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -955,6 +995,18 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tDecodeI32(&decoder, &pReq->mload.syncState) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->qload.dnodeId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedQuery) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedCQuery) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedFetch) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedDrop) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedHb) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.cacheDataSize) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.numOfQueryInQueue) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.numOfFetchInQueue) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.timeInQueryQueue) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.timeInFetchQueue) < 0) return -1; + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -1921,11 +1973,11 @@ int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - int32_t num = taosArrayGetSize(pRsp->addrsList); + int32_t num = taosArrayGetSize(pRsp->qnodeList); if (tEncodeI32(&encoder, num) < 0) return -1; for (int32_t i = 0; i < num; ++i) { - SQueryNodeAddr *addr = taosArrayGet(pRsp->addrsList, i); - if (tEncodeSQueryNodeAddr(&encoder, addr) < 0) return -1; + SQueryNodeLoad *pLoad = taosArrayGet(pRsp->qnodeList, i); + if (tEncodeSQueryNodeLoad(&encoder, pLoad) < 0) return -1; } tEndEncode(&encoder); @@ -1941,15 +1993,15 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp if (tStartDecode(&decoder) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; - if (NULL == pRsp->addrsList) { - pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr)); - if (NULL == pRsp->addrsList) return -1; + if (NULL == pRsp->qnodeList) { + pRsp->qnodeList = taosArrayInit(num, sizeof(SQueryNodeLoad)); + if (NULL == pRsp->qnodeList) return -1; } for (int32_t i = 0; i < num; ++i) { - SQueryNodeAddr addr = {0}; - if (tDecodeSQueryNodeAddr(&decoder, &addr) < 0) return -1; - taosArrayPush(pRsp->addrsList, &addr); + SQueryNodeLoad load = {0}; + if (tDecodeSQueryNodeLoad(&decoder, &load) < 0) return -1; + taosArrayPush(pRsp->qnodeList, &load); } tEndDecode(&decoder); @@ -1957,7 +2009,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp return 0; } -void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->addrsList); } +void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->qnodeList); } int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) { SEncoder encoder = {0}; diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index ae8879326d..ee811c0071 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -35,6 +35,7 @@ typedef struct SDnodeMgmt { SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; GetMnodeLoadsFp getMnodeLoadsFp; + GetQnodeLoadsFp getQnodeLoadsFp; } SDnodeMgmt; // dmHandle.c @@ -58,4 +59,4 @@ void dmStopWorker(SDnodeMgmt *pMgmt); } #endif -#endif /*_TD_DND_QNODE_INT_H_*/ \ No newline at end of file +#endif /*_TD_DND_QNODE_INT_H_*/ diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 2533f268e5..fbd46db183 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -79,6 +79,8 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { (*pMgmt->getMnodeLoadsFp)(&minfo); req.mload = minfo.load; + (*pMgmt->getQnodeLoadsFp)(&req.qload); + int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); void *pHead = rpcMallocCont(contLen); tSerializeSStatusReq(pHead, contLen, &req); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 59c926545e..d2db1a4a62 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -48,6 +48,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; + pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp; if (dmStartWorker(pMgmt) != 0) { return -1; diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 65794b7b81..864f5b485a 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -20,6 +20,14 @@ void qmGetMonitorInfo(SQnodeMgmt *pMgmt, SMonQmInfo *qmInfo) { SQnodeLoad qload = {0}; qndGetLoad(pMgmt->pQnode, &qload); + qload.dnodeId = pMgmt->pData->dnodeId; + +} + +void qmGetQnodeLoads(SQnodeMgmt *pMgmt, SQnodeLoad *pInfo) { + qndGetLoad(pMgmt->pQnode, pInfo); + + pInfo->dnodeId = pMgmt->pData->dnodeId; } int32_t qmProcessGetMonitorInfoReq(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 6183794bdd..a945358d34 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -104,7 +104,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO dTrace("msg:%p, get from vnode-write queue", pMsg); if (taosArrayPush(pArray, &pMsg) == NULL) { - dTrace("msg:%p, failed to process since %s", pMsg, terrstr()); + dTrace("msg:%p, failed to push to array since %s", pMsg, terrstr()); vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); } } diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 27f1140f23..adde055796 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -168,6 +168,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); void dmSendMonitorReport(); void dmGetVnodeLoads(SMonVloadInfo *pInfo); void dmGetMnodeLoads(SMonMloadInfo *pInfo); +void dmGetQnodeLoads(SQnodeLoad *pInfo); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h index 3ac71de530..8c2d57808f 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h @@ -37,6 +37,7 @@ void bmGetMonitorInfo(void *pMgmt, SMonBmInfo *pInfo); void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo); void mmGetMnodeLoads(void *pMgmt, SMonMloadInfo *pInfo); +void qmGetQnodeLoads(void *pMgmt, SQnodeLoad *pInfo); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 07d0c43360..5f1bf30523 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -178,6 +178,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .sendMonitorReportFp = dmSendMonitorReport, .getVnodeLoadsFp = dmGetVnodeLoads, .getMnodeLoadsFp = dmGetMnodeLoads, + .getQnodeLoadsFp = dmGetQnodeLoads, }; opt.msgCb = dmGetMsgcb(pWrapper->pDnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 0b74d865fd..ecad390ef9 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -170,3 +170,17 @@ void dmGetMnodeLoads(SMonMloadInfo *pInfo) { dmReleaseWrapper(pWrapper); } } + +void dmGetQnodeLoads(SQnodeLoad *pInfo) { + SDnode *pDnode = dmInstance(); + SMgmtWrapper *pWrapper = &pDnode->wrappers[QNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_QM_LOAD, tDeserializeSQnodeLoad, pInfo); + } else if (pWrapper->pMgmt != NULL) { + qmGetQnodeLoads(pWrapper->pMgmt, pInfo); + } + dmReleaseWrapper(pWrapper); + } +} + diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 987fc54416..02b38ed85b 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -130,7 +130,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { _OVER: if (code != 0) { - dError("msg:%p, failed to process since %s", pMsg, terrstr()); + dError("msg:%s, failed to process since %s", TMSG_INFO(pRpc->msgType), terrstr()); if (terrno != 0) code = terrno; if (IsReq(pRpc)) { diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 0d921c2e8b..c142a6cfd8 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -34,6 +34,7 @@ #include "dnode.h" #include "mnode.h" +#include "qnode.h" #include "monitor.h" #include "sync.h" #include "wal.h" @@ -92,6 +93,7 @@ typedef int32_t (*ProcessDropNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg); typedef void (*SendMonitorReportFp)(); typedef void (*GetVnodeLoadsFp)(SMonVloadInfo *pInfo); typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo); +typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo); typedef struct { int32_t dnodeId; @@ -118,6 +120,7 @@ typedef struct { SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; GetMnodeLoadsFp getMnodeLoadsFp; + GetQnodeLoadsFp getQnodeLoadsFp; } SMgmtInputOpt; typedef struct { @@ -180,4 +183,4 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); } #endif -#endif /*_TD_DM_INT_H_*/ \ No newline at end of file +#endif /*_TD_DM_INT_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9e036d7b25..f3af135e6d 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -216,6 +216,7 @@ typedef struct { int64_t createdTime; int64_t updateTime; SDnodeObj* pDnode; + SQnodeLoad load; } SQnodeObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndQnode.h b/source/dnode/mnode/impl/inc/mndQnode.h index 5d177b3f6d..3e38565a4f 100644 --- a/source/dnode/mnode/impl/inc/mndQnode.h +++ b/source/dnode/mnode/impl/inc/mndQnode.h @@ -22,9 +22,15 @@ extern "C" { #endif +#define QNODE_LOAD_VALUE(pQnode) (pQnode ? (pQnode->load.numOfQueryInQueue + pQnode->load.numOfFetchInQueue) : 0) + int32_t mndInitQnode(SMnode *pMnode); void mndCleanupQnode(SMnode *pMnode); +SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId); +void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj); +int32_t mndCreateQnodeList(SMnode *pMnode, SArray** pList, int32_t limit); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 22f858c60b..047aac37a5 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -17,6 +17,7 @@ #include "mndDnode.h" #include "mndAuth.h" #include "mndMnode.h" +#include "mndQnode.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" @@ -388,6 +389,12 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { mndReleaseMnode(pMnode, pObj); } + SQnodeObj *pQnode = mndAcquireQnode(pMnode, statusReq.qload.dnodeId); + if (pQnode != NULL) { + pQnode->load = statusReq.qload; + mndReleaseQnode(pMnode, pQnode); + } + int64_t curMs = taosGetTimestampMs(); bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); bool dnodeChanged = (statusReq.dnodeVer != sdbGetTableVer(pMnode->pSdb, SDB_DNODE)); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index c9c52af0fe..bacdf2f366 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -18,6 +18,7 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndQnode.h" #include "mndShow.h" #include "mndStb.h" #include "mndUser.h" @@ -382,6 +383,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb rspBasic->totalDnodes = mndGetDnodeSize(pMnode); rspBasic->onlineDnodes = 1; // TODO mndGetMnodeEpSet(pMnode, &rspBasic->epSet); + + mndCreateQnodeList(pMnode, &rspBasic->pQnodeList, -1); + mndReleaseConn(pMnode, pConn); hbRsp.query = rspBasic; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 3dc6200229..7c7bdc2e3a 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -60,7 +60,7 @@ int32_t mndInitQnode(SMnode *pMnode) { void mndCleanupQnode(SMnode *pMnode) {} -static SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId) { +SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId) { SQnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_QNODE, &qnodeId); if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_QNODE_NOT_EXIST; @@ -68,7 +68,7 @@ static SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId) { return pObj; } -static void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj) { +void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pObj); } @@ -429,12 +429,49 @@ _OVER: return code; } +int32_t mndCreateQnodeList(SMnode *pMnode, SArray** pList, int32_t limit) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SQnodeObj *pObj = NULL; + int32_t numOfRows = 0; + + SArray* qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad)); + if (NULL == qnodeList) { + mError("failed to alloc epSet while process qnode list req"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + while (1) { + pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SQueryNodeLoad nodeLoad = {0}; + nodeLoad.addr.nodeId = QNODE_HANDLE; + nodeLoad.addr.epSet.numOfEps = 1; + tstrncpy(nodeLoad.addr.epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); + nodeLoad.addr.epSet.eps[0].port = pObj->pDnode->port; + nodeLoad.load = QNODE_LOAD_VALUE(pObj); + + (void)taosArrayPush(qnodeList, &nodeLoad); + + numOfRows++; + sdbRelease(pSdb, pObj); + + if (limit > 0 && numOfRows >= limit) { + break; + } + } + + *pList = qnodeList; + + return TSDB_CODE_SUCCESS; +} + + static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) { int32_t code = -1; - int32_t numOfRows = 0; SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - SQnodeObj *pObj = NULL; SQnodeListReq qlistReq = {0}; SQnodeListRsp qlistRsp = {0}; @@ -444,34 +481,10 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) { goto _OVER; } - qlistRsp.addrsList = taosArrayInit(5, sizeof(SQueryNodeAddr)); - if (NULL == qlistRsp.addrsList) { - mError("failed to alloc epSet while process qnode list req"); - terrno = TSDB_CODE_OUT_OF_MEMORY; + if (mndCreateQnodeList(pMnode, &qlistRsp.qnodeList, qlistReq.rowNum) != 0) { goto _OVER; } - void *pIter = NULL; - while (1) { - pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SQueryNodeAddr nodeAddr = {0}; - nodeAddr.nodeId = QNODE_HANDLE; - nodeAddr.epSet.numOfEps = 1; - tstrncpy(nodeAddr.epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); - nodeAddr.epSet.eps[0].port = pObj->pDnode->port; - - (void)taosArrayPush(qlistRsp.addrsList, &nodeAddr); - - numOfRows++; - sdbRelease(pSdb, pObj); - - if (qlistReq.rowNum > 0 && numOfRows >= qlistReq.rowNum) { - break; - } - } - int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 40aa572a56..438982ac6a 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -41,12 +41,24 @@ void qndClose(SQnode *pQnode) { } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { - SMsgCb* pCb = &pQnode->msgCb; + SReadHandle handle = {.pMsgCb = &pQnode->msgCb}; + SQWorkerStat stat = {0}; - pLoad->numOfQueryInQueue = pCb->qsizeFp(pCb->mgmt, pQnode->qndId, QUERY_QUEUE); - pLoad->numOfFetchInQueue = pCb->qsizeFp(pCb->mgmt, pQnode->qndId, FETCH_QUEUE); - pLoad->waitTimeInQueryQUeue = qWorkerGetWaitTimeInQueue(pQnode->pQuery, QUERY_QUEUE); - pLoad->waitTimeInFetchQUeue = qWorkerGetWaitTimeInQueue(pQnode->pQuery, FETCH_QUEUE); + int32_t code = qWorkerGetStat(&handle, pQnode->pQuery, &stat); + if (code) { + return code; + } + + pLoad->numOfQueryInQueue = stat.numOfQueryInQueue; + pLoad->numOfFetchInQueue = stat.numOfFetchInQueue; + pLoad->timeInQueryQueue = stat.timeInQueryQueue; + pLoad->timeInFetchQueue = stat.timeInFetchQueue; + pLoad->cacheDataSize = stat.cacheDataSize; + pLoad->numOfProcessedQuery = stat.queryProcessed; + pLoad->numOfProcessedCQuery = stat.cqueryProcessed; + pLoad->numOfProcessedFetch = stat.fetchProcessed; + pLoad->numOfProcessedDrop = stat.dropProcessed; + pLoad->numOfProcessedHb = stat.hbProcessed; return 0; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 230949ab7f..57f651ed69 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -342,16 +342,16 @@ typedef struct SCtgOperation { ctgOpFunc func; } SCtgOperation; -#define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) +#define CTG_QUEUE_INC() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) +#define CTG_QUEUE_DEC() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_STAT_ADD(_item, _n) atomic_add_fetch_64(&(_item), _n) -#define CTG_STAT_SUB(_item, _n) atomic_sub_fetch_64(&(_item), _n) +#define CTG_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n) +#define CTG_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) #define CTG_STAT_GET(_item) atomic_load_64(&(_item)) -#define CTG_RUNTIME_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.runtime.item, n)) -#define CTG_CACHE_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.cache.item, n)) -#define CTG_CACHE_STAT_SUB(item, n) (CTG_STAT_SUB(gCtgMgmt.stat.cache.item, n)) +#define CTG_RT_STAT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.stat.runtime.item, n)) +#define CTG_CACHE_STAT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.stat.cache.item, n)) +#define CTG_CACHE_STAT_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.stat.cache.item, n)) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6519440dad..a874764468 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -558,7 +558,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { *catalogHandle = clusterCtg; - CTG_CACHE_STAT_ADD(clusterNum, 1); + CTG_CACHE_STAT_INC(clusterNum, 1); return TSDB_CODE_SUCCESS; @@ -579,7 +579,7 @@ void catalogFreeHandle(SCatalog* pCtg) { return; } - CTG_CACHE_STAT_SUB(clusterNum, 1); + CTG_CACHE_STAT_DEC(clusterNum, 1); uint64_t clusterId = pCtg->clusterId; @@ -990,7 +990,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } if (pReq->qNodeRequired) { - pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr)); + pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeLoad)); CTG_ERR_JRET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), pRsp->pQnodeList, NULL)); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index d1e2056bec..2fbb8b499d 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -178,7 +178,7 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac *pCache = dbCache; - CTG_CACHE_STAT_ADD(vgHitNum, 1); + CTG_CACHE_STAT_INC(vgHitNum, 1); ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); @@ -192,7 +192,7 @@ _return: *pCache = NULL; - CTG_CACHE_STAT_ADD(vgMissNum, 1); + CTG_CACHE_STAT_INC(vgMissNum, 1); return TSDB_CODE_SUCCESS; } @@ -279,7 +279,7 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctgReleaseDBCache(pCtg, dbCache); ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, ctx->pName->tname); - CTG_CACHE_STAT_ADD(tblHitNum, 1); + CTG_CACHE_STAT_INC(tblHitNum, 1); return TSDB_CODE_SUCCESS; } @@ -312,7 +312,7 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctgReleaseDBCache(pCtg, dbCache); - CTG_CACHE_STAT_ADD(tblHitNum, 1); + CTG_CACHE_STAT_INC(tblHitNum, 1); ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); @@ -323,7 +323,7 @@ _return: ctgReleaseDBCache(pCtg, dbCache); taosMemoryFreeClear(*pTableMeta); - CTG_CACHE_STAT_ADD(tblMissNum, 1); + CTG_CACHE_STAT_INC(tblMissNum, 1); CTG_RET(code); } @@ -462,7 +462,7 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFNam *inCache = true; ctgDebug("Got user from cache, user:%s", user); - CTG_CACHE_STAT_ADD(userHitNum, 1); + CTG_CACHE_STAT_INC(userHitNum, 1); if (pUser->superUser) { *pass = true; @@ -491,7 +491,7 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFNam _return: *inCache = false; - CTG_CACHE_STAT_ADD(userMissNum, 1); + CTG_CACHE_STAT_INC(userMissNum, 1); return TSDB_CODE_SUCCESS; } @@ -521,7 +521,7 @@ void ctgDequeue(SCtgCacheOperation **op) { SCtgQNode *node = gCtgMgmt.queue.head->next; gCtgMgmt.queue.head = gCtgMgmt.queue.head->next; - CTG_QUEUE_SUB(); + CTG_QUEUE_DEC(); taosMemoryFreeClear(orig); @@ -545,8 +545,8 @@ int32_t ctgEnqueue(SCatalog* pCtg, SCtgCacheOperation *operation) { gCtgMgmt.queue.tail = node; CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); - CTG_QUEUE_ADD(); - CTG_RUNTIME_STAT_ADD(qNum, 1); + CTG_QUEUE_INC(); + CTG_RT_STAT_INC(qNum, 1); tsem_post(&gCtgMgmt.queue.reqSem); @@ -988,7 +988,7 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_CACHE_STAT_ADD(dbNum, 1); + CTG_CACHE_STAT_INC(dbNum, 1); SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); @@ -1048,7 +1048,7 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } - CTG_CACHE_STAT_SUB(dbNum, 1); + CTG_CACHE_STAT_DEC(dbNum, 1); ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); @@ -1187,7 +1187,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } else { - CTG_CACHE_STAT_SUB(stblNum, 1); + CTG_CACHE_STAT_DEC(stblNum, 1); } CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); @@ -1214,7 +1214,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam } if (NULL == orig) { - CTG_CACHE_STAT_ADD(tblNum, 1); + CTG_CACHE_STAT_INC(tblNum, 1); } ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); @@ -1233,7 +1233,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_CACHE_STAT_ADD(stblNum, 1); + CTG_CACHE_STAT_INC(stblNum, 1); CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); @@ -1371,14 +1371,14 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { - CTG_CACHE_STAT_SUB(stblNum, 1); + CTG_CACHE_STAT_DEC(stblNum, 1); } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { - CTG_CACHE_STAT_SUB(tblNum, 1); + CTG_CACHE_STAT_DEC(tblNum, 1); } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); @@ -1419,7 +1419,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { - CTG_CACHE_STAT_SUB(tblNum, 1); + CTG_CACHE_STAT_DEC(tblNum, 1); } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); @@ -1578,7 +1578,7 @@ void* ctgUpdateThreadFunc(void* param) { tsem_post(&gCtgMgmt.queue.rspSem); } - CTG_RUNTIME_STAT_ADD(qDoneNum, 1); + CTG_RT_STAT_INC(qDoneNum, 1); ctgdShowClusterCache(pCtg); } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 4def1fff4f..b16a082f75 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -275,7 +275,7 @@ int32_t ctgGetQnodeListFromMnode(CTG_PARAMS, SArray *out, SCtgTask* pTask) { } if (pTask) { - void* pOut = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + void* pOut = taosArrayInit(4, sizeof(SQueryNodeLoad)); if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 4fbf1463d8..4625203dd8 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -85,7 +85,7 @@ void ctgFreeTbMetaCache(SCtgTbMetaCache *cache) { int32_t stblNum = taosHashGetSize(cache->stbCache); taosHashCleanup(cache->stbCache); cache->stbCache = NULL; - CTG_CACHE_STAT_SUB(stblNum, stblNum); + CTG_CACHE_STAT_DEC(stblNum, stblNum); } CTG_UNLOCK(CTG_WRITE, &cache->stbLock); @@ -94,7 +94,7 @@ void ctgFreeTbMetaCache(SCtgTbMetaCache *cache) { int32_t tblNum = taosHashGetSize(cache->metaCache); taosHashCleanup(cache->metaCache); cache->metaCache = NULL; - CTG_CACHE_STAT_SUB(tblNum, tblNum); + CTG_CACHE_STAT_DEC(tblNum, tblNum); } CTG_UNLOCK(CTG_WRITE, &cache->metaLock); } @@ -145,7 +145,7 @@ void ctgFreeHandle(SCatalog* pCtg) { taosHashCleanup(pCtg->dbCache); - CTG_CACHE_STAT_SUB(dbNum, dbNum); + CTG_CACHE_STAT_DEC(dbNum, dbNum); } if (pCtg->userCache) { @@ -162,7 +162,7 @@ void ctgFreeHandle(SCatalog* pCtg) { taosHashCleanup(pCtg->userCache); - CTG_CACHE_STAT_SUB(userNum, userNum); + CTG_CACHE_STAT_DEC(userNum, userNum); } taosMemoryFree(pCtg); diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 85356a862c..8f49440105 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -37,6 +37,7 @@ typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, uint64_t useconds); typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd); typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput); typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle); +typedef int32_t (*FGetCacheSize)(struct SDataSinkHandle* pHandle, uint64_t* size); typedef struct SDataSinkHandle { FPutDataBlock fPut; @@ -44,6 +45,7 @@ typedef struct SDataSinkHandle { FGetDataLength fGetLen; FGetDataBlock fGetData; FDestroyDataSinker fDestroy; + FGetCacheSize fGetCacheSize; } SDataSinkHandle; int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index fa9e27a5f8..080cf5c2ad 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -22,6 +22,8 @@ #include "tglobal.h" #include "tqueue.h" +extern SDataSinkStat gDataSinkStat; + typedef struct SDataDispatchBuf { int32_t useSize; int32_t allocSize; @@ -45,6 +47,7 @@ typedef struct SDataDispatchHandle { int32_t status; bool queryEnd; uint64_t useconds; + uint64_t cachedSize; TdThreadMutex mutex; } SDataDispatchHandle; @@ -71,7 +74,7 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header -static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { +static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots); SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; @@ -84,6 +87,9 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat blockCompressEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); pBuf->useSize += pEntry->dataLen; + + atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); + atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); } static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { @@ -156,6 +162,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryE taosFreeQitem(pBuf); *pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; *pQueryEnd = pDispatcher->queryEnd; + qDebug("got data len %d, row num %d in sink", *pLen, ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows); } static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { @@ -173,6 +180,10 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; + + atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); + atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); taosThreadMutexLock(&pDispatcher->mutex); @@ -180,11 +191,14 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->pSchema->precision; taosThreadMutexUnlock(&pDispatcher->mutex); + + return TSDB_CODE_SUCCESS; } static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; + atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize); taosMemoryFreeClear(pDispatcher->nextOutput.pData); while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { SDataDispatchBuf* pBuf = NULL; @@ -197,6 +211,13 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { return TSDB_CODE_SUCCESS; } +int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { + SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; + + *size = atomic_load_64(&pDispatcher->cachedSize); + return TSDB_CODE_SUCCESS; +} + int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { @@ -208,6 +229,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->sink.fGetLen = getDataLength; dispatcher->sink.fGetData = getDataBlock; dispatcher->sink.fDestroy = destroyDataSinker; + dispatcher->sink.fGetCacheSize = getCacheSize; dispatcher->pManager = pManager; dispatcher->pSchema = pDataSink->pInputDataBlockDesc; dispatcher->status = DS_BUF_EMPTY; diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 64206fc10a..9016ca274a 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -19,6 +19,7 @@ #include "planner.h" static SDataSinkManager gDataSinkManager = {0}; +SDataSinkStat gDataSinkStat = {0}; int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg) { gDataSinkManager.cfg = *cfg; @@ -26,6 +27,13 @@ int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg) { return 0; // to avoid compiler eror } +int32_t dsDataSinkGetCacheSize(SDataSinkStat *pStat) { + pStat->cachedSize = atomic_load_64(&gDataSinkStat.cachedSize); + + return 0; +} + + int32_t dsCreateDataSinker(const SDataSinkNode *pDataSink, DataSinkHandle* pHandle) { if (QUERY_NODE_PHYSICAL_PLAN_DISPATCH == nodeType(pDataSink)) { return createDataDispatcher(&gDataSinkManager, pDataSink, pHandle); @@ -53,6 +61,12 @@ int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { return pHandleImpl->fGetData(pHandleImpl, pOutput); } +int32_t dsGetCacheSize(DataSinkHandle handle, uint64_t *pSize) { + SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; + return pHandleImpl->fGetCacheSize(pHandleImpl, pSize); +} + + void dsScheduleProcess(void* ahandle, void* pItem) { // todo } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index bbdb3b2b7e..785060fa30 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -303,7 +303,9 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) int32_t dstSlotId = pExpr->base.resSchema.slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); + colInfoDataEnsureCapacity(pColInfoData, 0, pBlock->info.rows); + colInfoDataCleanup(pColInfoData, pBlock->info.rows); int32_t functionId = pExpr->pExpr->_function.functionId; diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index e106cbd428..944a7b5475 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -556,4 +556,50 @@ int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInf tDecoderClear(&decoder); return 0; -} \ No newline at end of file +} + + +int32_t tSerializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->numOfProcessedQuery) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->numOfProcessedCQuery) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->numOfProcessedFetch) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->numOfProcessedDrop) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->numOfProcessedHb) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->cacheDataSize) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->numOfQueryInQueue) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->numOfFetchInQueue) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->timeInQueryQueue) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->timeInFetchQueue) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->numOfProcessedQuery) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->numOfProcessedCQuery) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->numOfProcessedFetch) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->numOfProcessedDrop) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->numOfProcessedHb) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->cacheDataSize) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->numOfQueryInQueue) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->numOfFetchInQueue) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->timeInQueryQueue) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->timeInFetchQueue) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + + diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0f88a54e91..50819ffc87 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -468,6 +468,7 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpla return TSDB_CODE_OUT_OF_MEMORY; } vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode); } @@ -489,7 +490,8 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pSubplan->execNodeStat.tableNum = pScanLogicNode->pVgroupList->vgroups[0].numOfTable; } if (pCxt->pExecNodeList) { - taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); + SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; + taosArrayPush(pCxt->pExecNodeList, &node); } tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); pTableScan->dataRequired = pScanLogicNode->dataRequired; @@ -520,10 +522,11 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->accountId = pCxt->pPlanCxt->acctId; if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLES)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + SQueryNodeLoad node = { .addr = pSubplan->execNode, .load = 0}; taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { - SQueryNodeAddr addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}; - taosArrayPush(pCxt->pExecNodeList, &addr); + SQueryNodeLoad node = { .addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; + taosArrayPush(pCxt->pExecNodeList, &node); } pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); @@ -1243,7 +1246,8 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)pLogicSubplan->pNode; pSubplan->msgType = pModif->msgType; pSubplan->execNode.epSet = pModif->pVgDataBlocks->vg.epSet; - taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); + SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; + taosArrayPush(pCxt->pExecNodeList, &node); code = createDataInserter(pCxt, pModif->pVgDataBlocks, &pSubplan->pDataSink); } else { pSubplan->msgType = TDMT_VND_QUERY; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 636b2b50a8..810c0153b1 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -373,7 +373,7 @@ int32_t queryProcessQnodeListRsp(void *output, char *msg, int32_t msgSize) { return code; } - out.addrsList = (SArray *)output; + out.qnodeList = (SArray *)output; if (tDeserializeSQnodeListRsp(msg, msgSize, &out) != 0) { qError("invalid qnode list rsp msg, msgSize:%d", msgSize); code = TSDB_CODE_INVALID_MSG; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index b0a102069d..4fe3c18393 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -145,13 +145,30 @@ typedef struct SQWSchStatus { SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus } SQWSchStatus; -typedef struct SQWWaitTimeStat { +typedef struct SQWTimeInQ { uint64_t num; uint64_t total; -} SQWWaitTimeStat; +} SQWTimeInQ; + +typedef struct SQWMsgStat { + SQWTimeInQ waitTime[2]; + uint64_t queryProcessed; + uint64_t cqueryProcessed; + uint64_t fetchProcessed; + uint64_t fetchRspProcessed; + uint64_t cancelProcessed; + uint64_t dropProcessed; + uint64_t hbProcessed; +} SQWMsgStat; + +typedef struct SQWRTStat { + uint64_t startTaskNum; + uint64_t stopTaskNum; +} SQWRTStat; typedef struct SQWStat { - SQWWaitTimeStat msgWait[2]; + SQWMsgStat msgStat; + SQWRTStat rtStat; } SQWStat; // Qnode/Vnode level task management @@ -182,10 +199,13 @@ typedef struct SQWorkerMgmt { #define QW_IDS() sId, qId, tId, rId #define QW_FPARAMS() mgmt, QW_IDS() -#define QW_GET_EVENT_VALUE(ctx, event) atomic_load_8(&(ctx)->events[event]) +#define QW_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n) +#define QW_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) +#define QW_STAT_GET(_item) atomic_load_64(&(_item)) -#define QW_IS_EVENT_RECEIVED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_RECEIVED) -#define QW_IS_EVENT_PROCESSED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_PROCESSED) +#define QW_GET_EVENT(ctx, event) atomic_load_8(&(ctx)->events[event]) +#define QW_IS_EVENT_RECEIVED(ctx, event) (QW_GET_EVENT(ctx, event) == QW_EVENT_RECEIVED) +#define QW_IS_EVENT_PROCESSED(ctx, event) (QW_GET_EVENT(ctx, event) == QW_EVENT_PROCESSED) #define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED) #define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED) @@ -332,8 +352,8 @@ int32_t qwDropTask(QW_FPARAMS_DEF); void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx); int32_t qwOpenRef(void); void qwSetHbParam(int64_t refId, SQWHbParam **pParam); -int32_t qwUpdateWaitTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type); -int64_t qwGetWaitTimeInQueue(SQWorker *mgmt, EQueueType type); +int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type); +int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type); void qwDbgDumpMgmtInfo(SQWorker *mgmt); int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index b9dc18cd2f..f8205a6bb4 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -257,7 +257,8 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int SSubQueryMsg *msg = pMsg->pCont; SQWorker * mgmt = (SQWorker *)qWorkerMgmt; - qwUpdateWaitTimeInQueue(mgmt, ts, QUERY_QUEUE); + qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE); + QW_STAT_INC(mgmt->stat.msgStat.queryProcessed, 1); if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -297,7 +298,8 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in SQWTaskCtx * handles = NULL; SQWorker * mgmt = (SQWorker *)qWorkerMgmt; - qwUpdateWaitTimeInQueue(mgmt, ts, QUERY_QUEUE); + qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE); + QW_STAT_INC(mgmt->stat.msgStat.cqueryProcessed, 1); if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -328,7 +330,8 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int SResFetchReq *msg = pMsg->pCont; SQWorker * mgmt = (SQWorker *)qWorkerMgmt; - qwUpdateWaitTimeInQueue(mgmt, ts, FETCH_QUEUE); + qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); + QW_STAT_INC(mgmt->stat.msgStat.fetchProcessed, 1); if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -357,7 +360,10 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { SQWorker * mgmt = (SQWorker *)qWorkerMgmt; - qwUpdateWaitTimeInQueue(mgmt, ts, FETCH_QUEUE); + if (mgmt) { + qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); + QW_STAT_INC(mgmt->stat.msgStat.fetchRspProcessed, 1); + } qProcessFetchRsp(NULL, pMsg, NULL); pMsg->pCont = NULL; @@ -373,7 +379,8 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in int32_t code = 0; STaskCancelReq *msg = pMsg->pCont; - qwUpdateWaitTimeInQueue(mgmt, ts, FETCH_QUEUE); + qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); + QW_STAT_INC(mgmt->stat.msgStat.cancelProcessed, 1); if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task cancel msg"); @@ -411,7 +418,8 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6 STaskDropReq *msg = pMsg->pCont; SQWorker * mgmt = (SQWorker *)qWorkerMgmt; - qwUpdateWaitTimeInQueue(mgmt, ts, FETCH_QUEUE); + qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); + QW_STAT_INC(mgmt->stat.msgStat.dropProcessed, 1); if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task drop msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -452,7 +460,8 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_ SSchedulerHbReq req = {0}; SQWorker * mgmt = (SQWorker *)qWorkerMgmt; - qwUpdateWaitTimeInQueue(mgmt, ts, FETCH_QUEUE); + qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); + QW_STAT_INC(mgmt->stat.msgStat.hbProcessed, 1); if (NULL == pMsg->pCont) { QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen); diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index a4bc22fc88..e5d606ffff 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -499,7 +499,7 @@ int32_t qwOpenRef(void) { return TSDB_CODE_SUCCESS; } -int32_t qwUpdateWaitTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type) { +int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type) { if (ts <= 0) { return TSDB_CODE_SUCCESS; } @@ -507,12 +507,12 @@ int32_t qwUpdateWaitTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type) { int64_t duration = taosGetTimestampUs() - ts; switch (type) { case QUERY_QUEUE: - ++mgmt->stat.msgWait[0].num; - mgmt->stat.msgWait[0].total += duration; + ++mgmt->stat.msgStat.waitTime[0].num; + mgmt->stat.msgStat.waitTime[0].total += duration; break; case FETCH_QUEUE: - ++mgmt->stat.msgWait[1].num; - mgmt->stat.msgWait[1].total += duration; + ++mgmt->stat.msgStat.waitTime[1].num; + mgmt->stat.msgStat.waitTime[1].total += duration; break; default: qError("unsupported queue type %d", type); @@ -522,19 +522,20 @@ int32_t qwUpdateWaitTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type) { return TSDB_CODE_SUCCESS; } -int64_t qwGetWaitTimeInQueue(SQWorker *mgmt, EQueueType type) { - SQWWaitTimeStat *pStat = NULL; +int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type) { + SQWTimeInQ *pStat = NULL; switch (type) { case QUERY_QUEUE: - pStat = &mgmt->stat.msgWait[0]; + pStat = &mgmt->stat.msgStat.waitTime[0]; return pStat->num ? (pStat->total/pStat->num) : 0; case FETCH_QUEUE: - pStat = &mgmt->stat.msgWait[1]; + pStat = &mgmt->stat.msgStat.waitTime[1]; return pStat->num ? (pStat->total/pStat->num) : 0; default: qError("unsupported queue type %d", type); - return -1; } + + return -1; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7201820854..03b8cd5cca 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,4 +1,3 @@ -#include "qworker.h" #include "dataSinkMgt.h" #include "executor.h" #include "planner.h" @@ -8,6 +7,7 @@ #include "tcommon.h" #include "tmsg.h" #include "tname.h" +#include "qworker.h" SQWorkerMgmt gQwMgmt = { .lock = 0, @@ -950,8 +950,29 @@ void qWorkerDestroy(void **qWorkerMgmt) { } } -int64_t qWorkerGetWaitTimeInQueue(void *qWorkerMgmt, EQueueType type) { - return qwGetWaitTimeInQueue((SQWorker *)qWorkerMgmt, type); +int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pStat) { + if (NULL == handle || NULL == qWorkerMgmt || NULL == pStat) { + QW_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + SDataSinkStat sinkStat = {0}; + + dsDataSinkGetCacheSize(&sinkStat); + pStat->cacheDataSize = sinkStat.cachedSize; + + pStat->queryProcessed = QW_STAT_GET(mgmt->stat.msgStat.queryProcessed); + pStat->cqueryProcessed = QW_STAT_GET(mgmt->stat.msgStat.cqueryProcessed); + pStat->fetchProcessed = QW_STAT_GET(mgmt->stat.msgStat.fetchProcessed); + pStat->dropProcessed = QW_STAT_GET(mgmt->stat.msgStat.dropProcessed); + pStat->hbProcessed = QW_STAT_GET(mgmt->stat.msgStat.hbProcessed); + + pStat->numOfQueryInQueue = handle->pMsgCb->qsizeFp(handle->pMsgCb->mgmt, mgmt->nodeId, QUERY_QUEUE); + pStat->numOfFetchInQueue = handle->pMsgCb->qsizeFp(handle->pMsgCb->mgmt, mgmt->nodeId, FETCH_QUEUE); + pStat->timeInQueryQueue = qwGetTimeInQueue((SQWorker *)qWorkerMgmt, QUERY_QUEUE); + pStat->timeInFetchQueue = qwGetTimeInQueue((SQWorker *)qWorkerMgmt, FETCH_QUEUE); + + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index e5aa2bd523..00bebcd5ac 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -469,6 +469,34 @@ _return: SCH_RET(code); } +int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { + int32_t addNum = 0; + int32_t nodeNum = 0; + + if (pJob->nodeList) { + nodeNum = taosArrayGetSize(pJob->nodeList); + + for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { + SQueryNodeAddr *naddr = taosArrayGet(pJob->nodeList, i); + + if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { + SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ++addNum; + } + } + + if (addNum <= 0) { + SCH_TASK_ELOG("no available execNode as candidates, nodeNum:%d", nodeNum); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (NULL != pTask->candidateAddrs) { return TSDB_CODE_SUCCESS; @@ -492,27 +520,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - int32_t addNum = 0; - int32_t nodeNum = 0; - if (pJob->nodeList) { - nodeNum = taosArrayGetSize(pJob->nodeList); - - for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { - SQueryNodeAddr *naddr = taosArrayGet(pJob->nodeList, i); - - if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { - SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - ++addNum; - } - } - - if (addNum <= 0) { - SCH_TASK_ELOG("no available execNode as candidates, nodeNum:%d", nodeNum); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + SCH_ERR_RET(schSetAddrsFromNodeList(pJob, pTask)); /* for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3ecc4f4a30..522bd8044d 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -141,7 +141,7 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) { if (pJob->status < JOB_TASK_STATUS_NOT_START || pJob->levelNum <= 0 || NULL == pJob->levels) { qDebug("job not initialized or not executable job, refId:%" PRIx64, job); - SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } for (int32_t i = pJob->levelNum - 1; i >= 0; --i) { @@ -155,7 +155,11 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) { } } - return TSDB_CODE_SUCCESS; +_return: + + schReleaseJob(job); + + SCH_RET(code); } int32_t scheduleCancelJob(int64_t job) { From 990abaaab14aa8f8684ca65d646e54cfee0ad490 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 31 May 2022 16:03:36 +0800 Subject: [PATCH 03/57] test: finish 1-insert/influxdb_line_taosc_insert.py --- tests/pytest/util/common.py | 8 +- .../1-insert/influxdb_line_taosc_insert.py | 1333 +++++++++++++++++ 2 files changed, 1339 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/1-insert/influxdb_line_taosc_insert.py diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 8c791efbc6..7b00e6f331 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -95,9 +95,13 @@ class TDCom: stb_list = map(lambda x: x[0], res_row_list) for stb in stb_list: if type == "taosc": - tdSql.execute(f'drop table if exists {stb}') + tdSql.execute(f'drop table if exists `{stb}`') + if not stb[0].isdigit(): + tdSql.execute(f'drop table if exists {stb}') elif type == "restful": - self.restApiPost(f"drop table if exists {stb}") + self.restApiPost(f"drop table if exists `{stb}`") + if not stb[0].isdigit(): + self.restApiPost(f"drop table if exists {stb}") def dateToTs(self, datetime_input): return int(time.mktime(time.strptime(datetime_input, "%Y-%m-%d %H:%M:%S.%f"))) diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py new file mode 100644 index 0000000000..75e0c3d59c --- /dev/null +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -0,0 +1,1333 @@ +################################################################### +# Copyright (c) 2021 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import traceback +import random +from taos.error import SchemalessError +import time +from copy import deepcopy +import numpy as np +from util.log import * +from util.cases import * +from util.sql import * +import threading +from util.types import TDSmlProtocolType, TDSmlTimestampType +from util.common import tdCom + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self._conn = conn + + def createDb(self, name="test", db_update_tag=0): + if db_update_tag == 0: + tdSql.execute(f"drop database if exists {name}") + tdSql.execute(f"create database if not exists {name} precision 'ms'") + else: + tdSql.execute(f"drop database if exists {name}") + tdSql.execute(f"create database if not exists {name} precision 'ms' update 1") + tdSql.execute(f'use {name}') + + def timeTrans(self, time_value, ts_type): + if int(time_value) == 0: + ts = time.time() + else: + if ts_type == TDSmlTimestampType.NANO_SECOND.value or ts_type is None: + ts = int(''.join(list(filter(str.isdigit, time_value)))) / 1000000000 + elif ts_type == TDSmlTimestampType.MICRO_SECOND.value: + ts = int(''.join(list(filter(str.isdigit, time_value)))) / 1000000 + elif ts_type == TDSmlTimestampType.MILLI_SECOND.value: + ts = int(''.join(list(filter(str.isdigit, time_value)))) / 1000 + elif ts_type == TDSmlTimestampType.SECOND.value: + ts = int(''.join(list(filter(str.isdigit, time_value)))) / 1 + ulsec = repr(ts).split('.')[1][:6] + if len(ulsec) < 6 and int(ulsec) != 0: + ulsec = int(ulsec) * (10 ** (6 - len(ulsec))) + elif int(ulsec) == 0: + ulsec *= 6 + # * follow two rows added for tsCheckCase + td_ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) + return td_ts + #td_ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) + td_ts = time.strftime("%Y-%m-%d %H:%M:%S.{}".format(ulsec), time.localtime(ts)) + return td_ts + #return repr(datetime.datetime.strptime(td_ts, "%Y-%m-%d %H:%M:%S.%f")) + + def dateToTs(self, datetime_input): + return int(time.mktime(time.strptime(datetime_input, "%Y-%m-%d %H:%M:%S.%f"))) + + def getTdTypeValue(self, value, vtype="col"): + """ + vtype must be col or tag + """ + if vtype == "col": + if value.lower().endswith("i8"): + td_type = "TINYINT" + td_tag_value = ''.join(list(value)[:-2]) + elif value.lower().endswith("i16"): + td_type = "SMALLINT" + td_tag_value = ''.join(list(value)[:-3]) + elif value.lower().endswith("i32"): + td_type = "INT" + td_tag_value = ''.join(list(value)[:-3]) + elif value.lower().endswith("i64"): + td_type = "BIGINT" + td_tag_value = ''.join(list(value)[:-3]) + elif value.lower().lower().endswith("u64"): + td_type = "BIGINT UNSIGNED" + td_tag_value = ''.join(list(value)[:-3]) + elif value.lower().endswith("f32"): + td_type = "FLOAT" + td_tag_value = ''.join(list(value)[:-3]) + td_tag_value = '{}'.format(np.float32(td_tag_value)) + elif value.lower().endswith("f64"): + td_type = "DOUBLE" + td_tag_value = ''.join(list(value)[:-3]) + if "e" in value.lower(): + td_tag_value = str(float(td_tag_value)) + elif value.lower().startswith('l"'): + td_type = "NCHAR" + td_tag_value = ''.join(list(value)[2:-1]) + elif value.startswith('"') and value.endswith('"'): + td_type = "VARCHAR" + td_tag_value = ''.join(list(value)[1:-1]) + elif value.lower() == "t" or value.lower() == "true": + td_type = "BOOL" + td_tag_value = "True" + elif value.lower() == "f" or value.lower() == "false": + td_type = "BOOL" + td_tag_value = "False" + elif value.isdigit(): + td_type = "DOUBLE" + td_tag_value = str(float(value)) + else: + td_type = "DOUBLE" + if "e" in value.lower(): + td_tag_value = str(float(value)) + else: + td_tag_value = value + elif vtype == "tag": + td_type = "NCHAR" + td_tag_value = str(value) + return td_type, td_tag_value + + def typeTrans(self, type_list): + type_num_list = [] + for tp in type_list: + if tp.upper() == "TIMESTAMP": + type_num_list.append(9) + elif tp.upper() == "BOOL": + type_num_list.append(1) + elif tp.upper() == "TINYINT": + type_num_list.append(2) + elif tp.upper() == "SMALLINT": + type_num_list.append(3) + elif tp.upper() == "INT": + type_num_list.append(4) + elif tp.upper() == "BIGINT": + type_num_list.append(5) + elif tp.upper() == "FLOAT": + type_num_list.append(6) + elif tp.upper() == "DOUBLE": + type_num_list.append(7) + elif tp.upper() == "VARCHAR": + type_num_list.append(8) + elif tp.upper() == "NCHAR": + type_num_list.append(10) + elif tp.upper() == "BIGINT UNSIGNED": + type_num_list.append(14) + return type_num_list + + def inputHandle(self, input_sql, ts_type): + input_sql_split_list = input_sql.split(" ") + + stb_tag_list = input_sql_split_list[0].split(',') + stb_col_list = input_sql_split_list[1].split(',') + time_value = self.timeTrans(input_sql_split_list[2], ts_type) + + stb_name = stb_tag_list[0] + stb_tag_list.pop(0) + + tag_name_list = [] + tag_value_list = [] + td_tag_value_list = [] + td_tag_type_list = [] + + col_name_list = [] + col_value_list = [] + td_col_value_list = [] + td_col_type_list = [] + for elm in stb_tag_list: + if "id=" in elm.lower(): + tb_name = elm.split('=')[1] + tag_name_list.append(elm.split("=")[0]) + td_tag_value_list.append(tb_name) + td_tag_type_list.append("NCHAR") + else: + tag_name_list.append(elm.split("=")[0]) + tag_value_list.append(elm.split("=")[1]) + tb_name = "" + td_tag_value_list.append(self.getTdTypeValue(elm.split("=")[1], "tag")[1]) + td_tag_type_list.append(self.getTdTypeValue(elm.split("=")[1], "tag")[0]) + + for elm in stb_col_list: + col_name_list.append(elm.split("=")[0]) + col_value_list.append(elm.split("=")[1]) + td_col_value_list.append(self.getTdTypeValue(elm.split("=")[1])[1]) + td_col_type_list.append(self.getTdTypeValue(elm.split("=")[1])[0]) + + final_field_list = [] + final_field_list.extend(col_name_list) + final_field_list.extend(tag_name_list) + + final_type_list = [] + final_type_list.append("TIMESTAMP") + final_type_list.extend(td_col_type_list) + final_type_list.extend(td_tag_type_list) + final_type_list = self.typeTrans(final_type_list) + + final_value_list = [] + final_value_list.append(time_value) + final_value_list.extend(td_col_value_list) + final_value_list.extend(td_tag_value_list) + return final_value_list, final_field_list, final_type_list, stb_name, tb_name + + def gen_influxdb_line(self, stb_name, tb_name, id, t0, t1, t2, t3, t4, t5, t6, t7, t8, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, + ts, id_noexist_tag, id_change_tag, id_double_tag, ct_add_tag, ct_am_tag, ct_ma_tag, ct_min_tag, c_multi_tag, t_multi_tag, c_blank_tag, t_blank_tag, chinese_tag): + input_sql = f'{stb_name},{id}={tb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9} {ts}' + if id_noexist_tag is not None: + input_sql = f'{stb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9} {ts}' + if ct_add_tag is not None: + input_sql = f'{stb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8},t9={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9} {ts}' + if id_change_tag is not None: + input_sql = f'{stb_name},t0={t0},t1={t1},{id}={tb_name},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9} {ts}' + if id_double_tag is not None: + input_sql = f'{stb_name},{id}=\"{tb_name}_1\",t0={t0},t1={t1},{id}=\"{tb_name}_2\",t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9} {ts}' + if ct_add_tag is not None: + input_sql = f'{stb_name},{id}={tb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8},t11={t1},t10={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9},c11={c8},c10={t0} {ts}' + if ct_am_tag is not None: + input_sql = f'{stb_name},{id}={tb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9},c11={c8},c10={t0} {ts}' + if id_noexist_tag is not None: + input_sql = f'{stb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9},c11={c8},c10={t0} {ts}' + if ct_ma_tag is not None: + input_sql = f'{stb_name},{id}={tb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8},t11={t1},t10={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6} {ts}' + if id_noexist_tag is not None: + input_sql = f'{stb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8},t11={t1},t10={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6} {ts}' + if ct_min_tag is not None: + input_sql = f'{stb_name},{id}={tb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6} {ts}' + if c_multi_tag is not None: + input_sql = f'{stb_name},{id}={tb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9} c10={c9} {ts}' + if t_multi_tag is not None: + input_sql = f'{stb_name},{id}={tb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8} t9={t8} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9} {ts}' + if c_blank_tag is not None: + input_sql = f'{stb_name},{id}={tb_name},t0={t0},t1={t1},t2={t2},t3={t3},t4={t4},t5={t5},t6={t6},t7={t7},t8={t8} {ts}' + if t_blank_tag is not None: + input_sql = f'{stb_name} c0={c0},c1={c1},c2={c2},c3={c3},c4={c4},c5={c5},c6={c6},c7={c7},c8={c8},c9={c9} {ts}' + if chinese_tag is not None: + input_sql = f'{stb_name},to=L"涛思数据" c0=L"涛思数据" {ts}' + return input_sql + + def genFullTypeSql(self, stb_name="", tb_name="", value="", t0="", t1="127i8", t2="32767i16", t3="2147483647i32", + t4="9223372036854775807i64", t5="11.12345f32", t6="22.123456789f64", t7="\"binaryTagValue\"", + t8="L\"ncharTagValue\"", c0="", c1="127i8", c2="32767i16", c3="2147483647i32", + c4="9223372036854775807i64", c5="11.12345f32", c6="22.123456789f64", c7="\"binaryColValue\"", + c8="L\"ncharColValue\"", c9="7u64", ts=None, + id_noexist_tag=None, id_change_tag=None, id_upper_tag=None, id_mixul_tag=None, id_double_tag=None, + ct_add_tag=None, ct_am_tag=None, ct_ma_tag=None, ct_min_tag=None, c_multi_tag=None, t_multi_tag=None, + c_blank_tag=None, t_blank_tag=None, chinese_tag=None, t_add_tag=None, t_mul_tag=None, point_trans_tag=None, + tcp_keyword_tag=None, multi_field_tag=None, protocol=None): + if stb_name == "": + stb_name = tdCom.getLongName(6, "letters") + if tb_name == "": + tb_name = f'{stb_name}_{random.randint(0, 65535)}_{random.randint(0, 65535)}' + if t0 == "": + t0 = "t" + if c0 == "": + c0 = random.choice(["f", "F", "false", "False", "t", "T", "true", "True"]) + if value == "": + value = random.choice(["f", "F", "false", "False", "t", "T", "true", "True", "TRUE", "FALSE"]) + if id_upper_tag is not None: + id = "ID" + else: + id = "id" + if id_mixul_tag is not None: + id = random.choice(["iD", "Id"]) + else: + id = "id" + if ts is None: + ts = "1626006833639000000" + input_sql = self.gen_influxdb_line(stb_name, tb_name, id, t0, t1, t2, t3, t4, t5, t6, t7, t8, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, ts, + id_noexist_tag, id_change_tag, id_double_tag, ct_add_tag, ct_am_tag, ct_ma_tag, ct_min_tag, c_multi_tag, t_multi_tag, c_blank_tag, t_blank_tag, chinese_tag) + return input_sql, stb_name + + def genMulTagColStr(self, gen_type, count): + """ + gen_type must be "tag"/"col" + """ + if gen_type == "tag": + return ','.join(map(lambda i: f't{i}=f', range(count))) + " " + if gen_type == "col": + return ','.join(map(lambda i: f'c{i}=t', range(count))) + " " + + def genLongSql(self, tag_count, col_count): + stb_name = tdCom.getLongName(7, "letters") + tb_name = f'{stb_name}_1' + tag_str = self.genMulTagColStr("tag", tag_count) + col_str = self.genMulTagColStr("col", col_count) + ts = "1626006833640000000" + long_sql = stb_name + ',' + f'id={tb_name}' + ',' + tag_str + col_str + ts + return long_sql, stb_name + + def getNoIdTbName(self, stb_name): + query_sql = f"select tbname from {stb_name}" + tb_name = self.resHandle(query_sql, True)[0][0] + return tb_name + + def resHandle(self, query_sql, query_tag, protocol=None): + tdSql.execute('reset query cache') + if protocol == "telnet-tcp": + time.sleep(0.5) + row_info = tdSql.query(query_sql, query_tag) + col_info = tdSql.getColNameList(query_sql, query_tag) + res_row_list = [] + sub_list = [] + for row_mem in row_info: + for i in row_mem: + if "11.1234" in str(i) and str(i) != "11.12345f32" and str(i) != "11.12345027923584F32": + sub_list.append("11.12345027923584") + elif "22.1234" in str(i) and str(i) != "22.123456789f64" and str(i) != "22.123456789F64": + sub_list.append("22.123456789") + else: + sub_list.append(str(i)) + res_row_list.append(sub_list) + res_field_list_without_ts = col_info[0][1:] + res_type_list = col_info[1] + return res_row_list, res_field_list_without_ts, res_type_list + + def resCmp(self, input_sql, stb_name, query_sql="select * from", condition="", ts=None, id=True, none_check_tag=None, ts_type=None, precision=None): + expect_list = self.inputHandle(input_sql, ts_type) + if precision == None: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, ts_type) + else: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, precision) + query_sql = f"{query_sql} {stb_name} {condition}" + res_row_list, res_field_list_without_ts, res_type_list = self.resHandle(query_sql, True) + if ts == 0: + res_ts = self.dateToTs(res_row_list[0][0]) + current_time = time.time() + if current_time - res_ts < 60: + tdSql.checkEqual(res_row_list[0][1:], expect_list[0][1:]) + else: + print("timeout") + tdSql.checkEqual(res_row_list[0], expect_list[0]) + else: + if none_check_tag is not None: + none_index_list = [i for i,x in enumerate(res_row_list[0]) if x=="None"] + none_index_list.reverse() + for j in none_index_list: + res_row_list[0].pop(j) + expect_list[0].pop(j) + tdSql.checkEqual(sorted(res_row_list[0]), sorted(expect_list[0])) + tdSql.checkEqual(sorted(res_field_list_without_ts), sorted(expect_list[1])) + tdSql.checkEqual(res_type_list, expect_list[2]) + + def cleanStb(self): + query_sql = "show stables" + res_row_list = tdSql.query(query_sql, True) + stb_list = map(lambda x: x[0], res_row_list) + for stb in stb_list: + tdSql.execute(f'drop table if exists {stb}') + + def initCheckCase(self): + """ + normal tags and cols, one for every elm + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql() + self.resCmp(input_sql, stb_name) + + def boolTypeCheckCase(self): + """ + check all normal type + """ + tdCom.cleanTb() + full_type_list = ["f", "F", "false", "False", "t", "T", "true", "True"] + for t_type in full_type_list: + input_sql, stb_name = self.genFullTypeSql(c0=t_type, t0=t_type) + self.resCmp(input_sql, stb_name) + + def symbolsCheckCase(self): + """ + check symbols = `~!@#$%^&*()_-+={[}]\|:;'\",<.>/? + """ + ''' + please test : + binary_symbols = '\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal"\'\'"\"' + ''' + tdCom.cleanTb() + binary_symbols = '"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal"' + nchar_symbols = f'L{binary_symbols}' + input_sql, stb_name = self.genFullTypeSql(c7=binary_symbols, c8=nchar_symbols, t7=binary_symbols, t8=nchar_symbols) + self.resCmp(input_sql, stb_name) + + def tsCheckCase(self): + """ + test ts list --> ["1626006833639000000", "1626006833639019us", "1626006833640ms", "1626006834s", "1626006822639022"] + # ! us级时间戳都为0时，数据库中查询显示，但python接口拿到的结果不显示 .000000的情况请确认，目前修改时间处理代码可以通过 + """ + tdCom.cleanTb() + ts_list = ["1626006833639000000", "1626006833639019us", "1626006833640ms", "1626006834s", "1626006822639022", 0] + for ts in ts_list: + input_sql, stb_name = self.genFullTypeSql(ts=ts) + self.resCmp(input_sql, stb_name, ts=ts) + + def idSeqCheckCase(self): + """ + check id.index in tags + eg: t0=**,id=**,t1=** + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql(id_change_tag=True) + self.resCmp(input_sql, stb_name) + + def idUpperCheckCase(self): + """ + check id param + eg: id and ID + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql(id_upper_tag=True) + self.resCmp(input_sql, stb_name) + input_sql, stb_name = self.genFullTypeSql(id_change_tag=True, id_upper_tag=True) + self.resCmp(input_sql, stb_name) + + def noIdCheckCase(self): + """ + id not exist + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql(id_noexist_tag=True) + self.resCmp(input_sql, stb_name) + query_sql = f"select tbname from {stb_name}" + res_row_list = self.resHandle(query_sql, True)[0] + if len(res_row_list[0][0]) > 0: + tdSql.checkColNameList(res_row_list, res_row_list) + else: + tdSql.checkColNameList(res_row_list, "please check noIdCheckCase") + + def maxColTagCheckCase(self): + """ + max tag count is 128 + max col count is ?? + """ + for input_sql in [self.genLongSql(127, 1)[0], self.genLongSql(1, 4093)[0]]: + tdCom.cleanTb() + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + for input_sql in [self.genLongSql(129, 1)[0], self.genLongSql(1, 4095)[0]]: + tdCom.cleanTb() + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def idIllegalNameCheckCase(self): + """ + test illegal id name + mix "~!@#$¥%^&*()-+|[]、「」【】;:《》<>?" + """ + tdCom.cleanTb() + rstr = list("~!@#$¥%^&*()-+|[]、「」【】;:《》<>?") + for i in rstr: + stb_name=f"aaa{i}bbb" + input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name) + self.resCmp(input_sql, f'`{stb_name}`') + tdSql.execute(f'drop table if exists `{stb_name}`') + + def idStartWithNumCheckCase(self): + """ + id is start with num + """ + tdCom.cleanTb() + input_sql = self.genFullTypeSql(tb_name=f"\"1aaabbb\"")[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def nowTsCheckCase(self): + """ + check now unsupported + """ + tdCom.cleanTb() + input_sql = self.genFullTypeSql(ts="now")[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def dateFormatTsCheckCase(self): + """ + check date format ts unsupported + """ + tdCom.cleanTb() + input_sql = self.genFullTypeSql(ts="2021-07-21\ 19:01:46.920")[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def illegalTsCheckCase(self): + """ + check ts format like 16260068336390us19 + """ + tdCom.cleanTb() + input_sql = self.genFullTypeSql(ts="16260068336390us19")[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def tagValueLengthCheckCase(self): + """ + check full type tag value limit + """ + tdCom.cleanTb() + # i8 + for t1 in ["-128i8", "127i8"]: + input_sql, stb_name = self.genFullTypeSql(t1=t1) + self.resCmp(input_sql, stb_name) + for t1 in ["-129i8", "128i8"]: + input_sql = self.genFullTypeSql(t1=t1)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + #i16 + for t2 in ["-32768i16", "32767i16"]: + input_sql, stb_name = self.genFullTypeSql(t2=t2) + self.resCmp(input_sql, stb_name) + for t2 in ["-32769i16", "32768i16"]: + input_sql = self.genFullTypeSql(t2=t2)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + #i32 + for t3 in ["-2147483648i32", "2147483647i32"]: + input_sql, stb_name = self.genFullTypeSql(t3=t3) + self.resCmp(input_sql, stb_name) + for t3 in ["-2147483649i32", "2147483648i32"]: + input_sql = self.genFullTypeSql(t3=t3)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + #i64 + for t4 in ["-9223372036854775808i64", "9223372036854775807i64"]: + input_sql, stb_name = self.genFullTypeSql(t4=t4) + self.resCmp(input_sql, stb_name) + for t4 in ["-9223372036854775809i64", "9223372036854775808i64"]: + input_sql = self.genFullTypeSql(t4=t4)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # f32 + for t5 in [f"{-3.4028234663852885981170418348451692544*(10**38)}f32", f"{3.4028234663852885981170418348451692544*(10**38)}f32"]: + input_sql, stb_name = self.genFullTypeSql(t5=t5) + self.resCmp(input_sql, stb_name) + # * limit set to 4028234664*(10**38) + for t5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: + input_sql = self.genFullTypeSql(t5=t5)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # f64 + for t6 in [f'{-1.79769*(10**308)}f64', f'{-1.79769*(10**308)}f64']: + input_sql, stb_name = self.genFullTypeSql(t6=t6) + self.resCmp(input_sql, stb_name) + # * limit set to 1.797693134862316*(10**308) + for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: + input_sql = self.genFullTypeSql(c6=c6)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # binary + stb_name = tdCom.getLongName(7, "letters") + input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(16374, "letters")}" c0=f 1626006833639000000' + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(16375, "letters")}" c0=f 1626006833639000000' + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # nchar + # * legal nchar could not be larger than 16374/4 + stb_name = tdCom.getLongName(7, "letters") + input_sql = f'{stb_name},t0=t,t1=L"{tdCom.getLongName(4093, "letters")}" c0=f 1626006833639000000' + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + input_sql = f'{stb_name},t0=t,t1=L"{tdCom.getLongName(4094, "letters")}" c0=f 1626006833639000000' + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def colValueLengthCheckCase(self): + """ + check full type col value limit + """ + tdCom.cleanTb() + # i8 + for c1 in ["-128i8", "127i8"]: + input_sql, stb_name = self.genFullTypeSql(c1=c1) + self.resCmp(input_sql, stb_name) + + for c1 in ["-129i8", "128i8"]: + input_sql = self.genFullTypeSql(c1=c1)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + # i16 + for c2 in ["-32768i16"]: + input_sql, stb_name = self.genFullTypeSql(c2=c2) + self.resCmp(input_sql, stb_name) + for c2 in ["-32769i16", "32768i16"]: + input_sql = self.genFullTypeSql(c2=c2)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # i32 + for c3 in ["-2147483648i32"]: + input_sql, stb_name = self.genFullTypeSql(c3=c3) + self.resCmp(input_sql, stb_name) + for c3 in ["-2147483649i32", "2147483648i32"]: + input_sql = self.genFullTypeSql(c3=c3)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # i64 + for c4 in ["-9223372036854775808i64"]: + input_sql, stb_name = self.genFullTypeSql(c4=c4) + self.resCmp(input_sql, stb_name) + for c4 in ["-9223372036854775809i64", "9223372036854775808i64"]: + input_sql = self.genFullTypeSql(c4=c4)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # f32 + for c5 in [f"{-3.4028234663852885981170418348451692544*(10**38)}f32", f"{3.4028234663852885981170418348451692544*(10**38)}f32"]: + input_sql, stb_name = self.genFullTypeSql(c5=c5) + self.resCmp(input_sql, stb_name) + # * limit set to 4028234664*(10**38) + for c5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: + input_sql = self.genFullTypeSql(c5=c5)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # f64 + for c6 in [f'{-1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308)}f64', f'{-1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308)}f64']: + input_sql, stb_name = self.genFullTypeSql(c6=c6) + self.resCmp(input_sql, stb_name) + # * limit set to 1.797693134862316*(10**308) + for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: + input_sql = self.genFullTypeSql(c6=c6)[0] + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # # # binary + # stb_name = tdCom.getLongName(7, "letters") + # input_sql = f'{stb_name},t0=t c0=f,c1="{tdCom.getLongName(16374, "letters")}" 1626006833639000000' + # self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + # input_sql = f'{stb_name},t0=t c0=f,c1="{tdCom.getLongName(16375, "letters")}" 1626006833639000000' + # try: + # self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + # except SchemalessError as err: + # tdSql.checkNotEqual(err.errno, 0) + + # # nchar + # # * legal nchar could not be larger than 16374/4 + # stb_name = tdCom.getLongName(7, "letters") + # input_sql = f'{stb_name},t0=t c0=f,c1=L"{tdCom.getLongName(4093, "letters")}" 1626006833639000000' + # self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + # input_sql = f'{stb_name},t0=t c0=f,c1=L"{tdCom.getLongName(4094, "letters")}" 1626006833639000000' + # try: + # self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + # except SchemalessError as err: + # tdSql.checkNotEqual(err.errno, 0) + + def tagColIllegalValueCheckCase(self): + + """ + test illegal tag col value + """ + tdCom.cleanTb() + # bool + for i in ["TrUe", "tRue", "trUe", "truE", "FalsE", "fAlse", "faLse", "falSe", "falsE"]: + input_sql1 = self.genFullTypeSql(t0=i)[0] + try: + self._conn.schemaless_insert([input_sql1], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + input_sql2 = self.genFullTypeSql(c0=i)[0] + try: + self._conn.schemaless_insert([input_sql2], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # i8 i16 i32 i64 f32 f64 + for input_sql in [ + self.genFullTypeSql(t1="1s2i8")[0], + self.genFullTypeSql(t2="1s2i16")[0], + self.genFullTypeSql(t3="1s2i32")[0], + self.genFullTypeSql(t4="1s2i64")[0], + self.genFullTypeSql(t5="11.1s45f32")[0], + self.genFullTypeSql(t6="11.1s45f64")[0], + self.genFullTypeSql(c1="1s2i8")[0], + self.genFullTypeSql(c2="1s2i16")[0], + self.genFullTypeSql(c3="1s2i32")[0], + self.genFullTypeSql(c4="1s2i64")[0], + self.genFullTypeSql(c5="11.1s45f32")[0], + self.genFullTypeSql(c6="11.1s45f64")[0], + self.genFullTypeSql(c9="1s1u64")[0] + ]: + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # check binary and nchar blank + stb_name = tdCom.getLongName(7, "letters") + input_sql1 = f'{stb_name},t0=t c0=f,c1="abc aaa" 1626006833639000000' + input_sql2 = f'{stb_name},t0=t c0=f,c1=L"abc aaa" 1626006833639000000' + input_sql3 = f'{stb_name},t0=t,t1="abc aaa" c0=f 1626006833639000000' + input_sql4 = f'{stb_name},t0=t,t1=L"abc aaa" c0=f 1626006833639000000' + for input_sql in [input_sql1, input_sql2, input_sql3, input_sql4]: + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # check accepted binary and nchar symbols + # # * ~!@#$¥%^&*()-+={}|[]、「」:; + for symbol in list('~!@#$¥%^&*()-+={}|[]、「」:;'): + input_sql1 = f'{stb_name},t0=t c0=f,c1="abc{symbol}aaa" 1626006833639000000' + input_sql2 = f'{stb_name},t0=t,t1="abc{symbol}aaa" c0=f 1626006833639000000' + self._conn.schemaless_insert([input_sql1], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + # self._conn.schemaless_insert([input_sql2], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + def duplicateIdTagColInsertCheckCase(self): + """ + check duplicate Id Tag Col + """ + tdCom.cleanTb() + input_sql_id = self.genFullTypeSql(id_double_tag=True)[0] + try: + self._conn.schemaless_insert([input_sql_id], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + input_sql = self.genFullTypeSql()[0] + input_sql_tag = input_sql.replace("t5", "t6") + try: + self._conn.schemaless_insert([input_sql_tag], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + input_sql = self.genFullTypeSql()[0] + input_sql_col = input_sql.replace("c5", "c6") + try: + self._conn.schemaless_insert([input_sql_col], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + input_sql = self.genFullTypeSql()[0] + input_sql_col = input_sql.replace("c5", "C6") + try: + self._conn.schemaless_insert([input_sql_col], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + ##### stb exist ##### + @tdCom.smlPass + def noIdStbExistCheckCase(self): + """ + case no id when stb exist + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql(tb_name="sub_table_0123456", t0="f", c0="f") + self.resCmp(input_sql, stb_name) + input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, id_noexist_tag=True, t0="f", c0="f") + self.resCmp(input_sql, stb_name, condition='where tbname like "t_%"') + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + # TODO cover other case + + def duplicateInsertExistCheckCase(self): + """ + check duplicate insert when stb exist + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql() + self.resCmp(input_sql, stb_name) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + self.resCmp(input_sql, stb_name) + + @tdCom.smlPass + def tagColBinaryNcharLengthCheckCase(self): + """ + check length increase + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql() + self.resCmp(input_sql, stb_name) + tb_name = tdCom.getLongName(5, "letters") + input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, tb_name=tb_name,t7="\"binaryTagValuebinaryTagValue\"", t8="L\"ncharTagValuencharTagValue\"", c7="\"binaryTagValuebinaryTagValue\"", c8="L\"ncharTagValuencharTagValue\"") + self.resCmp(input_sql, stb_name, condition=f'where tbname like "{tb_name}"') + + @tdCom.smlPass + def tagColAddDupIDCheckCase(self): + """ + check column and tag count add, stb and tb duplicate + * tag: alter table ... + * col: when update==0 and ts is same, unchange + * so this case tag&&value will be added, + * col is added without value when update==0 + * col is added with value when update==1 + """ + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + for db_update_tag in [0, 1]: + if db_update_tag == 1 : + self.createDb("test_update", db_update_tag=db_update_tag) + input_sql, stb_name = self.genFullTypeSql(tb_name=tb_name, t0="f", c0="f") + self.resCmp(input_sql, stb_name) + self.genFullTypeSql(stb_name=stb_name, tb_name=tb_name, t0="f", c0="f", ct_add_tag=True) + if db_update_tag == 1 : + self.resCmp(input_sql, stb_name, condition=f'where tbname like "{tb_name}"') + else: + self.resCmp(input_sql, stb_name, condition=f'where tbname like "{tb_name}"', none_check_tag=True) + @tdCom.smlPass + def tagColAddCheckCase(self): + """ + check column and tag count add + """ + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_sql, stb_name = self.genFullTypeSql(tb_name=tb_name, t0="f", c0="f") + self.resCmp(input_sql, stb_name) + tb_name_1 = tdCom.getLongName(7, "letters") + input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, tb_name=tb_name_1, t0="f", c0="f", ct_add_tag=True) + self.resCmp(input_sql, stb_name, condition=f'where tbname like "{tb_name_1}"') + res_row_list = self.resHandle(f"select c10,c11,t10,t11 from {tb_name}", True)[0] + tdSql.checkEqual(res_row_list[0], ['None', 'None', 'None', 'None']) + self.resCmp(input_sql, stb_name, condition=f'where tbname like "{tb_name}"', none_check_tag=True) + + def tagMd5Check(self): + """ + condition: stb not change + insert two table, keep tag unchange, change col + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql(t0="f", c0="f", id_noexist_tag=True) + self.resCmp(input_sql, stb_name) + tb_name1 = self.getNoIdTbName(stb_name) + input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, t0="f", c0="f", id_noexist_tag=True) + self.resCmp(input_sql, stb_name) + tb_name2 = self.getNoIdTbName(stb_name) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(1) + tdSql.checkEqual(tb_name1, tb_name2) + input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, t0="f", c0="f", id_noexist_tag=True, ct_add_tag=True) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + tb_name3 = self.getNoIdTbName(stb_name) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + tdSql.checkNotEqual(tb_name1, tb_name3) + + # * tag binary max is 16384, col+ts binary max 49151 + def tagColBinaryMaxLengthCheckCase(self): + """ + every binary and nchar must be length+2 + """ + tdCom.cleanTb() + stb_name = tdCom.getLongName(7, "letters") + tb_name = f'{stb_name}_1' + input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000' + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + # * every binary and nchar must be length+2, so here is two tag, max length could not larger than 16384-2*2 + input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(16374, "letters")}",t2="{tdCom.getLongName(5, "letters")}" c0=f 1626006833639000000' + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(16374, "letters")}",t2="{tdCom.getLongName(6, "letters")}" c0=f 1626006833639000000' + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + + # # * check col，col+ts max in describe ---> 16143 + input_sql = f'{stb_name},t0=t c0=f,c1="{tdCom.getLongName(16374, "letters")}",c2="{tdCom.getLongName(16374, "letters")}",c3="{tdCom.getLongName(16374, "letters")}",c4="{tdCom.getLongName(12, "letters")}" 1626006833639000000' + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(3) + input_sql = f'{stb_name},t0=t c0=f,c1="{tdCom.getLongName(16374, "letters")}",c2="{tdCom.getLongName(16374, "letters")}",c3="{tdCom.getLongName(16374, "letters")}",c4="{tdCom.getLongName(13, "letters")}" 1626006833639000000' + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(3) + + # * tag nchar max is 16374/4, col+ts nchar max 49151 + def tagColNcharMaxLengthCheckCase(self): + """ + check nchar length limit + """ + tdCom.cleanTb() + stb_name = tdCom.getLongName(7, "letters") + tb_name = f'{stb_name}_1' + input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000' + code = self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + # * legal nchar could not be larger than 16374/4 + input_sql = f'{stb_name},t0=t,t1=L"{tdCom.getLongName(4093, "letters")}",t2=L"{tdCom.getLongName(1, "letters")}" c0=f 1626006833639000000' + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + input_sql = f'{stb_name},t0=t,t1=L"{tdCom.getLongName(4093, "letters")}",t2=L"{tdCom.getLongName(2, "letters")}" c0=f 1626006833639000000' + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + + input_sql = f'{stb_name},t0=t c0=f,c1=L"{tdCom.getLongName(4093, "letters")}",c2=L"{tdCom.getLongName(4093, "letters")}",c3=L"{tdCom.getLongName(4093, "letters")}",c4=L"{tdCom.getLongName(4, "letters")}" 1626006833639000000' + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(3) + input_sql = f'{stb_name},t0=t c0=f,c1=L"{tdCom.getLongName(4093, "letters")}",c2=L"{tdCom.getLongName(4093, "letters")}",c3=L"{tdCom.getLongName(4093, "letters")}",c4=L"{tdCom.getLongName(5, "letters")}" 1626006833639000000' + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(3) + + def batchInsertCheckCase(self): + """ + test batch insert + """ + tdCom.cleanTb() + stb_name = tdCom.getLongName(8, "letters") + # tdSql.execute(f'create stable {stb_name}(ts timestamp, f int) tags(t1 bigint)') + lines = ["st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", + "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000", + f"{stb_name},t2=5f64,t3=L\"ste\" c1=true,c2=4i64,c3=\"iam\" 1626056811823316532", + "stf567890,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000", + "st123456,t1=4i64,t2=5f64,t3=\"t4\" c1=3i64,c3=L\"passitagain\",c2=true,c4=5f64 1626006833642000000", + f"{stb_name},t2=5f64,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532", + f"{stb_name},t2=5f64,t3=L\"ste2\" c3=\"iamszhou\",c4=false,c5=32i8,c6=64i16,c7=32i32,c8=88.88f32 1626056812843316532", + "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000", + "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000" + ] + self._conn.schemaless_insert(lines, TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + def multiInsertCheckCase(self, count): + """ + test multi insert + """ + tdCom.cleanTb() + sql_list = [] + stb_name = tdCom.getLongName(8, "letters") + # tdSql.execute(f'create stable {stb_name}(ts timestamp, f int) tags(t1 bigint)') + for i in range(count): + input_sql = self.genFullTypeSql(stb_name=stb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', id_noexist_tag=True)[0] + sql_list.append(input_sql) + print(sql_list) + self._conn.schemaless_insert(sql_list, TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + def batchErrorInsertCheckCase(self): + """ + test batch error insert + """ + tdCom.cleanTb() + stb_name = tdCom.getLongName(8, "letters") + lines = ["st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", + f"{stb_name},t2=5f64,t3=L\"ste\" c1=tRue,c2=4i64,c3=\"iam\" 1626056811823316532ns"] + try: + self._conn.schemaless_insert(lines, TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def genSqlList(self, count=5, stb_name="", tb_name=""): + """ + stb --> supertable + tb --> table + ts --> timestamp, same default + col --> column, same default + tag --> tag, same default + d --> different + s --> same + a --> add + m --> minus + """ + d_stb_d_tb_list = list() + s_stb_s_tb_list = list() + s_stb_s_tb_a_col_a_tag_list = list() + s_stb_s_tb_m_col_m_tag_list = list() + s_stb_d_tb_list = list() + s_stb_d_tb_a_col_m_tag_list = list() + s_stb_d_tb_a_tag_m_col_list = list() + s_stb_s_tb_d_ts_list = list() + s_stb_s_tb_d_ts_a_col_m_tag_list = list() + s_stb_s_tb_d_ts_a_tag_m_col_list = list() + s_stb_d_tb_d_ts_list = list() + s_stb_d_tb_d_ts_a_col_m_tag_list = list() + s_stb_d_tb_d_ts_a_tag_m_col_list = list() + for i in range(count): + d_stb_d_tb_list.append(self.genFullTypeSql(t0="f", c0="f")) + s_stb_s_tb_list.append(self.genFullTypeSql(stb_name=stb_name, tb_name=tb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"')) + s_stb_s_tb_a_col_a_tag_list.append(self.genFullTypeSql(stb_name=stb_name, tb_name=tb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', ct_add_tag=True)) + s_stb_s_tb_m_col_m_tag_list.append(self.genFullTypeSql(stb_name=stb_name, tb_name=tb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', ct_min_tag=True)) + s_stb_d_tb_list.append(self.genFullTypeSql(stb_name=stb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', id_noexist_tag=True)) + s_stb_d_tb_a_col_m_tag_list.append(self.genFullTypeSql(stb_name=stb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', id_noexist_tag=True, ct_am_tag=True)) + s_stb_d_tb_a_tag_m_col_list.append(self.genFullTypeSql(stb_name=stb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', id_noexist_tag=True, ct_ma_tag=True)) + s_stb_s_tb_d_ts_list.append(self.genFullTypeSql(stb_name=stb_name, tb_name=tb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', ts=0)) + s_stb_s_tb_d_ts_a_col_m_tag_list.append(self.genFullTypeSql(stb_name=stb_name, tb_name=tb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', ts=0, ct_am_tag=True)) + s_stb_s_tb_d_ts_a_tag_m_col_list.append(self.genFullTypeSql(stb_name=stb_name, tb_name=tb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', ts=0, ct_ma_tag=True)) + s_stb_d_tb_d_ts_list.append(self.genFullTypeSql(stb_name=stb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', id_noexist_tag=True, ts=0)) + s_stb_d_tb_d_ts_a_col_m_tag_list.append(self.genFullTypeSql(stb_name=stb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', id_noexist_tag=True, ts=0, ct_am_tag=True)) + s_stb_d_tb_d_ts_a_tag_m_col_list.append(self.genFullTypeSql(stb_name=stb_name, t7=f'"{tdCom.getLongName(8, "letters")}"', c7=f'"{tdCom.getLongName(8, "letters")}"', id_noexist_tag=True, ts=0, ct_ma_tag=True)) + + return d_stb_d_tb_list, s_stb_s_tb_list, s_stb_s_tb_a_col_a_tag_list, s_stb_s_tb_m_col_m_tag_list, \ + s_stb_d_tb_list, s_stb_d_tb_a_col_m_tag_list, s_stb_d_tb_a_tag_m_col_list, s_stb_s_tb_d_ts_list, \ + s_stb_s_tb_d_ts_a_col_m_tag_list, s_stb_s_tb_d_ts_a_tag_m_col_list, s_stb_d_tb_d_ts_list, \ + s_stb_d_tb_d_ts_a_col_m_tag_list, s_stb_d_tb_d_ts_a_tag_m_col_list + + + def genMultiThreadSeq(self, sql_list): + tlist = list() + for insert_sql in sql_list: + t = threading.Thread(target=self._conn.schemaless_insert, args=([insert_sql[0]], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value,)) + tlist.append(t) + return tlist + + def multiThreadRun(self, tlist): + for t in tlist: + t.start() + for t in tlist: + t.join() + + def stbInsertMultiThreadCheckCase(self): + """ + thread input different stb + """ + tdCom.cleanTb() + input_sql = self.genSqlList()[0] + self.multiThreadRun(self.genMultiThreadSeq(input_sql)) + tdSql.query(f"show tables;") + tdSql.checkRows(5) + + def sStbStbDdataInsertMultiThreadCheckCase(self): + """ + thread input same stb tb, different data, result keep first data + """ + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_sql, stb_name = self.genFullTypeSql(tb_name=tb_name) + self.resCmp(input_sql, stb_name) + s_stb_s_tb_list = self.genSqlList(stb_name=stb_name, tb_name=tb_name)[1] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + expected_tb_name = self.getNoIdTbName(stb_name)[0] + tdSql.checkEqual(tb_name, expected_tb_name) + tdSql.query(f"select * from {stb_name};") + tdSql.checkRows(1) + + def sStbStbDdataAtcInsertMultiThreadCheckCase(self): + """ + thread input same stb tb, different data, add columes and tags, result keep first data + """ + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_sql, stb_name = self.genFullTypeSql(tb_name=tb_name) + self.resCmp(input_sql, stb_name) + s_stb_s_tb_a_col_a_tag_list = self.genSqlList(stb_name=stb_name, tb_name=tb_name)[2] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_a_col_a_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + expected_tb_name = self.getNoIdTbName(stb_name)[0] + tdSql.checkEqual(tb_name, expected_tb_name) + tdSql.query(f"select * from {stb_name};") + tdSql.checkRows(1) + + def sStbStbDdataMtcInsertMultiThreadCheckCase(self): + """ + thread input same stb tb, different data, minus columes and tags, result keep first data + """ + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_sql, stb_name = self.genFullTypeSql(tb_name=tb_name) + self.resCmp(input_sql, stb_name) + s_stb_s_tb_m_col_m_tag_list = self.genSqlList(stb_name=stb_name, tb_name=tb_name)[3] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_m_col_m_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + expected_tb_name = self.getNoIdTbName(stb_name)[0] + tdSql.checkEqual(tb_name, expected_tb_name) + tdSql.query(f"select * from {stb_name};") + tdSql.checkRows(1) + + def sStbDtbDdataInsertMultiThreadCheckCase(self): + """ + thread input same stb, different tb, different data + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql() + self.resCmp(input_sql, stb_name) + s_stb_d_tb_list = self.genSqlList(stb_name=stb_name)[4] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(6) + + def sStbDtbDdataAcMtInsertMultiThreadCheckCase(self): + """ + #! concurrency conflict + """ + """ + thread input same stb, different tb, different data, add col, mul tag + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql() + self.resCmp(input_sql, stb_name) + s_stb_d_tb_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[5] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_a_col_m_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(6) + + def sStbDtbDdataAtMcInsertMultiThreadCheckCase(self): + """ + #! concurrency conflict + """ + """ + thread input same stb, different tb, different data, add tag, mul col + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql() + self.resCmp(input_sql, stb_name) + s_stb_d_tb_a_tag_m_col_list = self.genSqlList(stb_name=stb_name)[6] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_a_tag_m_col_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(6) + + def sStbStbDdataDtsInsertMultiThreadCheckCase(self): + """ + thread input same stb tb, different ts + """ + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_sql, stb_name = self.genFullTypeSql(tb_name=tb_name) + self.resCmp(input_sql, stb_name) + s_stb_s_tb_d_ts_list = self.genSqlList(stb_name=stb_name, tb_name=tb_name)[7] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_d_ts_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(6) + + def sStbStbDdataDtsAcMtInsertMultiThreadCheckCase(self): + """ + thread input same stb tb, different ts, add col, mul tag + """ + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_sql, stb_name = self.genFullTypeSql(tb_name=tb_name) + self.resCmp(input_sql, stb_name) + s_stb_s_tb_d_ts_a_col_m_tag_list = self.genSqlList(stb_name=stb_name, tb_name=tb_name)[8] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_d_ts_a_col_m_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(6) + tdSql.query(f"select * from {stb_name} where t8 is not NULL") + tdSql.checkRows(6) + tdSql.query(f"select * from {tb_name} where c11 is not NULL;") + tdSql.checkRows(5) + + def sStbStbDdataDtsAtMcInsertMultiThreadCheckCase(self): + """ + thread input same stb tb, different ts, add tag, mul col + """ + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_sql, stb_name = self.genFullTypeSql(tb_name=tb_name) + self.resCmp(input_sql, stb_name) + s_stb_s_tb_d_ts_a_tag_m_col_list = self.genSqlList(stb_name=stb_name, tb_name=tb_name)[9] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_d_ts_a_tag_m_col_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(6) + for c in ["c7", "c8", "c9"]: + tdSql.query(f"select * from {stb_name} where {c} is NULL") + tdSql.checkRows(5) + for t in ["t10", "t11"]: + tdSql.query(f"select * from {stb_name} where {t} is not NULL;") + tdSql.checkRows(6) + + def sStbDtbDdataDtsInsertMultiThreadCheckCase(self): + """ + thread input same stb, different tb, data, ts + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql() + self.resCmp(input_sql, stb_name) + s_stb_d_tb_d_ts_list = self.genSqlList(stb_name=stb_name)[10] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_d_ts_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(6) + + def sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase(self): + """ + # ! concurrency conflict + """ + """ + thread input same stb, different tb, data, ts, add col, mul tag + """ + tdCom.cleanTb() + input_sql, stb_name = self.genFullTypeSql() + self.resCmp(input_sql, stb_name) + s_stb_d_tb_d_ts_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[11] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_d_ts_a_col_m_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(6) + + def test(self): + input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" + input_sql2 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64 c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64 1626006933640000000ns" + try: + self._conn.insert_lines([input_sql1]) + self._conn.insert_lines([input_sql2]) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + # self._conn.insert_lines([input_sql2]) + # input_sql3 = f'abcd,id="cc¥Ec",t0=True,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7="ndsfdrum",t8=L"ncharTagValue" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7="igwoehkm",c8=L"ncharColValue",c9=7u64 0' + # print(input_sql3) + # input_sql4 = 'hmemeb,id="kilrcrldgf",t0=F,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7="fysodjql",t8=L"ncharTagValue" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7="waszbfvc",c8=L"ncharColValue",c9=7u64 0' + # code = self._conn.insert_lines([input_sql3]) + # print(code) + # self._conn.insert_lines([input_sql4]) + + def runAll(self): + self.initCheckCase() + self.boolTypeCheckCase() + self.symbolsCheckCase() + # self.tsCheckCase() + self.idSeqCheckCase() + self.idUpperCheckCase() + self.noIdCheckCase() + # self.maxColTagCheckCase() + self.idIllegalNameCheckCase() + self.idStartWithNumCheckCase() + self.nowTsCheckCase() + self.dateFormatTsCheckCase() + self.illegalTsCheckCase() + # self.tagValueLengthCheckCase() + self.colValueLengthCheckCase() + self.tagColIllegalValueCheckCase() + self.duplicateIdTagColInsertCheckCase() + self.noIdStbExistCheckCase() + self.duplicateInsertExistCheckCase() + self.tagColBinaryNcharLengthCheckCase() + self.tagColAddDupIDCheckCase() + self.tagColAddCheckCase() + self.tagMd5Check() + # self.tagColBinaryMaxLengthCheckCase() + # self.tagColNcharMaxLengthCheckCase() + self.batchInsertCheckCase() + self.multiInsertCheckCase(10) + self.batchErrorInsertCheckCase() + # MultiThreads + # self.stbInsertMultiThreadCheckCase() + # self.sStbStbDdataInsertMultiThreadCheckCase() + # self.sStbStbDdataAtcInsertMultiThreadCheckCase() + # self.sStbStbDdataMtcInsertMultiThreadCheckCase() + # self.sStbDtbDdataInsertMultiThreadCheckCase() + + # # # ! concurrency conflict + # # self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() + # # self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() + + # self.sStbStbDdataDtsInsertMultiThreadCheckCase() + + # # # ! concurrency conflict + # # self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() + # # self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() + + # self.sStbDtbDdataDtsInsertMultiThreadCheckCase() + + # # ! concurrency conflict + # # self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() + + + + def run(self): + print("running {}".format(__file__)) + self.createDb() + try: + self.runAll() + except Exception as err: + print(''.join(traceback.format_exception(None, err, err.__traceback__))) + raise err + # self.tagColIllegalValueCheckCase() + # self.test() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 7b6b3a72b18df0031b97b49605fccc5284ae87e0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 31 May 2022 17:05:27 +0800 Subject: [PATCH 04/57] fix send on broken rpc connection issue --- source/libs/scheduler/inc/schedulerInt.h | 9 ++-- source/libs/scheduler/src/schJob.c | 61 ++++++++++++++++++++---- source/libs/scheduler/src/schRemote.c | 20 +------- 3 files changed, 59 insertions(+), 31 deletions(-) diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 3302a4b61a..09a4f322d4 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -300,7 +300,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask); int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp); int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp); void schProcessOnDataFetched(SSchJob *job); -int32_t schGetTaskFromTaskList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); +int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask); int32_t schUpdateTaskExecNodeHandle(SSchTask *pTask, void *handle, int32_t rspCode); void schFreeRpcCtxVal(const void *arg); int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb); @@ -314,12 +314,11 @@ int32_t schCancelJob(SSchJob *pJob); int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode); uint64_t schGenTaskId(void); void schCloseJobRef(void); -int32_t schExecJob(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, - int64_t startTs, SSchResInfo *pRes); -int32_t schAsyncExecJob(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, - int64_t startTs, SSchResInfo *pRes); +int32_t schExecJob(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SSchResInfo *pRes); +int32_t schAsyncExecJob(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SSchResInfo *pRes); int32_t schFetchRows(SSchJob *pJob); int32_t schAsyncFetchRows(SSchJob *pJob); +int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, int32_t msgType, void *handle, int32_t rspCode); #ifdef __cplusplus diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 00bebcd5ac..1c60dcccfd 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -342,6 +342,36 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad return TSDB_CODE_SUCCESS; } +int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle) { + if (NULL == pTask->execNodes) { + return TSDB_CODE_SUCCESS; + } + + int32_t num = taosArrayGetSize(pTask->execNodes); + for (int32_t i = 0; i < num; ++i) { + SSchNodeInfo* pNode = taosArrayGet(pTask->execNodes, i); + if (pNode->handle == handle) { + taosArrayRemove(pTask->execNodes, i); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, int32_t msgType, void *handle, int32_t rspCode) { + SCH_SET_TASK_HANDLE(pTask, handle); + + schUpdateTaskExecNodeHandle(pTask, handle, rspCode); + + if (msgType == TDMT_SCH_LINK_BROKEN) { + schDropTaskExecNode(pJob, pTask, handle); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t schRecordQueryDataSrc(SSchJob *pJob, SSchTask *pTask) { if (!SCH_IS_DATA_SRC_QRY_TASK(pTask)) { return TSDB_CODE_SUCCESS; @@ -1009,19 +1039,19 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { */ for (int32_t i = 0; i < parentNum; ++i) { - SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i); - int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); + SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i); + int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); - SCH_LOCK(SCH_WRITE, &par->lock); + SCH_LOCK(SCH_WRITE, &parent->lock); SDownstreamSourceNode source = {.type = QUERY_NODE_DOWNSTREAM_SOURCE, .taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; - qSetSubplanExecutionNode(par->plan, pTask->plan->id.groupId, &source); - SCH_UNLOCK(SCH_WRITE, &par->lock); + qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); + SCH_UNLOCK(SCH_WRITE, &parent->lock); - if (SCH_TASK_READY_FOR_LAUNCH(readyNum, par)) { - SCH_ERR_RET(schLaunchTask(pJob, par)); + if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { + SCH_ERR_RET(schLaunchTask(pJob, parent)); } } @@ -1095,7 +1125,7 @@ int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp) { return TSDB_CODE_SUCCESS; } -int32_t schGetTaskFromTaskList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { +int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { int32_t s = taosHashGetSize(pTaskList); if (s <= 0) { return TSDB_CODE_SUCCESS; @@ -1111,6 +1141,21 @@ int32_t schGetTaskFromTaskList(SHashObj *pTaskList, uint64_t taskId, SSchTask ** return TSDB_CODE_SUCCESS; } +int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask) { + schGetTaskFromList(pJob->execTasks, taskId, pTask); + if (NULL == *pTask) { + schGetTaskFromList(pJob->succTasks, taskId, pTask); + + if (NULL == *pTask) { + SCH_JOB_ELOG("task not found in execList & succList, taskId:%" PRIx64, taskId); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + } + + return TSDB_CODE_SUCCESS; +} + + int32_t schUpdateTaskExecNodeHandle(SSchTask *pTask, void *handle, int32_t rspCode) { if (rspCode || NULL == pTask->execNodes || taosArrayGetSize(pTask->execNodes) > 1 || taosArrayGetSize(pTask->execNodes) <= 0) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 312d587b6f..c15649106e 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -358,27 +358,11 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } - schGetTaskFromTaskList(pJob->execTasks, pParam->taskId, &pTask); - if (NULL == pTask) { - if (TDMT_VND_EXPLAIN_RSP == msgType) { - schGetTaskFromTaskList(pJob->succTasks, pParam->taskId, &pTask); - } else { - SCH_JOB_ELOG("task not found in execTask list, refId:%" PRIx64 ", taskId:%" PRIx64, pParam->refId, - pParam->taskId); - SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); - } - } - - if (NULL == pTask) { - SCH_JOB_ELOG("task not found in execList & succList, refId:%" PRIx64 ", taskId:%" PRIx64, pParam->refId, - pParam->taskId); - SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); - } + SCH_ERR_JRET(schGetTaskInJob(pJob, pParam->taskId, &pTask)); SCH_TASK_DLOG("rsp msg received, type:%s, handle:%p, code:%s", TMSG_INFO(msgType), pMsg->handle, tstrerror(rspCode)); - SCH_SET_TASK_HANDLE(pTask, pMsg->handle); - schUpdateTaskExecNodeHandle(pTask, pMsg->handle, rspCode); + SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, msgType, pMsg->handle, rspCode)); SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); From 04ef9e15e6b38716b02aa5464b38fd6feb3e941f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 31 May 2022 17:35:28 +0800 Subject: [PATCH 05/57] refactor: make trans support global conflict --- source/dnode/mnode/impl/inc/mndDef.h | 71 +------- source/dnode/mnode/impl/inc/mndTrans.h | 4 +- source/dnode/mnode/impl/src/mndAcct.c | 2 +- source/dnode/mnode/impl/src/mndBnode.c | 4 +- source/dnode/mnode/impl/src/mndCluster.c | 7 +- source/dnode/mnode/impl/src/mndConsumer.c | 6 +- source/dnode/mnode/impl/src/mndDb.c | 6 +- source/dnode/mnode/impl/src/mndDnode.c | 10 +- source/dnode/mnode/impl/src/mndFunc.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 14 +- source/dnode/mnode/impl/src/mndOffset.c | 2 +- source/dnode/mnode/impl/src/mndQnode.c | 4 +- source/dnode/mnode/impl/src/mndSma.c | 6 +- source/dnode/mnode/impl/src/mndSnode.c | 4 +- source/dnode/mnode/impl/src/mndStb.c | 6 +- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndSubscribe.c | 4 +- source/dnode/mnode/impl/src/mndTopic.c | 4 +- source/dnode/mnode/impl/src/mndTrans.c | 166 +++--------------- source/dnode/mnode/impl/src/mndUser.c | 12 +- source/dnode/mnode/impl/test/trans/trans2.cpp | 14 +- tests/script/jenkins/basic.txt | 1 - tests/script/tsim/trans/create_db.sim | 4 +- tests/test/c/sdbDump.c | 3 +- 24 files changed, 94 insertions(+), 266 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9a60ad860f..6b9014aad0 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -54,9 +54,10 @@ typedef enum { } EAuthOp; typedef enum { - TRN_STEP_LOG = 1, - TRN_STEP_ACTION = 2, -} ETrnStep; + TRN_CONFLICT_NOTHING = 0, + TRN_CONFLICT_GLOBAL = 1, + TRN_CONFLICT_DB = 2, +} ETrnConflct; typedef enum { TRN_STAGE_PREPARE = 0, @@ -68,69 +69,15 @@ typedef enum { TRN_STAGE_FINISHED = 6 } ETrnStage; -typedef enum { - TRN_TYPE_BASIC_SCOPE = 1000, - TRN_TYPE_CREATE_ACCT = 1001, - TRN_TYPE_CREATE_CLUSTER = 1002, - TRN_TYPE_CREATE_USER = 1003, - TRN_TYPE_ALTER_USER = 1004, - TRN_TYPE_DROP_USER = 1005, - TRN_TYPE_CREATE_FUNC = 1006, - TRN_TYPE_DROP_FUNC = 1007, - - TRN_TYPE_CREATE_SNODE = 1010, - TRN_TYPE_DROP_SNODE = 1011, - TRN_TYPE_CREATE_QNODE = 1012, - TRN_TYPE_DROP_QNODE = 10013, - TRN_TYPE_CREATE_BNODE = 1014, - TRN_TYPE_DROP_BNODE = 1015, - TRN_TYPE_CREATE_MNODE = 1016, - TRN_TYPE_DROP_MNODE = 1017, - - TRN_TYPE_CREATE_TOPIC = 1020, - TRN_TYPE_DROP_TOPIC = 1021, - TRN_TYPE_SUBSCRIBE = 1022, - TRN_TYPE_REBALANCE = 1023, - TRN_TYPE_COMMIT_OFFSET = 1024, - TRN_TYPE_CREATE_STREAM = 1025, - TRN_TYPE_DROP_STREAM = 1026, - TRN_TYPE_ALTER_STREAM = 1027, - TRN_TYPE_CONSUMER_LOST = 1028, - TRN_TYPE_CONSUMER_RECOVER = 1029, - TRN_TYPE_DROP_CGROUP = 1030, - TRN_TYPE_BASIC_SCOPE_END, - - TRN_TYPE_GLOBAL_SCOPE = 2000, - TRN_TYPE_CREATE_DNODE = 2001, - TRN_TYPE_DROP_DNODE = 2002, - TRN_TYPE_GLOBAL_SCOPE_END, - - TRN_TYPE_DB_SCOPE = 3000, - TRN_TYPE_CREATE_DB = 3001, - TRN_TYPE_ALTER_DB = 3002, - TRN_TYPE_DROP_DB = 3003, - TRN_TYPE_SPLIT_VGROUP = 3004, - TRN_TYPE_MERGE_VGROUP = 3015, - TRN_TYPE_DB_SCOPE_END, - - TRN_TYPE_STB_SCOPE = 4000, - TRN_TYPE_CREATE_STB = 4001, - TRN_TYPE_ALTER_STB = 4002, - TRN_TYPE_DROP_STB = 4003, - TRN_TYPE_CREATE_SMA = 4004, - TRN_TYPE_DROP_SMA = 4005, - TRN_TYPE_STB_SCOPE_END, -} ETrnType; - typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1, } ETrnPolicy; typedef enum { - TRN_EXEC_PARALLEL = 0, - TRN_EXEC_NO_PARALLEL = 1, -} ETrnExecType; + TRN_EXEC_PRARLLEL = 0, + TRN_EXEC_SERIAL = 1, +} ETrnExec; typedef enum { DND_REASON_ONLINE = 0, @@ -159,8 +106,8 @@ typedef struct { int32_t id; ETrnStage stage; ETrnPolicy policy; - ETrnType type; - ETrnExecType parallel; + ETrnConflct conflict; + ETrnExec exec; int32_t code; int32_t failedTimes; SRpcHandleInfo rpcInfo; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index ba6f5faf1e..c70c6c4bd9 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -52,7 +52,7 @@ void mndCleanupTrans(SMnode *pMnode); STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId); void mndReleaseTrans(SMnode *pMnode, STrans *pTrans); -STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq); +STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, const SRpcMsg *pReq); void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); @@ -62,7 +62,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen); void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); -void mndTransSetNoParallel(STrans *pTrans); +void mndTransSetSerial(STrans *pTrans); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransProcessRsp(SRpcMsg *pRsp); diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index f3ec3a421b..0ce4a8c76e 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -80,7 +80,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) { mDebug("acct:%s, will be created when deploying, raw:%p", acctObj.acct, pRaw); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_ACCT, NULL); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL); if (pTrans == NULL) { mError("acct:%s, failed to create since %s", acctObj.acct, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 3316a09462..801f335a80 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -246,7 +246,7 @@ static int32_t mndCreateBnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, bnodeObj.createdTime = taosGetTimestampMs(); bnodeObj.updateTime = bnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_BNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create bnode:%d", pTrans->id, pCreate->dnodeId); @@ -363,7 +363,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn static int32_t mndDropBnode(SMnode *pMnode, SRpcMsg *pReq, SBnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_BNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop bnode:%d", pTrans->id, pObj->id); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 76c8acf407..bb3377d16a 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -179,10 +179,8 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { sdbSetRawStatus(pRaw, SDB_STATUS_READY); mDebug("cluster:%" PRId64 ", will be created when deploying, raw:%p", clusterObj.id, pRaw); -#if 0 - return sdbWrite(pMnode->pSdb, pRaw); -#else - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_CLUSTER, NULL); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL); if (pTrans == NULL) { mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr()); return -1; @@ -204,7 +202,6 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { mndTransDrop(pTrans); return 0; -#endif } static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7cebeb35f5..dcc807e07f 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -97,7 +97,7 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { mndReleaseConsumer(pMnode, pConsumer); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_LOST, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pMsg); if (pTrans == NULL) goto FAIL; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; @@ -121,7 +121,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { mndReleaseConsumer(pMnode, pConsumer); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_RECOVER, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pMsg); if (pTrans == NULL) goto FAIL; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; @@ -403,7 +403,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t newTopicNum = taosArrayGetSize(newSub); // check topic existance - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pMsg); if (pTrans == NULL) goto SUBSCRIBE_OVER; for (int32_t i = 0; i < newTopicNum; i++) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e3f843f0c7..61bd3a48a5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -545,7 +545,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, } int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_DB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); @@ -775,7 +775,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); @@ -1036,7 +1036,7 @@ static int32_t mndBuildDropDbRsp(SDbObj *pDb, int32_t *pRspLen, void **ppRsp, bo static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_DB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 8e06139c8c..2381080365 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -100,10 +100,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { mDebug("dnode:%d, will be created when deploying, raw:%p", dnodeObj.id, pRaw); -#if 0 - return sdbWrite(pMnode->pSdb, pRaw); -#else - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_DNODE, NULL); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL); if (pTrans == NULL) { mError("dnode:%s, failed to create since %s", dnodeObj.ep, terrstr()); return -1; @@ -125,7 +122,6 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { mndTransDrop(pTrans); return 0; -#endif } static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { @@ -481,7 +477,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN); snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_DNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) { mError("dnode:%s, failed to create since %s", dnodeObj.ep, terrstr()); return -1; @@ -557,7 +553,7 @@ CREATE_DNODE_OVER: } static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_DNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) { mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 9107dab693..bf4baebd85 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -215,7 +215,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre } memcpy(func.pCode, pCreate->pCode, func.codeSize); - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_FUNC, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); @@ -245,7 +245,7 @@ _OVER: static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_FUNC, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 5b8ba6deaa..719fbfa45a 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -92,10 +92,7 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { mDebug("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw); -#if 0 - return sdbWrite(pMnode->pSdb, pRaw); -#else - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_DNODE, NULL); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL); if (pTrans == NULL) { mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr()); return -1; @@ -117,7 +114,6 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { mndTransDrop(pTrans); return 0; -#endif } static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) { @@ -363,11 +359,11 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, mnodeObj.createdTime = taosGetTimestampMs(); mnodeObj.updateTime = mnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_MNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); - mndTransSetNoParallel(pTrans); + mndTransSetSerial(pTrans); if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; @@ -535,11 +531,11 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_MNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); - mndTransSetNoParallel(pTrans); + mndTransSetSerial(pTrans); if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 01516d03f2..d22da4f76c 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -178,7 +178,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) { tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_COMMIT_OFFSET, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pMsg); for (int32_t i = 0; i < commitOffsetReq.num; i++) { SMqOffset *pOffset = &commitOffsetReq.offsets[i]; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 3dc6200229..9bf0c4c832 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -248,7 +248,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, qnodeObj.createdTime = taosGetTimestampMs(); qnodeObj.updateTime = qnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_QNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); @@ -365,7 +365,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_QNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0493b00d33..7f81a99fd2 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -502,12 +502,12 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.fixedSinkVgId = smaObj.dstVgId; int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_SMA, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name); mndTransSetDbInfo(pTrans, pDb); - mndTransSetNoParallel(pTrans); + mndTransSetSerial(pTrans); if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; @@ -747,7 +747,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); if (pVgroup == NULL) goto _OVER; - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_SMA, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 87b61f59ec..c6acb4fef4 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -253,7 +253,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, snodeObj.createdTime = taosGetTimestampMs(); snodeObj.updateTime = snodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_SNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); @@ -372,7 +372,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_SNODE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 81c3b24d97..838265ef41 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -735,7 +735,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); @@ -1257,7 +1257,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p if (code != 0) goto _OVER; code = -1; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_STB, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name); @@ -1403,7 +1403,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_STB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index cbef1facdc..30ef2b5210 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -393,7 +393,7 @@ static int32_t mndCreateStream(SMnode *pMnode, SRpcMsg *pReq, SCMCreateStreamReq streamObj.trigger = pCreate->triggerType; streamObj.waterMark = pCreate->watermark; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 7e72aa2425..683301f5c6 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -403,7 +403,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_REBALANCE, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pMsg); if (pTrans == NULL) { return -1; } @@ -612,7 +612,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_CGROUP, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) { mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 2048c79847..e22d0daf5b 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -376,7 +376,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.withSchema = 1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); taosMemoryFreeClear(topicObj.ast); @@ -520,7 +520,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_TOPIC, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) { mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ad6388c585..84a1308271 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -117,8 +117,8 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pTrans->id, _OVER) SDB_SET_INT16(pRaw, dataPos, pTrans->stage, _OVER) SDB_SET_INT16(pRaw, dataPos, pTrans->policy, _OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->type, _OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->parallel, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->conflict, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->exec, _OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, pTrans->redoActionPos, _OVER) @@ -250,16 +250,16 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int16_t stage = 0; int16_t policy = 0; - int16_t type = 0; - int16_t parallel = 0; + int16_t conflict = 0; + int16_t exec = 0; SDB_GET_INT16(pRaw, dataPos, &stage, _OVER) SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) - SDB_GET_INT16(pRaw, dataPos, &type, _OVER) - SDB_GET_INT16(pRaw, dataPos, ¶llel, _OVER) + SDB_GET_INT16(pRaw, dataPos, &conflict, _OVER) + SDB_GET_INT16(pRaw, dataPos, &exec, _OVER) pTrans->stage = stage; pTrans->policy = policy; - pTrans->type = type; - pTrans->parallel = parallel; + pTrans->conflict = conflict; + pTrans->exec = exec; SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, &pTrans->redoActionPos, _OVER) @@ -408,81 +408,6 @@ static const char *mndTransStr(ETrnStage stage) { } } -static const char *mndTransType(ETrnType type) { - switch (type) { - case TRN_TYPE_CREATE_USER: - return "create-user"; - case TRN_TYPE_ALTER_USER: - return "alter-user"; - case TRN_TYPE_DROP_USER: - return "drop-user"; - case TRN_TYPE_CREATE_FUNC: - return "create-func"; - case TRN_TYPE_DROP_FUNC: - return "drop-func"; - case TRN_TYPE_CREATE_SNODE: - return "create-snode"; - case TRN_TYPE_DROP_SNODE: - return "drop-snode"; - case TRN_TYPE_CREATE_QNODE: - return "create-qnode"; - case TRN_TYPE_DROP_QNODE: - return "drop-qnode"; - case TRN_TYPE_CREATE_BNODE: - return "create-bnode"; - case TRN_TYPE_DROP_BNODE: - return "drop-bnode"; - case TRN_TYPE_CREATE_MNODE: - return "create-mnode"; - case TRN_TYPE_DROP_MNODE: - return "drop-mnode"; - case TRN_TYPE_CREATE_TOPIC: - return "create-topic"; - case TRN_TYPE_DROP_TOPIC: - return "drop-topic"; - case TRN_TYPE_SUBSCRIBE: - return "subscribe"; - case TRN_TYPE_REBALANCE: - return "rebalance"; - case TRN_TYPE_COMMIT_OFFSET: - return "commit-offset"; - case TRN_TYPE_CREATE_STREAM: - return "create-stream"; - case TRN_TYPE_DROP_STREAM: - return "drop-stream"; - case TRN_TYPE_CONSUMER_LOST: - return "consumer-lost"; - case TRN_TYPE_CONSUMER_RECOVER: - return "consumer-recover"; - case TRN_TYPE_CREATE_DNODE: - return "create-qnode"; - case TRN_TYPE_DROP_DNODE: - return "drop-qnode"; - case TRN_TYPE_CREATE_DB: - return "create-db"; - case TRN_TYPE_ALTER_DB: - return "alter-db"; - case TRN_TYPE_DROP_DB: - return "drop-db"; - case TRN_TYPE_SPLIT_VGROUP: - return "split-vgroup"; - case TRN_TYPE_MERGE_VGROUP: - return "merge-vgroup"; - case TRN_TYPE_CREATE_STB: - return "create-stb"; - case TRN_TYPE_ALTER_STB: - return "alter-stb"; - case TRN_TYPE_DROP_STB: - return "drop-stb"; - case TRN_TYPE_CREATE_SMA: - return "create-sma"; - case TRN_TYPE_DROP_SMA: - return "drop-sma"; - default: - return "invalid"; - } -} - static void mndTransTestStartFunc(SMnode *pMnode, void *param, int32_t paramLen) { mInfo("test trans start, param:%s, len:%d", (char *)param, paramLen); } @@ -594,7 +519,7 @@ void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { sdbRelease(pSdb, pTrans); } -STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq) { +STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, const SRpcMsg *pReq) { STrans *pTrans = taosMemoryCalloc(1, sizeof(STrans)); if (pTrans == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -605,8 +530,8 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; - pTrans->type = type; - pTrans->parallel = TRN_EXEC_PARALLEL; + pTrans->conflict = conflict; + pTrans->exec = TRN_EXEC_PRARLLEL; pTrans->createdTime = taosGetTimestampMs(); pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); @@ -698,7 +623,7 @@ void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) { memcpy(pTrans->dbname, pDb->name, TSDB_DB_FNAME_LEN); } -void mndTransSetNoParallel(STrans *pTrans) { pTrans->parallel = TRN_EXEC_NO_PARALLEL; } +void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; } static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { SSdbRaw *pRaw = mndTransActionEncode(pTrans); @@ -721,76 +646,43 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { return 0; } -static bool mndIsBasicTrans(STrans *pTrans) { - return pTrans->type > TRN_TYPE_BASIC_SCOPE && pTrans->type < TRN_TYPE_BASIC_SCOPE_END; -} - -static bool mndIsGlobalTrans(STrans *pTrans) { - return pTrans->type > TRN_TYPE_GLOBAL_SCOPE && pTrans->type < TRN_TYPE_GLOBAL_SCOPE_END; -} - -static bool mndIsDbTrans(STrans *pTrans) { - return pTrans->type > TRN_TYPE_DB_SCOPE && pTrans->type < TRN_TYPE_DB_SCOPE_END; -} - -static bool mndIsStbTrans(STrans *pTrans) { - return pTrans->type > TRN_TYPE_STB_SCOPE && pTrans->type < TRN_TYPE_STB_SCOPE_END; -} - static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNewTrans) { STrans *pTrans = NULL; void *pIter = NULL; bool conflict = false; - if (mndIsBasicTrans(pNewTrans)) return conflict; + if (pNewTrans->conflict == TRN_CONFLICT_NOTHING) return conflict; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); if (pIter == NULL) break; - if (mndIsGlobalTrans(pNewTrans)) { - if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { + if (pNewTrans->conflict == TRN_CONFLICT_GLOBAL) { + mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); + conflict = true; + } else if (pNewTrans->conflict == TRN_CONFLICT_DB) { + if (pTrans->conflict == TRN_CONFLICT_GLOBAL || strcmp(pNewTrans->dbname, pTrans->dbname) == 0) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); conflict = true; - } else { - } - } - - else if (mndIsDbTrans(pNewTrans)) { - if (mndIsGlobalTrans(pTrans)) { - mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - conflict = true; - } else if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { - if (strcmp(pNewTrans->dbname, pTrans->dbname) == 0) { - mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - conflict = true; - } - } else { - } - } - - else if (mndIsStbTrans(pNewTrans)) { - if (mndIsGlobalTrans(pTrans)) { - mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - conflict = true; - } else if (mndIsDbTrans(pTrans)) { - if (strcmp(pNewTrans->dbname, pTrans->dbname) == 0) { - mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - conflict = true; - } - } else { } + } else { } sdbRelease(pMnode->pSdb, pTrans); } - sdbCancelFetch(pMnode->pSdb, pIter); - sdbRelease(pMnode->pSdb, pTrans); return conflict; } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { + if (pTrans->conflict == TRN_CONFLICT_DB) { + if (strlen(pTrans->dbname) == 0) { + terrno = TSDB_CODE_MND_TRANS_CONFLICT; + mError("trans:%d, failed to prepare conflict db not set", pTrans->id); + return -1; + } + } + if (mndCheckTransConflict(pMnode, pTrans)) { terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -1144,7 +1036,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { bool continueExec = true; int32_t code = 0; - if (pTrans->parallel == TRN_EXEC_NO_PARALLEL) { + if (pTrans->exec == TRN_EXEC_SERIAL) { code = mndTransExecuteRedoActionsNoParallel(pMnode, pTrans); } else { code = mndTransExecuteRedoActions(pMnode, pTrans); @@ -1456,7 +1348,7 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl colDataAppend(pColInfo, numOfRows, (const char *)dbname, false); char type[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(type, mndTransType(pTrans->type), pShow->pMeta->pSchemas[cols].bytes); + STR_WITH_MAXSIZE_TO_VARSTR(type, "todo", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)type, false); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 83d00c86e3..345d756f43 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -79,10 +79,7 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char mDebug("user:%s, will be created when deploying, raw:%p", userObj.user, pRaw); -#if 0 - return sdbWrite(pMnode->pSdb, pRaw); -#else - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_USER, NULL); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL); if (pTrans == NULL) { mError("user:%s, failed to create since %s", userObj.user, terrstr()); return -1; @@ -104,7 +101,6 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char mndTransDrop(pTrans); return 0; -#endif } static int32_t mndCreateDefaultUsers(SMnode *pMnode) { @@ -291,7 +287,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate userObj.updateTime = userObj.createdTime; userObj.superUser = pCreate->superUser; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { mError("user:%s, failed to create since %s", pCreate->user, terrstr()); return -1; @@ -371,7 +367,7 @@ _OVER: } static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpcMsg *pReq) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { mError("user:%s, failed to alter since %s", pOld->user, terrstr()); return -1; @@ -578,7 +574,7 @@ _OVER: } static int32_t mndDropUser(SMnode *pMnode, SRpcMsg *pReq, SUserObj *pUser) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_USER, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index cfcfc2490e..d518db2d38 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -11,6 +11,8 @@ #include +#if 0 + #include "mndTrans.h" #include "mndUser.h" #include "tcache.h" @@ -103,7 +105,7 @@ class MndTestTrans2 : public ::testing::Test { void SetUp() override {} void TearDown() override {} - int32_t CreateUserLog(const char *acct, const char *user, ETrnType type, SDbObj *pDb) { + int32_t CreateUserLog(const char *acct, const char *user, ETrnConflct conflict, SDbObj *pDb) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -113,7 +115,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, type, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, conflict, &rpcMsg); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -135,7 +137,7 @@ class MndTestTrans2 : public ::testing::Test { return code; } - int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy, ETrnType type, + int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy, ETrnConflct conflict, SDbObj *pDb) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); @@ -146,7 +148,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, policy, type, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, policy, conflict, &rpcMsg); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -218,7 +220,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, &rpcMsg); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -528,3 +530,5 @@ TEST_F(MndTestTrans2, 04_Conflict) { mndReleaseUser(pMnode, pUser); } } + +#endif \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 217c23158d..2f7279e66c 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -57,7 +57,6 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -./test.sh -f tsim/mnode/basic3.sim # ---- show ./test.sh -f tsim/show/basic.sim diff --git a/tests/script/tsim/trans/create_db.sim b/tests/script/tsim/trans/create_db.sim index ae6b7eab16..14415c7193 100644 --- a/tests/script/tsim/trans/create_db.sim +++ b/tests/script/tsim/trans/create_db.sim @@ -76,7 +76,7 @@ if $data[0][3] != d1 then return -1 endi -if $data[0][4] != create-db then +if $data[0][4] != todo then return -1 endi @@ -126,7 +126,7 @@ if $data[0][3] != d2 then return -1 endi -if $data[0][4] != create-db then +if $data[0][4] != todo then return -1 endi diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c index 7343b4f829..31da855164 100644 --- a/tests/test/c/sdbDump.c +++ b/tests/test/c/sdbDump.c @@ -279,7 +279,8 @@ void dumpTrans(SSdb *pSdb, SJson *json) { tjsonAddIntegerToObject(item, "id", pObj->id); tjsonAddIntegerToObject(item, "stage", pObj->stage); tjsonAddIntegerToObject(item, "policy", pObj->policy); - tjsonAddIntegerToObject(item, "type", pObj->type); + tjsonAddIntegerToObject(item, "conflict", pObj->conflict); + tjsonAddIntegerToObject(item, "exec", pObj->exec); tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid)); tjsonAddStringToObject(item, "dbname", pObj->dbname); From 816cea07f99d583849a7396bbfdacd3f038b7635 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 17:49:33 +0800 Subject: [PATCH 06/57] feat:add new logic for new tag format --- include/common/tdataformat.h | 44 +-- include/common/tmsg.h | 2 +- source/client/src/clientImpl.c | 51 +-- source/common/src/tdatablock.c | 21 +- source/common/src/tdataformat.c | 115 ++----- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 16 +- source/dnode/vnode/src/meta/metaTable.c | 21 +- source/libs/executor/src/scanoperator.c | 55 ++- source/libs/parser/inc/parUtil.h | 4 +- source/libs/parser/src/parInsert.c | 430 ++++++++++++++++++------ source/libs/parser/src/parTranslater.c | 255 +++++++------- source/libs/parser/src/parUtil.c | 74 ++-- source/libs/scalar/src/sclvector.c | 39 +-- 14 files changed, 587 insertions(+), 542 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 957b93d359..38bbee1f1d 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -59,14 +59,11 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder); int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); -// STagVal -static FORCE_INLINE void tTagValPush(SArray *pTagArray, void *key, int8_t type, uint8_t *pData, uint32_t nData, - bool isJson); - // STag int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); bool tTagGet(const STag *pTag, STagVal *pTagVal); +char* tTagValToData(const STagVal *pTagVal, bool isJson); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); @@ -130,16 +127,7 @@ struct STagVal { }; int8_t type; union { - int8_t i8; - uint8_t u8; - int16_t i16; - uint16_t u16; - int32_t i32; - uint32_t u32; int64_t i64; - uint64_t u64; - float f; - double d; struct { uint32_t nData; uint8_t *pData; @@ -147,24 +135,9 @@ struct STagVal { }; }; -static FORCE_INLINE void tTagValPush(SArray *pTagArray, void *key, int8_t type, uint8_t *pData, uint32_t nData, - bool isJson) { - STagVal tagVal = {0}; - if (isJson) { - tagVal.pKey = (char *)key; - } else { - tagVal.cid = *(int16_t *)key; - } - - tagVal.type = type; - tagVal.pData = pData; - tagVal.nData = nData; - taosArrayPush(pTagArray, &tagVal); -} - #pragma pack(push, 1) -#define TD_TAG_JSON ((int8_t)0x1) -#define TD_TAG_LARGE ((int8_t)0x2) +#define TD_TAG_JSON ((int8_t)0x80) // distinguish JSON string and JSON value with the highest bit +#define TD_TAG_LARGE ((int8_t)0x40) struct STag { int8_t flags; int16_t len; @@ -424,14 +397,3 @@ int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToM #endif /*_TD_COMMON_DATA_FORMAT_H_*/ -// SKVRowBuilder; - -// int32_t tdInitKVRowBuilder(SKVRowBuilder *pBuilder); -// void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); -// void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); -// SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); - -// static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) - -// #ifdef JSON_TAG_REFACTOR -// TODO: JSON_TAG_TODO diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 37e4cc5e8c..4d69cf5a32 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -287,7 +287,7 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; -#define COL_IS_SET(FLG) ((FLG) & (COL_SET_VAL | COL_SET_NULL) != 0) +#define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index fc899b9b83..db5e6ab4c0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -958,30 +958,11 @@ static char* parseTagDatatoJson(void* p) { char tagJsonKey[256] = {0}; for (int j = 0; j < nCols; ++j) { STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); - - // JSON_TAG_REFACTOR - pTagVal->nData; - pTagVal->pData; // for varChar, len not included - // TODO: adapt below code; - char* val = (char*)pTagVal->pData; - // TODO: adapt below code; - - if (j == 0) { - if (*val == TSDB_DATA_TYPE_NULL) { - string = taosMemoryCalloc(1, 8); - sprintf(string, "%s", TSDB_DATA_NULL_STR_L); - goto end; - } - continue; - } - // json key encode by binary memset(tagJsonKey, 0, sizeof(tagJsonKey)); - memcpy(tagJsonKey, varDataVal(val), varDataLen(val)); + memcpy(tagJsonKey, pTagVal->pKey, strlen(pTagVal->pKey)); // json value - val += varDataTLen(val); - char* realData = POINTER_SHIFT(val, CHAR_BYTES); - char type = *val; + char type = pTagVal->type; if (type == TSDB_DATA_TYPE_NULL) { cJSON* value = cJSON_CreateNull(); if (value == NULL) { @@ -990,11 +971,11 @@ static char* parseTagDatatoJson(void* p) { cJSON_AddItemToObject(json, tagJsonKey, value); } else if (type == TSDB_DATA_TYPE_NCHAR) { cJSON* value = NULL; - if (varDataLen(realData) > 0) { - char* tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1); - int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(realData), varDataLen(realData), tagJsonValue); + if (pTagVal->nData > 0) { + char* tagJsonValue = taosMemoryCalloc(pTagVal->nData, 1); + int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue); if (length < 0) { - tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, val); + tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, pTagVal->pData); taosMemoryFree(tagJsonValue); goto end; } @@ -1003,7 +984,7 @@ static char* parseTagDatatoJson(void* p) { if (value == NULL) { goto end; } - } else if (varDataLen(realData) == 0) { + } else if (pTagVal->nData == 0) { value = cJSON_CreateString(""); } else { ASSERT(0); @@ -1011,22 +992,14 @@ static char* parseTagDatatoJson(void* p) { cJSON_AddItemToObject(json, tagJsonKey, value); } else if (type == TSDB_DATA_TYPE_DOUBLE) { - double jsonVd = *(double*)(realData); + double jsonVd = *(double*)(&pTagVal->i64); cJSON* value = cJSON_CreateNumber(jsonVd); if (value == NULL) { goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); - // }else if(type == TSDB_DATA_TYPE_BIGINT){ - // int64_t jsonVd = *(int64_t*)(realData); - // cJSON* value = cJSON_CreateNumber((double)jsonVd); - // if (value == NULL) - // { - // goto end; - // } - // cJSON_AddItemToObject(json, tagJsonKey, value); } else if (type == TSDB_DATA_TYPE_BOOL) { - char jsonVd = *(char*)(realData); + char jsonVd = *(char*)(&pTagVal->i64); cJSON* value = cJSON_CreateBool(jsonVd); if (value == NULL) { goto end; @@ -1091,7 +1064,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int if (jsonInnerType == TSDB_DATA_TYPE_NULL) { sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { + } else if (jsonInnerType == TD_TAG_JSON) { char* jsonString = parseTagDatatoJson(jsonInnerData); STR_TO_VARSTR(dst, jsonString); taosMemoryFree(jsonString); @@ -1110,10 +1083,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int double jsonVd = *(double*)(jsonInnerData); sprintf(varDataVal(dst), "%.9lf", jsonVd); varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) { - int64_t jsonVd = *(int64_t*)(jsonInnerData); - sprintf(varDataVal(dst), "%" PRId64, jsonVd); - varDataSetLen(dst, strlen(varDataVal(dst))); } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); varDataSetLen(dst, strlen(varDataVal(dst))); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1918061837..e6d532f2dc 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -116,22 +116,23 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { - int32_t dataLen = varDataTLen(pData); + int32_t dataLen = 0; if (type == TSDB_DATA_TYPE_JSON) { if (*pData == TSDB_DATA_TYPE_NULL) { - dataLen = 0; - } else if (*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData + CHAR_BYTES); - } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { - dataLen = DOUBLE_BYTES; - } else if (*pData == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES; - } else if (*pData == TSDB_DATA_TYPE_JSON) { - dataLen = ((STag*)(pData + CHAR_BYTES))->len; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES) + CHAR_BYTES; + } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = DOUBLE_BYTES + CHAR_BYTES; + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES + CHAR_BYTES; + } else if (*pData == TD_TAG_JSON) { // json string + dataLen = ((STag*)(pData))->len; } else { ASSERT(0); } - dataLen += CHAR_BYTES; + }else { + dataLen = varDataTLen(pData); } SVarColAttr* pAttr = &pColumnInfoData->varmeta; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 34e78c21db..722fa57bad 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -123,7 +123,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal ASSERT(iCol != 0); ASSERT(pTColumn->colId != 0); - ASSERT(pRow->flags & 0xf != 0); + ASSERT((pRow->flags & 0xf) != 0); switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: *pColVal = ColValNONE; @@ -432,7 +432,6 @@ static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) { } int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { int32_t nDataTP, nDataKV; - uint32_t flags; STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; int32_t nCols = pBuilder->pTSchema->numOfCols; @@ -446,7 +445,7 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { pBuilder->row.flags |= TSROW_HAS_NONE; } - ASSERT(pBuilder->row.flags & 0xf != 0); + ASSERT((pBuilder->row.flags & 0xf) != 0); *(ppRow) = &pBuilder->row; switch (pBuilder->row.flags & 0xf) { case TSROW_HAS_NONE: @@ -476,7 +475,7 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { if (nDataKV < nDataTP) { // generate KV row - ASSERT(pBuilder->row.flags & 0xf != TSROW_HAS_VAL); + ASSERT((pBuilder->row.flags & 0xf) != TSROW_HAS_VAL); pBuilder->row.flags |= TSROW_KV_ROW; pBuilder->row.nData = nDataKV; @@ -492,12 +491,12 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { pBuilder->row.nData = nDataTP; uint8_t *p; - uint8_t flags = pBuilder->row.flags & 0xf; + uint8_t flags = (pBuilder->row.flags & 0xf); if (flags == TSROW_HAS_VAL) { pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2; } else { - if (flags == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) { + if (flags == (TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE)) { pBuilder->row.pData = pBuilder->pTPBuf; } else { pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1; @@ -620,7 +619,11 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { } printf("%s:%d loop[%d-%d] offset=%d\n", __func__, __LINE__, (int32_t)pTag->nTag, (int32_t)n, (int32_t)offset); tGetTagVal(p + offset, &tagVal, isJson); - debugPrintTagVal(tagVal.type, tagVal.pData, tagVal.nData, __func__, __LINE__); + if(IS_VAR_DATA_TYPE(tagVal.type)){ + debugPrintTagVal(tagVal.type, tagVal.pData, tagVal.nData, __func__, __LINE__); + }else{ + debugPrintTagVal(tagVal.type, &tagVal.i64, tDataTypes[tagVal.type].bytes, __func__, __LINE__); + } } printf("\n"); } @@ -643,44 +646,8 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData); } else { p = p ? p + n : p; - switch (pTagVal->type) { - case TSDB_DATA_TYPE_BOOL: - n += tPutI8(p, pTagVal->i8 ? 1 : 0); - break; - case TSDB_DATA_TYPE_TINYINT: - n += tPutI8(p, pTagVal->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tPutI16(p, pTagVal->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tPutI32(p, pTagVal->i32); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - n += tPutI64(p, pTagVal->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tPutFloat(p, pTagVal->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tPutDouble(p, pTagVal->d); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tPutU8(p, pTagVal->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tPutU16(p, pTagVal->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tPutU32(p, pTagVal->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tPutU64(p, pTagVal->u64); - break; - default: - ASSERT(0); - } + n += tDataTypes[pTagVal->type].bytes; + if(p) memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); } return n; @@ -702,42 +669,8 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); } else { - switch (pTagVal->type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - n += tGetI8(p + n, &pTagVal->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tGetI16(p, &pTagVal->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tGetI32(p, &pTagVal->i32); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - n += tGetI64(p, &pTagVal->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tGetFloat(p, &pTagVal->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tGetDouble(p, &pTagVal->d); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tGetU8(p, &pTagVal->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tGetU16(p, &pTagVal->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tGetU32(p, &pTagVal->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tGetU64(p, &pTagVal->u64); - break; - default: - ASSERT(0); - } + n += tDataTypes[pTagVal->type].bytes; + memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); } return n; @@ -814,6 +747,26 @@ void tTagFree(STag *pTag) { if (pTag) taosMemoryFree(pTag); } +char *tTagValToData(const STagVal *value, bool isJson){ + if(!value) return NULL; + char *data = NULL; + int8_t typeBytes = 0; + if (isJson) { + typeBytes = CHAR_BYTES; + } + if(IS_VAR_DATA_TYPE(value->type)){ + data = taosMemoryCalloc(1, typeBytes + VARSTR_HEADER_SIZE + value->nData); + if(data == NULL) return NULL; + if(isJson) *data = value->type; + varDataLen(data + typeBytes) = value->nData; + memcpy(varDataVal(data + typeBytes), value->pData, value->nData); + }else{ + data = ((char*)&(value->i64)) - typeBytes; // json with type + } + + return data; +} + bool tTagGet(const STag *pTag, STagVal *pTagVal) { int16_t lidx = 0; int16_t ridx = pTag->nTag - 1; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2b713ff980..df5785e20a 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -78,7 +78,7 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderClear(SMetaReader *pReader); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaReadNext(SMetaReader *pReader); -const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid); +const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *tagVal); #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index b59b6a6fd0..dead85f854 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -564,9 +564,17 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { #endif -const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid) { +const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { ASSERT(pEntry->type == TSDB_CHILD_TABLE); - STagVal tagVal = {.cid = cid}; - tTagGet((const STag *)pEntry->ctbEntry.pTags, &tagVal); - return tagVal.pData; + STag *tag = (STag *)pEntry->ctbEntry.pTags; + tTagGet(tag, val); + + if(val->type == TSDB_DATA_TYPE_NULL){ + return NULL; + } + if (type == TSDB_DATA_TYPE_JSON){ + return tag; + }else{ + return val; + } } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index def024667b..984688fc19 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -573,15 +573,20 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA for (int32_t i = 0; i < pTagSchema->nCols; i++) { SSchema *pCol = &pTagSchema->pSchema[i]; if (iCol == i) { - tTagValPush(pTagArray, &pCol->colId, pCol->type, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal, false); + STagVal val = {0}; + val.type = pCol->type; + val.cid = pCol->colId; + if (IS_VAR_DATA_TYPE(pCol->type)) { + val.pData = pAlterTbReq->pTagVal; + val.nData = pAlterTbReq->nTagVal; + }else{ + memcpy(&val.i64, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); + } + taosArrayPush(pTagArray, &val); } else { - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet(pOldTag, &tagVal) && tagVal.pData) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - tTagValPush(pTagArray, &pCol->colId, pCol->type, varDataVal(tagVal.pData), varDataLen(tagVal.pData), false); - } else { - tTagValPush(pTagArray, &pCol->colId, pCol->type, tagVal.pData, pCol->bytes, false); - } + STagVal val = {0}; + if (tTagGet(pOldTag, &val)) { + taosArrayPush(pTagArray, &val); } } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4a275660bc..973e52d1a3 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -303,29 +303,22 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) if (fmIsScanPseudoColumnFunc(functionId)) { setTbNameColData(pTableScanInfo->readHandle.meta, pBlock, pColInfoData, functionId); } else { // these are tags - const char* p = NULL; - if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { - const STag* tmp = (const STag*)mr.me.ctbEntry.pTags; + STagVal tagVal = {0}; + tagVal.cid = pExpr->base.pParam[0].pCol->colId; + const char *p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal); - char* data = taosMemoryCalloc(tmp->len + 1, 1); - if (data == NULL) { - metaReaderClear(&mr); - qError("doTagScan calloc error:%d", tmp->len + 1); - return; - } - - *data = TSDB_DATA_TYPE_JSON; - memcpy(data + 1, tmp, tmp->len); - p = data; - } else { - p = metaGetTableTagVal(&mr.me, pExpr->base.pParam[0].pCol->colId); + char *data = NULL; + if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL){ + data = tTagValToData((const STagVal *)p, false); + }else { + data = (char*)p; } for (int32_t i = 0; i < pBlock->info.rows; ++i) { - colDataAppend(pColInfoData, i, p, (p == NULL)); + colDataAppend(pColInfoData, i, data, (data == NULL)); } - if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { - taosMemoryFree((void*)p); + if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ + taosMemoryFree(data); } } } @@ -1632,22 +1625,20 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { STR_TO_VARSTR(str, mr.me.name); colDataAppend(pDst, count, str, false); } else { // it is a tag value - if (pDst->info.type == TSDB_DATA_TYPE_JSON) { - const STag* tmp = (const STag*)mr.me.ctbEntry.pTags; - // TODO opt perf by realloc memory - char* data = taosMemoryCalloc(tmp->len + 1, 1); - if (data == NULL) { - qError("%s failed to malloc memory, size:%d", GET_TASKID(pTaskInfo), tmp->len + 1); - longjmp(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); - } + STagVal val = {0}; + val.cid = pExprInfo[j].base.pParam[0].pCol->colId; + const char* p = metaGetTableTagVal(&mr.me, pDst->info.type, &val); - *data = TSDB_DATA_TYPE_JSON; - memcpy(data + 1, tmp, tmp->len); - colDataAppend(pDst, count, data, false); + char *data = NULL; + if(pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL){ + data = tTagValToData((const STagVal *)p, false); + }else { + data = (char*)p; + } + colDataAppend(pDst, count, data, (data == NULL)); + + if(pDst->info.type != TSDB_DATA_TYPE_JSON && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ taosMemoryFree(data); - } else { - const char* p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId); - colDataAppend(pDst, count, p, (p == NULL)); } } } diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 5098ea9232..35cab7db36 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -55,9 +55,7 @@ int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); -#ifdef JSON_TAG_REFACTOR -int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId); -#endif +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* errMsg); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 37ee6df682..d670f5ad38 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -54,7 +54,6 @@ typedef struct SInsertParseContext { SMsgBuf msg; // input STableMeta* pTableMeta; // each table SParsedDataColInfo tags; // each table - SArray* pTagVals; // each table SVCreateTbReq createTblReq; // each table SHashObj* pVgroupsHashObj; // global SHashObj* pTableBlockHashObj; // global @@ -425,7 +424,7 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { +static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && @@ -471,7 +470,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int uint64_t uv; char* endptr = NULL; - int32_t code = checkAndTrimValue(pToken, pSchema->type, tmpTokenBuf, pMsgBuf); + int32_t code = checkAndTrimValue(pToken, tmpTokenBuf, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -620,14 +619,12 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_NCHAR: { return func(pMsgBuf, pToken->z, pToken->n, param); } -#ifdef JSON_TAG_REFACTOR case TSDB_DATA_TYPE_JSON: { if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); } return func(pMsgBuf, pToken->z, pToken->n, param); } -#endif case TSDB_DATA_TYPE_TIMESTAMP: { int64_t tmpVal; if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { @@ -752,108 +749,287 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_SUCCESS; } -static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { - SKvParam* pa = (SKvParam*)param; - - int8_t type = pa->schema->type; - int16_t colId = pa->schema->colId; - -#ifdef JSON_TAG_REFACTOR - if (TSDB_DATA_TYPE_JSON == type) { - return parseJsontoTagData(value, pa->builder, pMsgBuf, colId); - } -#endif - - if (value == NULL) { // it is a null data - // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, - // pa->colIdx); - return TSDB_CODE_SUCCESS; - } - - if (TSDB_DATA_TYPE_BINARY == type) { - memcpy(pa->buf + pa->pos, value, len); - tTagValPush(pa->pTagVals, &colId, type, (uint8_t*)(pa->buf + pa->pos), len, false); - pa->pos += len; - } else if (TSDB_DATA_TYPE_NCHAR == type) { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - - ASSERT((pa->pos + pa->schema->bytes - VARSTR_HEADER_SIZE) <= TSDB_MAX_TAGS_LEN); - - int32_t output = 0; - if (!taosMbsToUcs4(value, len, (TdUcs4*)(pa->buf + pa->pos), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { - if (errno == E2BIG) { - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pa->schema->name); - } - char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); - return buildSyntaxErrMsg(pMsgBuf, buf, value); - } - tTagValPush(pa->pTagVals, &colId, type, (uint8_t*)(pa->buf + pa->pos), output, false); - pa->pos += output; - } else { - memcpy(pa->buf + pa->pos, value, TYPE_BYTES[type]); - tTagValPush(pa->pTagVals, &colId, type, (uint8_t*)(pa->buf + pa->pos), TYPE_BYTES[type], false); - pa->pos + TYPE_BYTES[type]; - } - ASSERT(pa->pos <= TSDB_MAX_TAGS_LEN); - - return TSDB_CODE_SUCCESS; -} - -static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid) { +static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid) { pTbReq->type = TD_CHILD_TABLE; pTbReq->name = strdup(tname); pTbReq->ctb.suid = suid; pTbReq->ctb.pTag = (uint8_t*)pTag; - return TSDB_CODE_SUCCESS; + return; +} + +static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, + int16_t timePrec, char* tmpTokenBuf, STagVal *val, SMsgBuf* pMsgBuf) { + int64_t iv; + uint64_t uv; + char* endptr = NULL; + + if (isNullStr(pToken)) { + if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { + return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z); + } + + return TSDB_CODE_SUCCESS; + } + + val->cid = pSchema->colId; + val->type = pSchema->bytes; + + switch (pSchema->type) { + case TSDB_DATA_TYPE_BOOL: { + if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + *(int8_t*)(&val->i64) = TRUE_VALUE; + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + *(int8_t*)(&val->i64) = FALSE_VALUE; + } else { + return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_NK_INTEGER) { + *(int8_t*)(&val->i64) = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); + } else if (pToken->type == TK_NK_FLOAT) { + *(int8_t*)(&val->i64) = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); + } else { + return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); + } + break; + } + + case TSDB_DATA_TYPE_TINYINT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); + } + + *(int8_t*)(&val->i64) = iv; + break; + } + + case TSDB_DATA_TYPE_UTINYINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(uv)) { + return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); + } + *(uint8_t*)(&val->i64) = uv; + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); + } + *(int16_t*)(&val->i64) = iv; + break; + } + + case TSDB_DATA_TYPE_USMALLINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(uv)) { + return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); + } + *(uint16_t*)(&val->i64) = uv; + break; + } + + case TSDB_DATA_TYPE_INT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); + } + *(int32_t*)(&val->i64) = iv; + break; + } + + case TSDB_DATA_TYPE_UINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(uv)) { + return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); + } + *(uint32_t*)(&val->i64) = uv; + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); + } else if (!IS_VALID_BIGINT(iv)) { + return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); + } + + val->i64 = iv; + break; + } + + case TSDB_DATA_TYPE_UBIGINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT(uv)) { + return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); + } + *(uint64_t*)(&val->i64) = uv; + break; + } + + case TSDB_DATA_TYPE_FLOAT: { + double dv; + if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); + } + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || + isnan(dv)) { + return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); + } + *(float*)(&val->i64) = dv; + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double dv; + if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + } + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + } + + *(double*)(&val->i64) = dv; + break; + } + + case TSDB_DATA_TYPE_BINARY: { + // Too long values will raise the invalid sql error message + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + val->pData = pToken->z; + val->nData = pToken->n; + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + int32_t output = 0; + void *p = taosMemoryCalloc(1, pToken->n * TSDB_NCHAR_SIZE); + if(p == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (errno == E2BIG) { + taosMemoryFree(p); + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + taosMemoryFree(p); + return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z); + } + val->pData = p; + val->nData = output; + break; + } + case TSDB_DATA_TYPE_JSON: { + if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); + } + //return func(pMsgBuf, pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_TIMESTAMP: { + if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); + } + + val->i64 = iv; + break; + } + } + + return TSDB_CODE_FAILED; } // pSql -> tag1_value, ...) static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) { - ASSERT(!pCxt->pTagVals); - if (!(pCxt->pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)))) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SKvParam param = {.pTagVals = pCxt->pTagVals, .pos = 0}; + int32_t code = TSDB_CODE_SUCCESS; + SArray *pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); SToken sToken; bool isParseBindParam = false; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - // TODO: JSON_TAG_REFACTOR => here would have json tag? for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); if (sToken.type == TK_NK_QUESTION) { isParseBindParam = true; if (NULL == pCxt->pStmtCb) { - return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + goto end; } continue; } if (isParseBindParam) { - return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); + code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); + goto end; } SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; - param.schema = pTagSchema; - CHECK_CODE( - parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); + char *tmpTokenBuf = taosMemoryCalloc(1, sToken.n); // this can be optimize with parse column + code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(tmpTokenBuf); + goto end; + } + if(pTagSchema->type == TSDB_DATA_TYPE_JSON){ + if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z); + taosMemoryFree(tmpTokenBuf); + goto end; + } + code = parseJsontoTagData(sToken.z, pTagVals, &pCxt->msg); + taosMemoryFree(tmpTokenBuf); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } + }else{ + STagVal val = {0}; + code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, &val, &pCxt->msg); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFree(tmpTokenBuf); + goto end; + } + if (pTagSchema->type != TSDB_DATA_TYPE_BINARY){ + taosMemoryFree(tmpTokenBuf); + } + taosArrayPush(pTagVals, &val); + } } if (isParseBindParam) { - return TSDB_CODE_SUCCESS; + code = TSDB_CODE_SUCCESS; + goto end; } - // TODO: JSON_TAG_REFACTOR (would be JSON tag or normal tag) STag* pTag = NULL; - if (tTagNew(param.pTagVals, 1, false, &pTag) != 0) { - return buildInvalidOperationMsg(&pCxt->msg, "out of memory"); + code = tTagNew(pTagVals, 1, false, &pTag); + if (code != TSDB_CODE_SUCCESS) { + goto end; } - return buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid); + buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid); + +end: + for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); + if(IS_VAR_DATA_TYPE(p->type)){ + taosMemoryFree(p->pData); + } + } + taosArrayDestroy(pTagVals); + return code; } static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { @@ -1072,7 +1248,6 @@ void destroyCreateSubTbReq(SVCreateTbReq* pReq) { static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); - taosArrayDestroy(pCxt->pTagVals); destroyCreateSubTbReq(&pCxt->createTblReq); } @@ -1348,41 +1523,75 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN return buildInvalidOperationMsg(&pBuf, "out of memory"); } + int32_t code = TSDB_CODE_SUCCESS; SSchema* pSchema = pDataBlock->pTableMeta->schema; - SKvParam param = {.pTagVals = pTagArray, .pos = 0}; for (int c = 0; c < tags->numOfBound; ++c) { if (bind[c].is_null && bind[c].is_null[0]) { - KvRowAppend(&pBuf, NULL, 0, ¶m); continue; } SSchema* pTagSchema = &pSchema[tags->boundColumns[c]]; - param.schema = pTagSchema; int32_t colLen = pTagSchema->bytes; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; } - CHECK_CODE(KvRowAppend(&pBuf, (char*)bind[c].buffer, colLen, ¶m)); + STagVal val = {0}; + if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ + val.pData = (uint8_t*)bind[c].buffer; + val.nData = colLen; + }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ + int32_t output = 0; + void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE); + if(p == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (errno == E2BIG) { + taosMemoryFree(p); + code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + goto end; + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + taosMemoryFree(p); + code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer); + goto end; + } + val.pData = p; + val.nData = output; + }else{ + memcpy(&val.i64, bind[c].buffer, colLen); + } + taosArrayPush(pTagArray, &val); } STag* pTag = NULL; - // TODO: JSON_TAG_REFACTOR (if is json or not)? + // TODO: stmt support json if (0 != tTagNew(pTagArray, 1, false, &pTag)) { - return buildInvalidOperationMsg(&pBuf, "out of memory"); + code = buildInvalidOperationMsg(&pBuf, "out of memory"); + goto end; } SVCreateTbReq tbReq = {0}; - CHECK_CODE(buildCreateTbReq(&tbReq, tName, pTag, suid)); - CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); - + buildCreateTbReq(&tbReq, tName, pTag, suid); + code = buildCreateTbMsg(pDataBlock, &tbReq); destroyCreateSubTbReq(&tbReq); + +end: + for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); + if(p->type == TSDB_DATA_TYPE_NCHAR){ + taosMemoryFree(p->pData); + } + } taosArrayDestroy(pTagArray); - return TSDB_CODE_SUCCESS; + return code; } int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { @@ -1714,25 +1923,52 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SKvParam param = {.pTagVals = pTagArray, .pos = 0}; + int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; - param.schema = pTagSchema; SSmlKv* kv = taosArrayGetP(cols, i); - if (IS_VAR_DATA_TYPE(kv->type)) { - KvRowAppend(msg, kv->value, kv->length, ¶m); - } else { - KvRowAppend(msg, &(kv->value), kv->length, ¶m); + + STagVal val = {0}; + if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ + val.pData = (uint8_t *)kv->value; + val.nData = kv->length; + }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ + int32_t output = 0; + void *p = taosMemoryCalloc(1, kv->length * TSDB_NCHAR_SIZE); + if(p == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (errno == E2BIG) { + taosMemoryFree(p); + code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + goto end; + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + taosMemoryFree(p); + code = buildSyntaxErrMsg(msg, buf, kv->value); + goto end; + } + val.pData = p; + val.nData = output; + }else{ + memcpy(&val.i64, &(kv->value), kv->length); + } + taosArrayPush(pTagArray, &val); + } + + code = tTagNew(pTagArray, 1, false, ppTag); +end: + for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); + if(p->type == TSDB_DATA_TYPE_NCHAR){ + taosMemoryFree(p->pData); } } - - if (tTagNew(pTagArray, 1, false, ppTag) != 0) { - taosArrayDestroy(pTagArray); - return TSDB_CODE_OUT_OF_MEMORY; - } - taosArrayDestroy(pTagArray); - return TSDB_CODE_SUCCESS; + return code; } int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 22357cab44..96ee694e1f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4144,26 +4144,6 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S } } -// static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, -// SKVRowBuilder* pBuilder) { -#ifdef JSON_TAG_REFACTOR -if (pSchema->type == TSDB_DATA_TYPE_JSON) { - if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); - } - - return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId); -} -#endif - -// if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { -// tdAddColToKVRow(pBuilder, pSchema->colId, nodesGetValueFromNode(pVal), -// IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); -// } - -// return TSDB_CODE_SUCCESS; -// } - static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* pFunc, SValueNode** pVal) { int32_t code = getFuncInfo(pCxt, pFunc); if (TSDB_CODE_SUCCESS == code) { @@ -4199,16 +4179,14 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } SArray* pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); - char* pTagBuf = taosMemoryCalloc(1, TSDB_MAX_TAGS_LEN); - if (!pTagArray || !pTagBuf) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); + if (!pTagArray) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY); } - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; int16_t nTags = 0, nBufPos = 0; SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); - SNode * pTag, *pNode; + SNode * pTag = NULL, *pNode = NULL; + bool isJson = false; FORBOTH(pTag, pStmt->pSpecificTags, pNode, pStmt->pValsOfTags) { SColumnNode* pCol = (SColumnNode*)pTag; SSchema* pSchema = NULL; @@ -4219,57 +4197,56 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } if (NULL == pSchema) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName); + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName); + goto end; } SValueNode* pVal = NULL; code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pSchema, pNode, &pVal); - if (TSDB_CODE_SUCCESS == code) { - if (NULL == pVal) { - pVal = (SValueNode*)pNode; - } else { - REPLACE_LIST2_NODE(pVal); - } - } -#ifdef JSON_TAG_REFACTOR - if (TSDB_CODE_SUCCESS == code) { - code = addValToKVRow(pCxt, pVal, pSchema, pBuilder); - } -#endif - - if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - // TODO: JSON_TAG_TODO: is copy is a must? - void* nodeVal = nodesGetValueFromNode(pVal); - if (IS_VAR_DATA_TYPE(pSchema->type)) { - memcpy(pTagBuf + nBufPos, varDataVal(nodeVal), varDataLen(nodeVal)); - tTagValPush(pTagArray, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, varDataLen(nodeVal), false); - nBufPos += varDataLen(pVal->datum.p); - } else { - memcpy(pTagBuf + nBufPos, varDataVal(nodeVal), TYPE_BYTES[pSchema->type]); - tTagValPush(pTagArray, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, TYPE_BYTES[pSchema->type], - false); - nBufPos += TYPE_BYTES[pSchema->type]; - } - } - if (TSDB_CODE_SUCCESS != code) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); - return code; + goto end; + } + + if (NULL == pVal) { + pVal = (SValueNode*)pNode; + } else { + REPLACE_LIST2_NODE(pVal); + } + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); + goto end; + } + + isJson = true; + code = parseJsontoTagData(pVal->literal, pTagArray, &pCxt->msgBuf); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } + }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + void* nodeVal = nodesGetValueFromNode(pVal); + STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + val.pData = varDataVal(nodeVal); + val.nData = varDataLen(nodeVal); + } else { + memcpy(&val.i64, nodeVal, pTagSchema->bytes); + } + taosArrayPush(pTagArray, &val); } } - // TODO: JSON_TAG_TODO: version code = tTagNew(pTagArray, 1, false, ppTag); - if (TSDB_CODE_SUCCESS != code) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); - return code; - } +end: + if(isJson){ + for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); + if(IS_VAR_DATA_TYPE(p->type)){ + taosMemoryFree(p->pData); + } + } + } taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); return TSDB_CODE_SUCCESS; } @@ -4281,71 +4258,64 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau SSchema* pTagSchemas = getTableTagSchema(pSuperTableMeta); SNode* pNode; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; SArray* pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); - char* pTagBuf = taosMemoryCalloc(1, TSDB_MAX_TAGS_LEN); - - const char* qTagBuf = pTagBuf; - - if (!pTagArray || !pTagBuf) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(qTagBuf); + if (!pTagArray) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY); } + bool isJson = false; FOREACH(pNode, pStmt->pValsOfTags) { SValueNode* pVal = NULL; SSchema* pTagSchema = pTagSchemas + index; code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pTagSchema, pNode, &pVal); - if (TSDB_CODE_SUCCESS == code) { - if (NULL == pVal) { - pVal = (SValueNode*)pNode; - } else { - REPLACE_NODE(pVal); - } - } -#ifdef JSON_TAG_REFACTOR - if (TSDB_CODE_SUCCESS == code) { - code = addValToKVRow(pCxt, pVal, pTagSchema + index++, pBuilder); - } -#endif - if (TSDB_CODE_SUCCESS == code) { - if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - char* tmpVal = nodesGetValueFromNode(pVal); - if (IS_VAR_DATA_TYPE(pTagSchema->type)) { - memcpy(pTagBuf, varDataVal(tmpVal), varDataLen(tmpVal)); - tTagValPush(pTagArray, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, varDataLen(tmpVal), false); - pTagBuf += varDataLen(tmpVal); - } else { - memcpy(pTagBuf, tmpVal, TYPE_BYTES[pTagSchema->type]); - tTagValPush(pTagArray, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, TYPE_BYTES[pTagSchema->type], - false); - pTagBuf += TYPE_BYTES[pTagSchema->type]; - } - } - ++index; - } - // TODO: buf is need to store the tags - - // TODO: JSON_TAG_TODO remove below codes if code is 0 all the time. if (TSDB_CODE_SUCCESS != code) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(qTagBuf); - return generateSyntaxErrMsg(&pCxt->msgBuf, code); + goto end; } + if (NULL == pVal) { + pVal = (SValueNode*)pNode; + } else { + REPLACE_NODE(pVal); + } + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); + goto end; + } + + isJson = true; + code = parseJsontoTagData(pVal->literal, pTagArray, &pCxt->msgBuf); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } + }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + char* tmpVal = nodesGetValueFromNode(pVal); + STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + val.pData = varDataVal(tmpVal); + val.nData = varDataLen(tmpVal); + } else { + memcpy(&val.i64, tmpVal, pTagSchema->bytes); + } + taosArrayPush(pTagArray, &val); + } + ++index; } - // TODO: JSON_TAG_TODO: version? - // TODO: JSON_TAG_REFACTOR: json or not - if (0 != (code = tTagNew(pTagArray, 1, false, ppTag))) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(qTagBuf); - return generateSyntaxErrMsg(&pCxt->msgBuf, code); + code = tTagNew(pTagArray, 1, false, ppTag); + +end: + if(isJson){ + for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); + if(IS_VAR_DATA_TYPE(p->type)){ + taosMemoryFree(p->pData); + } + } } taosArrayDestroy(pTagArray); - taosMemoryFreeClear(qTagBuf); - return TSDB_CODE_SUCCESS; + return code; } static int32_t checkCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt) { @@ -4602,39 +4572,42 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); if (pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON) { -#ifdef JSON_TAG_REFACTOR - SKVRowBuilder kvRowBuilder = {0}; - int32_t code = tdInitKVRowBuilder(&kvRowBuilder); - - if (TSDB_CODE_SUCCESS != code) { - return TSDB_CODE_OUT_OF_MEMORY; - } if (pStmt->pVal->literal && strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal); } - - code = parseJsontoTagData(pStmt->pVal->literal, &kvRowBuilder, &pCxt->msgBuf, pSchema->colId); - if (TSDB_CODE_SUCCESS != code) { + SArray *pTagVals = taosArrayInit(1, sizeof(STagVal)); + int32_t code = TSDB_CODE_SUCCESS; + STag* pTag = NULL; + do{ + code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pCxt->msgBuf); + if (TSDB_CODE_SUCCESS != code) { + break; + } + code = tTagNew(pTagVals, 1, false, &pTag); + }while(0); + for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); + if(IS_VAR_DATA_TYPE(p->type)){ + taosMemoryFree(p->pData); + } + } + taosArrayDestroy(pTagVals); + if (code != TSDB_CODE_SUCCESS){ return code; } - - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - if (NULL == row) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return TSDB_CODE_OUT_OF_MEMORY; - } - pReq->nTagVal = kvRowLen(row); - pReq->pTagVal = row; - pStmt->pVal->datum.p = row; // for free - tdDestroyKVRowBuilder(&kvRowBuilder); -#endif + pReq->nTagVal = pTag->len; + pReq->pTagVal = (uint8_t *)pTag; + pStmt->pVal->datum.p = (char*)pTag; // for free } else { pReq->nTagVal = pStmt->pVal->node.resType.bytes; - if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) { - pReq->nTagVal = pReq->nTagVal * TSDB_NCHAR_SIZE; - } pReq->pTagVal = nodesGetValueFromNode(pStmt->pVal); + + // data and length are seperated for new tag format STagVal + if (IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type)) { + pReq->nTagVal = varDataLen(pReq->pTagVal); + pReq->pTagVal = varDataVal(pReq->pTagVal); + } } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index d070ffaf5c..726028355f 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -328,13 +328,9 @@ static bool isValidateTag(char* input) { return true; } -#ifdef JSON_TAG_REFACTOR -int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId) { +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* pMsgBuf) { // set json NULL data - uint8_t jsonNULL = TSDB_DATA_TYPE_NULL; - int32_t jsonIndex = startColId + 1; if (!json || strtrim((char*)json) == 0 || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) { - tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); return TSDB_CODE_SUCCESS; } @@ -349,13 +345,12 @@ int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBu return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); } - int32_t retCode = 0; - char* tagKV = NULL; + int32_t retCode = TSDB_CODE_SUCCESS; SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); for (int32_t i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); if (!item) { - qError("json inner error:%d", i); + uError("json inner error:%d", i); retCode = buildSyntaxErrMsg(pMsgBuf, "json inner error", json); goto end; } @@ -367,89 +362,60 @@ int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBu } size_t keyLen = strlen(jsonKey); if (keyLen > TSDB_MAX_JSON_KEY_LEN) { - qError("json key too long error"); + uError("json key too long error"); retCode = buildSyntaxErrMsg(pMsgBuf, "json key too long, more than 256", jsonKey); goto end; } if (keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL) { continue; } - // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: DOUBLE_BYTES - tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + DOUBLE_BYTES, 1); - if (!tagKV) { - retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto end; - } - strncpy(varDataVal(tagKV), jsonKey, keyLen); - varDataSetLen(tagKV, keyLen); - if (taosHashGetSize(keyHash) == 0) { - uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type - } - taosHashPut(keyHash, jsonKey, keyLen, &keyLen, - CHAR_BYTES); // add key to hash to remove dumplicate, value is useless + STagVal val = {0}; + val.pKey = jsonKey; + taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless if (item->type == cJSON_String) { // add json value format: type|data char* jsonValue = item->valuestring; int32_t valLen = (int32_t)strlen(jsonValue); - int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES; - char* tmp = taosMemoryRealloc(tagKV, totalLen); + char* tmp = taosMemoryCalloc(1, valLen * TSDB_NCHAR_SIZE); if (!tmp) { retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; goto end; } - tagKV = tmp; - char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); - char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); - *valueType = TSDB_DATA_TYPE_NCHAR; - if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData), + val.type = TSDB_DATA_TYPE_NCHAR; + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { - qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, + uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); goto end; } - - varDataSetLen(valueData, valLen); - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen); + val.nData = valLen; + val.pData = tmp; } else if (item->type == cJSON_Number) { if (!isfinite(item->valuedouble)) { - qError("json value is invalidate"); + uError("json value is invalidate"); retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json); goto end; } - char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); - char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); - *valueType = TSDB_DATA_TYPE_DOUBLE; - *((double*)valueData) = item->valuedouble; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + DOUBLE_BYTES); + val.type = TSDB_DATA_TYPE_DOUBLE; + *((double*)&(val.i64)) = item->valuedouble; } else if (item->type == cJSON_True || item->type == cJSON_False) { - char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); - char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); - *valueType = TSDB_DATA_TYPE_BOOL; - *valueData = (char)(item->valueint); - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES); + val.type = TSDB_DATA_TYPE_BOOL; + *((char*)&(val.i64)) = (char)(item->valueint); } else if (item->type == cJSON_NULL) { - char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); - *valueType = TSDB_DATA_TYPE_NULL; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); + val.type = TSDB_DATA_TYPE_NULL; } else { retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json); goto end; } - } - - if (taosHashGetSize(keyHash) == 0) { // set json NULL true - tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); + taosArrayPush(pTagVals, &val); } end: - taosMemoryFree(tagKV); taosHashCleanup(keyHash); cJSON_Delete(root); return retCode; } -#endif static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) { if (NULL != pTablesHash) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index b68c292feb..c4b3809035 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -922,29 +922,12 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } -char *getJsonValue(char *json, char *key) { // todo - json++; // jump type - - STagVal tagVal = {.pKey = key}; - tTagGet(((const STag *)json), &tagVal); - return (char *)tagVal.pData; -#if 0 - int16_t cols = kvRowNCols(json); - for (int i = 0; i < cols; ++i) { - SColIdx *pColIdx = kvRowColIdxAt(json, i); - char *data = kvRowColVal(json, pColIdx); - if(i == 0){ - if(*data == TSDB_DATA_TYPE_NULL) { - return NULL; - } - continue; - } - if(memcmp(key, data, varDataTLen(data)) == 0){ - return data + varDataTLen(data); - } +STagVal *getJsonValue(char *json, STagVal *tagVal) { + bool find = tTagGet(((const STag *)json), tagVal); // json value is null and not exist is different + if(!find){ + return NULL; } - return NULL; -#endif + return tagVal; } void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { @@ -963,13 +946,13 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO continue; } char *pLeftData = colDataGetVarData(pLeft->columnData, i); - char *value = getJsonValue(pLeftData, pRightData); - if (!value) { - colDataSetNull_var(pOutputCol, i); - pOutputCol->hasNull = true; - continue; + STagVal val = {.pKey = pRightData}; + STagVal *value = getJsonValue(pLeftData, &val); + char *data = tTagValToData(value, true); + colDataAppend(pOutputCol, i, data, data == NULL); + if(value && IS_VAR_DATA_TYPE(value->type) && data){ + taosMemoryFree(data) } - colDataAppend(pOutputCol, i, value, false); } } From 64ffbc1eea1aa3674474315b707eebd9c4af3f88 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 May 2022 18:34:17 +0800 Subject: [PATCH 07/57] enh: support tag filter --- include/libs/index/index.h | 1 + source/dnode/vnode/inc/vnode.h | 12 +++ source/dnode/vnode/src/inc/meta.h | 8 +- source/dnode/vnode/src/meta/metaQuery.c | 119 +++++++++++++++++++----- source/dnode/vnode/src/meta/metaTable.c | 4 +- source/libs/executor/src/executorimpl.c | 12 +-- source/libs/index/CMakeLists.txt | 1 + source/libs/index/inc/indexComm.h | 5 +- source/libs/index/src/index.c | 14 +-- source/libs/index/src/indexComm.c | 5 +- source/libs/index/src/indexFilter.c | 64 ++++++++++++- source/libs/index/src/indexJson.c | 6 +- source/libs/tdb/src/db/tdbTable.c | 8 +- 13 files changed, 206 insertions(+), 53 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index c3d31ffe38..180c7e7216 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -194,6 +194,7 @@ void indexInit(); /* index filter */ typedef struct SIndexMetaArg { void* metaHandle; + void* metaEx; uint64_t suid; } SIndexMetaArg; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e4343e3bbf..9cfcf2b086 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -80,6 +80,18 @@ int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid); +typedef struct SMetaFltParam { + tb_uid_t suid; + int16_t cid; + int16_t type; + char * val; + bool reverse; + int (*filterFunc)(void *a, void *b, int16_t type); + +} SMetaFltParam; + +int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); + #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; typedef SVCreateTSmaReq SSmaCfg; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 3340bbb91c..9da603f894 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -16,8 +16,8 @@ #ifndef _TD_VNODE_META_H_ #define _TD_VNODE_META_H_ -#include "vnodeInt.h" #include "index.h" +#include "vnodeInt.h" #ifdef __cplusplus extern "C" { @@ -45,8 +45,6 @@ int32_t metaULock(SMeta* pMeta); int metaEncodeEntry(SEncoder* pCoder, const SMetaEntry* pME); int metaDecodeEntry(SDecoder* pCoder, SMetaEntry* pME); -// metaTable ================== - // metaQuery ================== int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid); @@ -118,6 +116,10 @@ typedef struct { int64_t smaUid; } SSmaIdxKey; +// metaTable ================== +int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int8_t type, tb_uid_t uid, + STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey); + #ifndef META_REFACT // SMetaDB int metaOpenDB(SMeta* pMeta); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 605e804933..71429ff176 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -31,7 +31,7 @@ void metaReaderClear(SMetaReader *pReader) { } int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t uid) { - SMeta *pMeta = pReader->pMeta; + SMeta * pMeta = pReader->pMeta; STbDbKey tbDbKey = {.version = version, .uid = uid}; // query table.db @@ -54,7 +54,7 @@ _err: } int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { - SMeta *pMeta = pReader->pMeta; + SMeta * pMeta = pReader->pMeta; int64_t version; // query uid.idx @@ -68,7 +68,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { } int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { - SMeta *pMeta = pReader->pMeta; + SMeta * pMeta = pReader->pMeta; tb_uid_t uid; // query name.idx @@ -82,7 +82,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { } tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { - void *pData = NULL; + void * pData = NULL; int nData = 0; tb_uid_t uid = 0; @@ -134,7 +134,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { int metaTbCursorNext(SMTbCursor *pTbCur) { int ret; - void *pBuf; + void * pBuf; STbCfg tbCfg; for (;;) { @@ -155,7 +155,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) { } SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { - void *pData = NULL; + void * pData = NULL; int nData = 0; int64_t version; SSchemaWrapper schema = {0}; @@ -204,11 +204,11 @@ _err: } struct SMCtbCursor { - SMeta *pMeta; - TBC *pCur; + SMeta * pMeta; + TBC * pCur; tb_uid_t suid; - void *pKey; - void *pVal; + void * pKey; + void * pVal; int kLen; int vLen; }; @@ -280,10 +280,10 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { tb_uid_t quid; SMetaReader mr = {0}; - STSchema *pTSchema = NULL; + STSchema * pTSchema = NULL; SSchemaWrapper *pSW = NULL; STSchemaBuilder sb = {0}; - SSchema *pSchema; + SSchema * pSchema; metaReaderInit(&mr, pMeta, 0); metaGetTableEntryByUid(&mr, uid); @@ -320,11 +320,11 @@ int metaGetTbNum(SMeta *pMeta) { } typedef struct { - SMeta *pMeta; - TBC *pCur; + SMeta * pMeta; + TBC * pCur; tb_uid_t uid; - void *pKey; - void *pVal; + void * pKey; + void * pVal; int kLen; int vLen; } SMSmaCursor; @@ -396,7 +396,7 @@ tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) { STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { STSmaWrapper *pSW = NULL; - SArray *pSmaIds = NULL; + SArray * pSmaIds = NULL; if (!(pSmaIds = metaGetSmaIdsByTable(pMeta, uid))) { return NULL; @@ -420,7 +420,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { metaReaderInit(&mr, pMeta, 0); int64_t smaId; int smaIdx = 0; - STSma *pTSma = NULL; + STSma * pTSma = NULL; for (int i = 0; i < pSW->number; ++i) { smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i); if (metaGetTableEntryByUid(&mr, smaId) < 0) { @@ -468,7 +468,7 @@ _err: } STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { - STSma *pTSma = NULL; + STSma * pTSma = NULL; SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByUid(&mr, indexUid) < 0) { @@ -490,7 +490,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { } SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) { - SArray *pUids = NULL; + SArray * pUids = NULL; SSmaIdxKey *pSmaIdxKey = NULL; SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); @@ -528,7 +528,7 @@ SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) { } SArray *metaGetSmaTbUids(SMeta *pMeta) { - SArray *pUids = NULL; + SArray * pUids = NULL; SSmaIdxKey *pSmaIdxKey = NULL; tb_uid_t lastUid = 0; @@ -575,4 +575,79 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid) { ASSERT(pEntry->type == TSDB_CHILD_TABLE); return tdGetKVRowValOfCol((const SKVRow)pEntry->ctbEntry.pTags, cid); -} \ No newline at end of file +} +typedef struct { + SMeta * pMeta; + TBC * pCur; + tb_uid_t suid; + int16_t cid; + int16_t type; + void * pKey; + void * pVal; + int32_t kLen; + int32_t vLen; +} SIdxCursor; + +int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { + SIdxCursor *pCursor = NULL; + + char *tagData = param->val; + + int32_t ret = 0, valid = 0; + pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor)); + pCursor->pMeta = pMeta; + pCursor->suid = param->suid; + pCursor->cid = param->cid; + pCursor->type = param->type; + + metaRLock(pMeta); + ret = tdbTbcOpen(pMeta->pTagIdx, &pCursor->pCur, NULL); + if (ret < 0) { + goto END; + } + STagIdxKey *pKey = NULL; + int32_t nKey = 0; + + ret = metaCreateTagIdxKey(pCursor->suid, pCursor->cid, param->val, pCursor->type, + param->reverse ? INT64_MAX : INT64_MIN, &pKey, &nKey); + if (ret != 0) { + goto END; + } + int cmp = 0; + if (tdbTbcMoveTo(pCursor->pCur, pKey, nKey, &cmp) < 0) { + goto END; + } + void * entryKey = NULL, *entryVal = NULL; + int32_t nEntryKey, nEntryVal; + while (1) { + valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); + if (valid < 0) { + break; + } + STagIdxKey *p = entryKey; + if (p != NULL) { + int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); + if (cmp == 0) { + // match + tb_uid_t tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); + taosArrayPush(pUids, &tuid); + } else if (cmp == 1) { + // not match but should continue to iter + } else { + // not match and no more result + break; + } + } + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) { + break; + } + } +END: + if (pCursor->pMeta) metaULock(pCursor->pMeta); + if (pCursor->pCur) tdbTbcClose(pCursor->pCur); + + taosMemoryFree(pCursor); + + return ret; +} diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index f610f18126..665d571c8f 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -721,8 +721,8 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } -static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, - STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { +int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, + STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { int32_t nTagData = 0; if (pTagData) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 690e2d4bf6..c36b341546 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4495,8 +4495,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - STimeWindowAggSupp twSup = {.waterMark = pTableScanNode->watermark, - .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; + STimeWindowAggSupp twSup = { + .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; tsdbReaderT pDataReader = NULL; if (pHandle->vnode) { pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); @@ -4510,9 +4510,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else { qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); } - SArray* tableIdList = extractTableIdList(pTableListInfo); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, - tableIdList, pTableScanNode, pTaskInfo, &twSup, pTableScanNode->tsColId); + SArray* tableIdList = extractTableIdList(pTableListInfo); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, tableIdList, pTableScanNode, + pTaskInfo, &twSup, pTableScanNode->tsColId); taosArrayDestroy(tableIdList); return pOperator; @@ -4912,7 +4912,7 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa if (tableType == TSDB_SUPER_TABLE) { if (pTagCond) { - SIndexMetaArg metaArg = {.metaHandle = tsdbGetIdx(metaHandle), .suid = tableUid}; + SIndexMetaArg metaArg = {.metaEx = metaHandle, .metaHandle = tsdbGetIdx(metaHandle), .suid = tableUid}; SArray* res = taosArrayInit(8, sizeof(uint64_t)); code = doFilterTag(pTagCond, &metaArg, res); diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index e55b004972..75eac2430f 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common + PUBLIC vnode PUBLIC nodes PUBLIC scalar PUBLIC function diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 3066fd1c2c..c338300b57 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -33,8 +33,9 @@ typedef enum { MATCH, CONTINUE, BREAK } TExeCond; typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); -TExeCond tCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b, int8_t dType); -TExeCond tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b); +__compar_fn_t indexGetCompar(int8_t type); +TExeCond tCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b, int8_t dType); +TExeCond tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b); _cache_range_compare indexGetCompare(RangeType ty); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 8584d95bf2..4814cc14f7 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -80,7 +80,7 @@ static TdThreadOnce isInit = PTHREAD_ONCE_INIT; static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); static void indexInterResultsDestroy(SArray* results); -static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult); +static int indexMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out); static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); @@ -386,21 +386,21 @@ static void indexInterResultsDestroy(SArray* results) { taosArrayDestroy(results); } -static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { +static int indexMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out) { // refactor, merge interResults into fResults by oType - for (int i = 0; i < taosArrayGetSize(interResults); i--) { - SArray* t = taosArrayGetP(interResults, i); + for (int i = 0; i < taosArrayGetSize(in); i--) { + SArray* t = taosArrayGetP(in, i); taosArraySort(t, uidCompare); taosArrayRemoveDuplicate(t, uidCompare, NULL); } if (oType == MUST) { - iIntersection(interResults, fResults); + iIntersection(in, out); } else if (oType == SHOULD) { - iUnion(interResults, fResults); + iUnion(in, out); } else if (oType == NOT) { // just one column index, enhance later - taosArrayAddAll(fResults, interResults); + // taosArrayAddAll(fResults, interResults); // not use currently } return 0; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 78c7babb68..5310e1c345 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -75,7 +75,7 @@ char* indexInt2str(int64_t val, char* dst, int radix) { ; return dst - 1; } -static __compar_fn_t indexGetCompar(int8_t type) { +__compar_fn_t indexGetCompar(int8_t type) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { return (__compar_fn_t)strcmp; } @@ -182,6 +182,9 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { case QUERY_GREATER_EQUAL: { if (ret >= 0) return MATCH; } + case QUERY_TERM: { + if (ret == 0) return MATCH; + } } return CONTINUE; } diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index b882caa168..766746dd2a 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -14,11 +14,13 @@ */ #include "index.h" +#include "indexComm.h" #include "indexInt.h" #include "nodes.h" #include "querynodes.h" #include "scalar.h" #include "tdatablock.h" +#include "vnode.h" // clang-format off #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) @@ -259,10 +261,52 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu indexError("index-filter not support buildin function"); return TSDB_CODE_QRY_INVALID_INPUT; } + +typedef int (*Filter)(void *a, void *b, int16_t dtype); + +int sifGreaterThan(void *a, void *b, int16_t dtype) { + __compar_fn_t func = indexGetCompar(dtype); + return tDoCompare(func, QUERY_GREATER_THAN, a, b); +} +int sifGreaterEqual(void *a, void *b, int16_t dtype) { + __compar_fn_t func = indexGetCompar(dtype); + return tDoCompare(func, QUERY_GREATER_EQUAL, a, b); +} +int sifLessEqual(void *a, void *b, int16_t dtype) { + __compar_fn_t func = indexGetCompar(dtype); + return tDoCompare(func, QUERY_LESS_EQUAL, a, b); +} +int sifLessThan(void *a, void *b, int16_t dtype) { + __compar_fn_t func = indexGetCompar(dtype); + return (int)tDoCompare(func, QUERY_LESS_THAN, a, b); +} +int sifEqual(void *a, void *b, int16_t dtype) { + __compar_fn_t func = indexGetCompar(dtype); + return (int)tDoCompare(func, QUERY_TERM, a, b); +} +static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { + if (type == QUERY_LESS_EQUAL || type == QUERY_LESS_THAN) { + *reverse = true; + } else { + *reverse = false; + } + if (type == QUERY_LESS_EQUAL) + return sifLessEqual; + else if (type == QUERY_LESS_THAN) + return sifLessThan; + else if (type == QUERY_GREATER_EQUAL) + return sifGreaterEqual; + else if (type == QUERY_GREATER_THAN) + return sifGreaterThan; + else if (type == QUERY_TERM) { + return sifEqual; + } + return NULL; +} static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { -#ifdef USE_INVERTED_INDEX SIndexMetaArg *arg = &output->arg; - SIndexTerm * tm = indexTermCreate(arg->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), +#ifdef USE_INVERTED_INDEX + SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), right->condValue, strlen(right->condValue)); if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -278,8 +322,22 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP indexMultiTermQueryDestroy(mtm); return ret; #else - return 0; + EIndexQueryType qtype = 0; + SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); + bool reverse; + Filter filterFunc = sifGetFilterFunc(qtype, &reverse); + + SMetaFltParam param = {.suid = arg->suid, + .cid = left->colId, + .type = left->colValType, + .val = right->condValue, + .reverse = reverse, + .filterFunc = filterFunc}; + + int ret = metaFilteTableIds(arg->metaEx, ¶m, output->result); + return ret; #endif + return 0; } static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { diff --git a/source/libs/index/src/indexJson.c b/source/libs/index/src/indexJson.c index de88ff3c8a..a2f0563d47 100644 --- a/source/libs/index/src/indexJson.c +++ b/source/libs/index/src/indexJson.c @@ -24,8 +24,8 @@ int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { SIndexJsonTerm *p = taosArrayGetP(terms, i); INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); } - return indexPut(index, terms, uid); // handle put + return indexPut(index, terms, uid); } int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *result) { @@ -34,11 +34,11 @@ int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *re SIndexJsonTerm *p = taosArrayGetP(terms, i); INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); } - return indexSearch(index, tq, result); // handle search + return indexSearch(index, tq, result); } void tIndexJsonClose(SIndexJson *index) { - return indexClose(index); // handle close + return indexClose(index); } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 7211fe4926..239aa5d7ef 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -16,7 +16,7 @@ #include "tdbInt.h" struct STTB { - TDB *pEnv; + TDB * pEnv; SBTree *pBt; }; @@ -25,11 +25,11 @@ struct STBC { }; int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { - TTB *pTb; + TTB * pTb; SPager *pPager; int ret; char fFullName[TDB_FILENAME_LEN]; - SPage *pPage; + SPage * pPage; SPgno pgno; *ppTb = NULL; @@ -145,4 +145,4 @@ int tdbTbcClose(TBC *pTbc) { return 0; } -int tdbTbcIsValid(TBC *pTbc) { return tdbBtcIsValid(&pTbc->btc); } \ No newline at end of file +int tdbTbcIsValid(TBC *pTbc) { return tdbBtcIsValid(&pTbc->btc); } From 0bd8ba5f8e0d8f2673ef81191b4c51a96a1feac0 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 31 May 2022 18:58:31 +0800 Subject: [PATCH 08/57] test: finish 1-insert/opentsdb_json_taosc_insert.py --- .../1-insert/opentsdb_json_taosc_insert.py | 1788 +++++++++++++++++ 1 file changed, 1788 insertions(+) create mode 100644 tests/system-test/1-insert/opentsdb_json_taosc_insert.py diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py new file mode 100644 index 0000000000..360d878dfe --- /dev/null +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -0,0 +1,1788 @@ +################################################################### +# Copyright (c) 2021 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import traceback +import random +from taos.error import SchemalessError +import time +from util.log import * +from util.cases import * +from util.sql import * +from util.common import tdCom +from util.types import TDSmlProtocolType +import threading +import json + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self._conn = conn + self.defaultJSONStrType_value = "NCHAR" + + def createDb(self, name="test", db_update_tag=0, protocol=None): + if protocol == "telnet-tcp": + name = "opentsdb_telnet" + + if db_update_tag == 0: + tdSql.execute(f"drop database if exists {name}") + tdSql.execute(f"create database if not exists {name} precision 'ms'") + else: + tdSql.execute(f"drop database if exists {name}") + tdSql.execute(f"create database if not exists {name} precision 'ms' update 1") + tdSql.execute(f'use {name}') + + def timeTrans(self, time_value): + if type(time_value) is int: + if time_value != 0: + if len(str(time_value)) == 13: + ts = int(time_value)/1000 + elif len(str(time_value)) == 10: + ts = int(time_value)/1 + else: + ts = time_value/1000000 + else: + ts = time.time() + elif type(time_value) is dict: + if time_value["type"].lower() == "ns": + ts = time_value["value"]/1000000000 + elif time_value["type"].lower() == "us": + ts = time_value["value"]/1000000 + elif time_value["type"].lower() == "ms": + ts = time_value["value"]/1000 + elif time_value["type"].lower() == "s": + ts = time_value["value"]/1 + else: + ts = time_value["value"]/1000000 + ulsec = repr(ts).split('.')[1][:6] + if len(ulsec) < 6 and int(ulsec) != 0: + ulsec = int(ulsec) * (10 ** (6 - len(ulsec))) + elif int(ulsec) == 0: + ulsec *= 6 + # * follow two rows added for tsCheckCase + td_ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) + return td_ts + #td_ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) + td_ts = time.strftime("%Y-%m-%d %H:%M:%S.{}".format(ulsec), time.localtime(ts)) + return td_ts + + def dateToTs(self, datetime_input): + return int(time.mktime(time.strptime(datetime_input, "%Y-%m-%d %H:%M:%S.%f"))) + + def typeTrans(self, type_list): + type_num_list = [] + for tp in type_list: + if type(tp) is dict: + tp = tp['type'] + if tp.upper() == "TIMESTAMP": + type_num_list.append(9) + elif tp.upper() == "BOOL": + type_num_list.append(1) + elif tp.upper() == "TINYINT": + type_num_list.append(2) + elif tp.upper() == "SMALLINT": + type_num_list.append(3) + elif tp.upper() == "INT": + type_num_list.append(4) + elif tp.upper() == "BIGINT": + type_num_list.append(5) + elif tp.upper() == "FLOAT": + type_num_list.append(6) + elif tp.upper() == "DOUBLE": + type_num_list.append(7) + elif tp.upper() == "VARCHAR": + type_num_list.append(8) + elif tp.upper() == "NCHAR": + type_num_list.append(10) + elif tp.upper() == "BIGINT UNSIGNED": + type_num_list.append(14) + return type_num_list + + def inputHandle(self, input_json): + stb_name = input_json["metric"] + stb_tag_dict = input_json["tags"] + stb_col_dict = input_json["value"] + ts_value = self.timeTrans(input_json["timestamp"]) + tag_name_list = [] + tag_value_list = [] + td_tag_value_list = [] + td_tag_type_list = [] + + col_name_list = [] + col_value_list = [] + td_col_value_list = [] + td_col_type_list = [] + + # handle tag + for key,value in stb_tag_dict.items(): + if "id" == key.lower(): + tb_name = value + else: + if type(value) is dict: + tag_value_list.append(str(value["value"])) + td_tag_value_list.append(str(value["value"])) + tag_name_list.append(key.lower()) + if value["type"].lower() == "binary": + td_tag_type_list.append("VARCHAR") + else: + td_tag_type_list.append(value["type"].upper()) + tb_name = "" + else: + tag_value_list.append(str(value)) + # td_tag_value_list.append(str(value)) + tag_name_list.append(key.lower()) + tb_name = "" + + if type(value) is bool: + td_tag_type_list.append("BOOL") + td_tag_value_list.append(str(value)) + elif type(value) is int: + td_tag_type_list.append("DOUBLE") + td_tag_value_list.append(str(float(value))) + elif type(value) is float: + td_tag_type_list.append("DOUBLE") + td_tag_value_list.append(str(float(value))) + elif type(value) is str: + if self.defaultJSONStrType_value == "NCHAR": + td_tag_type_list.append("NCHAR") + td_tag_value_list.append(str(value)) + else: + td_tag_type_list.append("VARCHAR") + td_tag_value_list.append(str(value)) + + # handle col + if type(stb_col_dict) is dict: + if stb_col_dict["type"].lower() == "bool": + bool_value = f'{stb_col_dict["value"]}' + col_value_list.append(bool_value) + td_col_type_list.append(stb_col_dict["type"].upper()) + col_name_list.append("_value") + td_col_value_list.append(str(stb_col_dict["value"])) + else: + col_value_list.append(stb_col_dict["value"]) + if stb_col_dict["type"].lower() == "binary": + td_col_type_list.append("VARCHAR") + else: + td_col_type_list.append(stb_col_dict["type"].upper()) + col_name_list.append("_value") + td_col_value_list.append(str(stb_col_dict["value"])) + else: + col_name_list.append("_value") + col_value_list.append(str(stb_col_dict)) + # td_col_value_list.append(str(stb_col_dict)) + if type(stb_col_dict) is bool: + td_col_type_list.append("BOOL") + td_col_value_list.append(str(stb_col_dict)) + elif type(stb_col_dict) is int: + td_col_type_list.append("DOUBLE") + td_col_value_list.append(str(float(stb_col_dict))) + elif type(stb_col_dict) is float: + td_col_type_list.append("DOUBLE") + td_col_value_list.append(str(float(stb_col_dict))) + elif type(stb_col_dict) is str: + if self.defaultJSONStrType_value == "NCHAR": + td_col_type_list.append("NCHAR") + td_col_value_list.append(str(stb_col_dict)) + else: + td_col_type_list.append("VARCHAR") + td_col_value_list.append(str(stb_col_dict)) + + final_field_list = [] + final_field_list.extend(col_name_list) + final_field_list.extend(tag_name_list) + + final_type_list = [] + final_type_list.append("TIMESTAMP") + final_type_list.extend(td_col_type_list) + final_type_list.extend(td_tag_type_list) + final_type_list = self.typeTrans(final_type_list) + + final_value_list = [] + final_value_list.append(ts_value) + final_value_list.extend(td_col_value_list) + final_value_list.extend(td_tag_value_list) + return final_value_list, final_field_list, final_type_list, stb_name, tb_name + + def genTsColValue(self, value, t_type=None, value_type="obj"): + if value_type == "obj": + if t_type == None: + ts_col_value = value + else: + ts_col_value = {"value": value, "type": t_type} + elif value_type == "default": + ts_col_value = value + return ts_col_value + + def genTagValue(self, t0_type="bool", t0_value="", t1_type="tinyint", t1_value=127, t2_type="smallint", t2_value=32767, + t3_type="int", t3_value=2147483647, t4_type="bigint", t4_value=9223372036854775807, + t5_type="float", t5_value=11.12345027923584, t6_type="double", t6_value=22.123456789, + t7_type="binary", t7_value="binaryTagValue", t8_type="nchar", t8_value="ncharTagValue", value_type="obj"): + if t0_value == "": + t0_value = random.choice([True, False]) + if value_type == "obj": + tag_value = { + "t0": {"value": t0_value, "type": t0_type}, + "t1": {"value": t1_value, "type": t1_type}, + "t2": {"value": t2_value, "type": t2_type}, + "t3": {"value": t3_value, "type": t3_type}, + "t4": {"value": t4_value, "type": t4_type}, + "t5": {"value": t5_value, "type": t5_type}, + "t6": {"value": t6_value, "type": t6_type}, + "t7": {"value": t7_value, "type": t7_type}, + "t8": {"value": t8_value, "type": t8_type} + } + elif value_type == "default": + # t5_value = t6_value + tag_value = { + "t0": t0_value, + "t1": t1_value, + "t2": t2_value, + "t3": t3_value, + "t4": t4_value, + "t5": t5_value, + "t6": t6_value, + "t7": t7_value, + "t8": t8_value + } + return tag_value + + def genFullTypeJson(self, ts_value="", col_value="", tag_value="", stb_name="", tb_name="", + id_noexist_tag=None, id_change_tag=None, id_upper_tag=None, id_mixul_tag=None, id_double_tag=None, + t_add_tag=None, t_mul_tag=None, c_multi_tag=None, c_blank_tag=None, t_blank_tag=None, + chinese_tag=None, multi_field_tag=None, point_trans_tag=None, value_type="obj"): + if value_type == "obj": + if stb_name == "": + stb_name = tdCom.getLongName(6, "letters") + if tb_name == "": + tb_name = f'{stb_name}_{random.randint(0, 65535)}_{random.randint(0, 65535)}' + if ts_value == "": + ts_value = self.genTsColValue(1626006833639000000, "ns") + if col_value == "": + col_value = self.genTsColValue(random.choice([True, False]), "bool") + if tag_value == "": + tag_value = self.genTagValue() + # if id_upper_tag is not None: + # id = "ID" + # else: + # id = "id" + # if id_mixul_tag is not None: + # id = random.choice(["iD", "Id"]) + # else: + # id = "id" + # if id_noexist_tag is None: + # tag_value[id] = tb_name + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if id_noexist_tag is not None: + if t_add_tag is not None: + tag_value["t9"] = {"value": "ncharTagValue", "type": "nchar"} + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if id_change_tag is not None: + tag_value.pop('t8') + tag_value["t8"] = {"value": "ncharTagValue", "type": "nchar"} + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if id_double_tag is not None: + tag_value["ID"] = f'"{tb_name}_2"' + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if t_add_tag is not None: + tag_value["t10"] = {"value": "ncharTagValue", "type": "nchar"} + tag_value["t11"] = {"value": True, "type": "bool"} + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if t_mul_tag is not None: + tag_value.pop('t8') + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if c_multi_tag is not None: + col_value = [{"value": True, "type": "bool"}, {"value": False, "type": "bool"}] + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if t_blank_tag is not None: + tag_value = "" + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if chinese_tag is not None: + tag_value = {"t0": {"value": "涛思数据", "type": "nchar"}} + col_value = {"value": "涛思数据", "type": "nchar"} + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if c_blank_tag is not None: + sql_json.pop("value") + if multi_field_tag is not None: + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value, "tags2": tag_value} + if point_trans_tag is not None: + sql_json = {"metric": ".point.trans.test", "timestamp": ts_value, "value": col_value, "tags": tag_value} + + elif value_type == "default": + if stb_name == "": + stb_name = tdCom.getLongName(6, "letters") + if tb_name == "": + tb_name = f'{stb_name}_{random.randint(0, 65535)}_{random.randint(0, 65535)}' + if ts_value == "": + ts_value = 1626006834 + if col_value == "": + col_value = random.choice([True, False]) + if tag_value == "": + tag_value = self.genTagValue(value_type=value_type) + # if id_upper_tag is not None: + # id = "ID" + # else: + # id = "id" + # if id_mixul_tag is not None: + # id = "iD" + # else: + # id = "id" + # if id_noexist_tag is None: + # tag_value[id] = tb_name + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if id_noexist_tag is not None: + if t_add_tag is not None: + tag_value["t9"] = {"value": "ncharTagValue", "type": "nchar"} + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if id_change_tag is not None: + tag_value.pop('t7') + tag_value["t7"] = {"value": "ncharTagValue", "type": "nchar"} + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if id_double_tag is not None: + tag_value["ID"] = f'"{tb_name}_2"' + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if t_add_tag is not None: + tag_value["t10"] = {"value": "ncharTagValue", "type": "nchar"} + tag_value["t11"] = True + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if t_mul_tag is not None: + tag_value.pop('t7') + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if c_multi_tag is not None: + col_value = True,False + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if t_blank_tag is not None: + tag_value = "" + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value} + if c_blank_tag is not None: + sql_json.pop("value") + if multi_field_tag is not None: + sql_json = {"metric": stb_name, "timestamp": ts_value, "value": col_value, "tags": tag_value, "tags2": tag_value} + if point_trans_tag is not None: + sql_json = {"metric": ".point.trans.test", "timestamp": ts_value, "value": col_value, "tags": tag_value} + return sql_json, stb_name + + def genMulTagColDict(self, genType, count=1, value_type="obj"): + """ + genType must be tag/col + """ + tag_dict = dict() + col_dict = dict() + if value_type == "obj": + if genType == "tag": + for i in range(0, count): + tag_dict[f't{i}'] = {'value': True, 'type': 'bool'} + return tag_dict + if genType == "col": + col_dict = {'value': True, 'type': 'bool'} + return col_dict + elif value_type == "default": + if genType == "tag": + for i in range(0, count): + tag_dict[f't{i}'] = True + return tag_dict + if genType == "col": + col_dict = True + return col_dict + + def genLongJson(self, tag_count, value_type="obj"): + stb_name = tdCom.getLongName(7, mode="letters") + # tb_name = f'{stb_name}_1' + tag_dict = self.genMulTagColDict("tag", tag_count, value_type) + col_dict = self.genMulTagColDict("col", 1, value_type) + # tag_dict["id"] = tb_name + ts_dict = {'value': 1626006833639000000, 'type': 'ns'} + long_json = {"metric": stb_name, "timestamp": ts_dict, "value": col_dict, "tags": tag_dict} + return long_json, stb_name + + def getNoIdTbName(self, stb_name): + query_sql = f"select tbname from {stb_name}" + tb_name = self.resHandle(query_sql, True)[0][0] + return tb_name + + def resHandle(self, query_sql, query_tag): + tdSql.execute('reset query cache') + row_info = tdSql.query(query_sql, query_tag) + col_info = tdSql.getColNameList(query_sql, query_tag) + res_row_list = [] + sub_list = [] + for row_mem in row_info: + for i in row_mem: + if "11.1234" in str(i) and str(i) != "11.12345f32" and str(i) != "11.12345027923584F32": + sub_list.append("11.12345027923584") + elif "22.1234" in str(i) and str(i) != "22.123456789f64" and str(i) != "22.123456789F64": + sub_list.append("22.123456789") + else: + sub_list.append(str(i)) + res_row_list.append(sub_list) + res_field_list_without_ts = col_info[0][1:] + res_type_list = col_info[1] + return res_row_list, res_field_list_without_ts, res_type_list + + def resCmp(self, input_json, stb_name, query_sql="select * from", condition="", ts=None, id=True, none_check_tag=None, none_type_check=None): + expect_list = self.inputHandle(input_json) + print("----", json.dumps(input_json)) + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + print("!!!!!----", json.dumps(input_json)) + query_sql = f"{query_sql} {stb_name} {condition}" + res_row_list, res_field_list_without_ts, res_type_list = self.resHandle(query_sql, True) + if ts == 0: + res_ts = self.dateToTs(res_row_list[0][0]) + current_time = time.time() + if current_time - res_ts < 60: + tdSql.checkEqual(res_row_list[0][1:], expect_list[0][1:]) + else: + print("timeout") + tdSql.checkEqual(res_row_list[0], expect_list[0]) + else: + if none_check_tag is not None: + none_index_list = [i for i,x in enumerate(res_row_list[0]) if x=="None"] + none_index_list.reverse() + for j in none_index_list: + res_row_list[0].pop(j) + expect_list[0].pop(j) + tdSql.checkEqual(sorted(res_row_list[0]), sorted(expect_list[0])) + tdSql.checkEqual(sorted(res_field_list_without_ts), sorted(expect_list[1])) + tdSql.checkEqual(res_type_list, expect_list[2]) + + def initCheckCase(self, value_type="obj"): + """ + normal tags and cols, one for every elm + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(value_type=value_type) + self.resCmp(input_json, stb_name) + + def boolTypeCheckCase(self): + """ + check all normal type + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + full_type_list = ["f", "F", "false", "False", "t", "T", "true", "True"] + for t_type in full_type_list: + input_json_list = [self.genFullTypeJson(tag_value=self.genTagValue(t0_value=t_type))[0], + self.genFullTypeJson(col_value=self.genTsColValue(value=t_type, t_type="bool"))[0]] + for input_json in input_json_list: + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def symbolsCheckCase(self, value_type="obj"): + """ + check symbols = `~!@#$%^&*()_-+={[}]\|:;'\",<.>/? + """ + ''' + please test : + binary_symbols = '\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal"\'\'"\"' + ''' + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + binary_symbols = '"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal"' + nchar_symbols = binary_symbols + input_sql1, stb_name1 = self.genFullTypeJson(col_value=self.genTsColValue(value=binary_symbols, t_type="binary", value_type=value_type), + tag_value=self.genTagValue(t7_value=binary_symbols, t8_value=nchar_symbols, value_type=value_type)) + input_sql2, stb_name2 = self.genFullTypeJson(col_value=self.genTsColValue(value=nchar_symbols, t_type="nchar", value_type=value_type), + tag_value=self.genTagValue(t7_value=binary_symbols, t8_value=nchar_symbols, value_type=value_type)) + self.resCmp(input_sql1, stb_name1) + self.resCmp(input_sql2, stb_name2) + + def tsCheckCase(self, value_type="obj"): + """ + test ts list --> ["1626006833639000000ns", "1626006833639019us", "1626006833640ms", "1626006834s", "1626006822639022"] + # ! us级时间戳都为0时，数据库中查询显示，但python接口拿到的结果不显示 .000000的情况请确认，目前修改时间处理代码可以通过 + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + ts_list = ["1626006833639000000ns", "1626006833639019us", "1626006833640ms", "1626006834s", "1626006834", 0] + for ts in ts_list: + if "s" in str(ts): + input_json, stb_name = self.genFullTypeJson(ts_value=self.genTsColValue(value=int(tdCom.splitNumLetter(ts)[0]), t_type=tdCom.splitNumLetter(ts)[1])) + self.resCmp(input_json, stb_name, ts=ts) + else: + input_json, stb_name = self.genFullTypeJson(ts_value=self.genTsColValue(value=int(ts), t_type="s", value_type=value_type)) + self.resCmp(input_json, stb_name, ts=ts) + if int(ts) == 0: + if value_type == "obj": + input_json_list = [self.genFullTypeJson(ts_value=self.genTsColValue(value=int(ts), t_type="")), + self.genFullTypeJson(ts_value=self.genTsColValue(value=int(ts), t_type="ns")), + self.genFullTypeJson(ts_value=self.genTsColValue(value=int(ts), t_type="us")), + self.genFullTypeJson(ts_value=self.genTsColValue(value=int(ts), t_type="ms")), + self.genFullTypeJson(ts_value=self.genTsColValue(value=int(ts), t_type="s"))] + elif value_type == "default": + input_json_list = [self.genFullTypeJson(ts_value=self.genTsColValue(value=int(ts), value_type=value_type))] + for input_json in input_json_list: + self.resCmp(input_json[0], input_json[1], ts=ts) + else: + input_json = self.genFullTypeJson(ts_value=self.genTsColValue(value=int(ts), t_type=""))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + # check result + #! bug + tdSql.execute(f"drop database if exists test_ts") + tdSql.execute(f"create database if not exists test_ts precision 'ms'") + tdSql.execute("use test_ts") + input_json = [{"metric": "test_ms", "timestamp": {"value": 1626006833640, "type": "ms"}, "value": True, "tags": {"t0": True}}, + {"metric": "test_ms", "timestamp": {"value": 1626006833641, "type": "ms"}, "value": False, "tags": {"t0": True}}] + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + res = tdSql.query('select * from test_ms', True) + tdSql.checkEqual(str(res[0][0]), "2021-07-11 20:33:53.640000") + tdSql.checkEqual(str(res[1][0]), "2021-07-11 20:33:53.641000") + + tdSql.execute(f"drop database if exists test_ts") + tdSql.execute(f"create database if not exists test_ts precision 'us'") + tdSql.execute("use test_ts") + input_json = [{"metric": "test_us", "timestamp": {"value": 1626006833639000, "type": "us"}, "value": True, "tags": {"t0": True}}, + {"metric": "test_us", "timestamp": {"value": 1626006833639001, "type": "us"}, "value": False, "tags": {"t0": True}}] + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + res = tdSql.query('select * from test_us', True) + tdSql.checkEqual(str(res[0][0]), "2021-07-11 20:33:53.639000") + tdSql.checkEqual(str(res[1][0]), "2021-07-11 20:33:53.639001") + + tdSql.execute(f"drop database if exists test_ts") + tdSql.execute(f"create database if not exists test_ts precision 'ns'") + tdSql.execute("use test_ts") + input_json = [{"metric": "test_ns", "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": True, "tags": {"t0": True}}, + {"metric": "test_ns", "timestamp": {"value": 1626006833639000001, "type": "ns"}, "value": False, "tags": {"t0": True}}] + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + res = tdSql.query('select * from test_ns', True) + tdSql.checkEqual(str(res[0][0]), "1626006833639000000") + tdSql.checkEqual(str(res[1][0]), "1626006833639000001") + self.createDb() + + def idSeqCheckCase(self, value_type="obj"): + """ + check id.index in tags + eg: t0=**,id=**,t1=** + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(id_change_tag=True, value_type=value_type) + self.resCmp(input_json, stb_name) + + def idLetterCheckCase(self, value_type="obj"): + """ + check id param + eg: id and ID + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(id_upper_tag=True, value_type=value_type) + self.resCmp(input_json, stb_name) + input_json, stb_name = self.genFullTypeJson(id_mixul_tag=True, value_type=value_type) + self.resCmp(input_json, stb_name) + input_json, stb_name = self.genFullTypeJson(id_change_tag=True, id_upper_tag=True, value_type=value_type) + self.resCmp(input_json, stb_name) + + def noIdCheckCase(self, value_type="obj"): + """ + id not exist + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(id_noexist_tag=True, value_type=value_type) + self.resCmp(input_json, stb_name) + query_sql = f"select tbname from {stb_name}" + res_row_list = self.resHandle(query_sql, True)[0] + if len(res_row_list[0][0]) > 0: + tdSql.checkColNameList(res_row_list, res_row_list) + else: + tdSql.checkColNameList(res_row_list, "please check noIdCheckCase") + + def maxColTagCheckCase(self, value_type="obj"): + """ + max tag count is 128 + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + for input_json in [self.genLongJson(128, value_type)[0]]: + tdCom.cleanTb() + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + for input_json in [self.genLongJson(129, value_type)[0]]: + tdCom.cleanTb() + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def idIllegalNameCheckCase(self, value_type="obj"): + """ + test illegal id name + mix "`~!@#$¥%^&*()-+={}|[]、「」【】\:;《》<>?" + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + rstr = list("`~!@#$¥%^&*()-+={}|[]、「」【】\:;《》<>?") + for i in rstr: + input_json = self.genFullTypeJson(tb_name=f'aa{i}bb', value_type=value_type)[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def idStartWithNumCheckCase(self, value_type="obj"): + """ + id is start with num + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(tb_name="1aaabbb", value_type=value_type)[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def nowTsCheckCase(self, value_type="obj"): + """ + check now unsupported + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(ts_value=self.genTsColValue(value="now", t_type="ns", value_type=value_type))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def dateFormatTsCheckCase(self, value_type="obj"): + """ + check date format ts unsupported + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(ts_value=self.genTsColValue(value="2021-07-21\ 19:01:46.920", t_type="ns", value_type=value_type))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def illegalTsCheckCase(self, value_type="obj"): + """ + check ts format like 16260068336390us19 + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(ts_value=self.genTsColValue(value="16260068336390us19", t_type="us", value_type=value_type))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def tbnameCheckCase(self, value_type="obj"): + """ + check length 192 + check upper tbname + chech upper tag + length of stb_name tb_name <= 192 + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tdSql.execute('reset query cache') + stb_name_192 = tdCom.getLongName(len=192, mode="letters") + tb_name_192 = tdCom.getLongName(len=192, mode="letters") + input_json, stb_name = self.genFullTypeJson(stb_name=stb_name_192, tb_name=tb_name_192, value_type=value_type) + self.resCmp(input_json, stb_name) + tdSql.query(f'select * from {stb_name}') + tdSql.checkRows(1) + for input_json in [self.genFullTypeJson(stb_name=tdCom.getLongName(len=193, mode="letters"), tb_name=tdCom.getLongName(len=5, mode="letters"), value_type=value_type)[0]]: + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + stbname = tdCom.getLongName(len=10, mode="letters") + input_json = {'metric': f'A{stbname}', 'timestamp': {'value': 1626006833639000000, 'type': 'ns'}, 'value': {'value': False, 'type': 'bool'}, 'tags': {'t1': {'value': 127, 'type': 'tinyint'}, "t2": 127}} + stb_name = f'`A{stbname}`' + self.resCmp(input_json, stb_name) + tdSql.execute(f"drop table {stb_name}") + + def tagNameLengthCheckCase(self): + """ + check tag name limit <= 62 + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tag_name = tdCom.getLongName(61, "letters") + tag_name = f't{tag_name}' + stb_name = tdCom.getLongName(7, "letters") + input_json = {'metric': stb_name, 'timestamp': {'value': 1626006833639000000, 'type': 'ns'}, 'value': "bcdaaa", 'tags': {tag_name: {'value': False, 'type': 'bool'}}} + self.resCmp(input_json, stb_name) + input_json = {'metric': stb_name, 'timestamp': {'value': 1626006833639000001, 'type': 'ns'}, 'value': "bcdaaaa", 'tags': {tdCom.getLongName(65, "letters"): {'value': False, 'type': 'bool'}}} + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def tagValueLengthCheckCase(self, value_type="obj"): + """ + check full type tag value limit + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + # i8 + for t1 in [-127, 127]: + input_json, stb_name = self.genFullTypeJson(tag_value=self.genTagValue(t1_value=t1, value_type=value_type)) + self.resCmp(input_json, stb_name) + for t1 in [-128, 128]: + input_json = self.genFullTypeJson(tag_value=self.genTagValue(t1_value=t1))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + #i16 + for t2 in [-32767, 32767]: + input_json, stb_name = self.genFullTypeJson(tag_value=self.genTagValue(t2_value=t2, value_type=value_type)) + self.resCmp(input_json, stb_name) + for t2 in [-32768, 32768]: + input_json = self.genFullTypeJson(tag_value=self.genTagValue(t2_value=t2))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + #i32 + for t3 in [-2147483647, 2147483647]: + input_json, stb_name = self.genFullTypeJson(tag_value=self.genTagValue(t3_value=t3, value_type=value_type)) + self.resCmp(input_json, stb_name) + for t3 in [-2147483648, 2147483648]: + input_json = self.genFullTypeJson(tag_value=self.genTagValue(t3_value=t3))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + #i64 + for t4 in [-9223372036854775807, 9223372036854775807]: + input_json, stb_name = self.genFullTypeJson(tag_value=self.genTagValue(t4_value=t4, value_type=value_type)) + self.resCmp(input_json, stb_name) + + for t4 in [-9223372036854775808, 9223372036854775808]: + input_json = self.genFullTypeJson(tag_value=self.genTagValue(t4_value=t4))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # f32 + for t5 in [-3.4028234663852885981170418348451692544*(10**38), 3.4028234663852885981170418348451692544*(10**38)]: + input_json, stb_name = self.genFullTypeJson(tag_value=self.genTagValue(t5_value=t5, value_type=value_type)) + self.resCmp(input_json, stb_name) + # * limit set to 3.4028234664*(10**38) + for t5 in [-3.4028234664*(10**38), 3.4028234664*(10**38)]: + input_json = self.genFullTypeJson(tag_value=self.genTagValue(t5_value=t5))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # f64 + for t6 in [-1.79769*(10**308), -1.79769*(10**308)]: + input_json, stb_name = self.genFullTypeJson(tag_value=self.genTagValue(t6_value=t6, value_type=value_type)) + self.resCmp(input_json, stb_name) + for t6 in [float(-1.797693134862316*(10**308)), -1.797693134862316*(10**308)]: + input_json = self.genFullTypeJson(tag_value=self.genTagValue(t6_value=t6, value_type=value_type))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + if value_type == "obj": + # binary + stb_name = tdCom.getLongName(7, "letters") + input_json = {"metric": stb_name, "timestamp": {'value': 1626006833639000000, 'type': 'ns'}, "value": {'value': True, 'type': 'bool'}, "tags": {"t0": {'value': True, 'type': 'bool'}, "t1":{'value': tdCom.getLongName(16374, "letters"), 'type': 'binary'}}} + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + input_json = {"metric": stb_name, "timestamp": {'value': 1626006833639000000, 'type': 'ns'}, "value": {'value': True, 'type': 'bool'}, "tags": {"t0": {'value': True, 'type': 'bool'}, "t1":{'value': tdCom.getLongName(16375, "letters"), 'type': 'binary'}}} + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # # nchar + # # * legal nchar could not be larger than 16374/4 + stb_name = tdCom.getLongName(7, "letters") + input_json = {"metric": stb_name, "timestamp": {'value': 1626006833639000000, 'type': 'ns'}, "value": {'value': True, 'type': 'bool'}, "tags": {"t0": {'value': True, 'type': 'bool'}, "t1":{'value': tdCom.getLongName(4093, "letters"), 'type': 'nchar'}}} + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + + input_json = {"metric": stb_name, "timestamp": {'value': 1626006833639000000, 'type': 'ns'}, "value": {'value': True, 'type': 'bool'}, "tags": {"t0": {'value': True, 'type': 'bool'}, "t1":{'value': tdCom.getLongName(4094, "letters"), 'type': 'nchar'}}} + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + elif value_type == "default": + stb_name = tdCom.getLongName(7, "letters") + if tdSql.getVariable("defaultJSONStrType")[0].lower() == "binary": + input_json = {"metric": stb_name, "timestamp": 1626006834, "value": True, "tags": {"t0": {'value': True, 'type': 'bool'}, "t1": tdCom.getLongName(16374, "letters")}} + elif tdSql.getVariable("defaultJSONStrType")[0].lower() == "nchar": + input_json = {"metric": stb_name, "timestamp": 1626006834, "value": True, "tags": {"t0": {'value': True, 'type': 'bool'}, "t1": tdCom.getLongName(4093, "letters")}} + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + if tdSql.getVariable("defaultJSONStrType")[0].lower() == "binary": + input_json = {"metric": stb_name, "timestamp": 1626006834, "value": True, "tags": {"t0": {'value': True, 'type': 'bool'}, "t1": tdCom.getLongName(16375, "letters")}} + elif tdSql.getVariable("defaultJSONStrType")[0].lower() == "nchar": + input_json = {"metric": stb_name, "timestamp": 1626006834, "value": True, "tags": {"t0": {'value': True, 'type': 'bool'}, "t1": tdCom.getLongName(4094, "letters")}} + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def colValueLengthCheckCase(self, value_type="obj"): + """ + check full type col value limit + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + # i8 + for value in [-128, 127]: + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="tinyint", value_type=value_type)) + self.resCmp(input_json, stb_name) + tdCom.cleanTb() + for value in [-129, 128]: + input_json = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="tinyint"))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + # i16 + tdCom.cleanTb() + for value in [-32768]: + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="smallint", value_type=value_type)) + self.resCmp(input_json, stb_name) + tdCom.cleanTb() + for value in [-32769, 32768]: + input_json = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="smallint"))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # i32 + tdCom.cleanTb() + for value in [-2147483648]: + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="int", value_type=value_type)) + self.resCmp(input_json, stb_name) + tdCom.cleanTb() + for value in [-2147483649, 2147483648]: + input_json = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="int"))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # i64 + tdCom.cleanTb() + for value in [-9223372036854775808]: + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="bigint", value_type=value_type)) + self.resCmp(input_json, stb_name) + # ! bug + # tdCom.cleanTb() + # for value in [-9223372036854775809, 9223372036854775808]: + # print(value) + # input_json = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="bigint"))[0] + # print(json.dumps(input_json)) + # try: + # self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + # raise Exception("should not reach here") + # except SchemalessError as err: + # tdSql.checkNotEqual(err.errno, 0) + + # f32 + tdCom.cleanTb() + for value in [-3.4028234663852885981170418348451692544*(10**38), 3.4028234663852885981170418348451692544*(10**38)]: + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="float", value_type=value_type)) + self.resCmp(input_json, stb_name) + # * limit set to 4028234664*(10**38) + tdCom.cleanTb() + for value in [-3.4028234664*(10**38), 3.4028234664*(10**38)]: + input_json = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="float"))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # f64 + tdCom.cleanTb() + for value in [-1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308), -1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308)]: + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="double", value_type=value_type)) + self.resCmp(input_json, stb_name) + # * limit set to 1.797693134862316*(10**308) + tdCom.cleanTb() + for value in [-1.797693134862316*(10**308), -1.797693134862316*(10**308)]: + input_json = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="double", value_type=value_type))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # if value_type == "obj": + # # binary + # tdCom.cleanTb() + # stb_name = tdCom.getLongName(7, "letters") + # input_json = {"metric": stb_name, "timestamp": {'value': 1626006833639000000, 'type': 'ns'}, "value": {'value': tdCom.getLongName(16374, "letters"), 'type': 'binary'}, "tags": {"t0": {'value': True, 'type': 'bool'}}} + # self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + + # tdCom.cleanTb() + # input_json = {"metric": stb_name, "timestamp": {'value': 1626006833639000000, 'type': 'ns'}, "value": {'value': tdCom.getLongName(16375, "letters"), 'type': 'binary'}, "tags": {"t0": {'value': True, 'type': 'bool'}}} + # try: + # self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + # raise Exception("should not reach here") + # except SchemalessError as err: + # tdSql.checkNotEqual(err.errno, 0) + + # # nchar + # # * legal nchar could not be larger than 16374/4 + # tdCom.cleanTb() + # stb_name = tdCom.getLongName(7, "letters") + # input_json = {"metric": stb_name, "timestamp": {'value': 1626006833639000000, 'type': 'ns'}, "value": {'value': tdCom.getLongName(4093, "letters"), 'type': 'nchar'}, "tags": {"t0": {'value': True, 'type': 'bool'}}} + # self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + + # tdCom.cleanTb() + # input_json = {"metric": stb_name, "timestamp": {'value': 1626006833639000000, 'type': 'ns'}, "value": {'value': tdCom.getLongName(4094, "letters"), 'type': 'nchar'}, "tags": {"t0": {'value': True, 'type': 'bool'}}} + # try: + # self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + # raise Exception("should not reach here") + # except SchemalessError as err: + # tdSql.checkNotEqual(err.errno, 0) + # elif value_type == "default": + # # binary + # tdCom.cleanTb() + # stb_name = tdCom.getLongName(7, "letters") + # if tdSql.getVariable("defaultJSONStrType")[0].lower() == "binary": + # input_json = {"metric": stb_name, "timestamp": 1626006834, "value": tdCom.getLongName(16374, "letters"), "tags": {"t0": {'value': True, 'type': 'bool'}}} + # elif tdSql.getVariable("defaultJSONStrType")[0].lower() == "nchar": + # input_json = {"metric": stb_name, "timestamp": 1626006834, "value": tdCom.getLongName(4093, "letters"), "tags": {"t0": {'value': True, 'type': 'bool'}}} + # self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + # tdCom.cleanTb() + # if tdSql.getVariable("defaultJSONStrType")[0].lower() == "binary": + # input_json = {"metric": stb_name, "timestamp": 1626006834, "value": tdCom.getLongName(16375, "letters"), "tags": {"t0": {'value': True, 'type': 'bool'}}} + # elif tdSql.getVariable("defaultJSONStrType")[0].lower() == "nchar": + # input_json = {"metric": stb_name, "timestamp": 1626006834, "value": tdCom.getLongName(4094, "letters"), "tags": {"t0": {'value': True, 'type': 'bool'}}} + # try: + # self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + # raise Exception("should not reach here") + # except SchemalessError as err: + # tdSql.checkNotEqual(err.errno, 0) + + def tagColIllegalValueCheckCase(self, value_type="obj"): + + """ + test illegal tag col value + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + # bool + for i in ["TrUe", "tRue", "trUe", "truE", "FalsE", "fAlse", "faLse", "falSe", "falsE"]: + try: + input_json1 = self.genFullTypeJson(tag_value=self.genTagValue(t0_value=i))[0] + self._conn.schemaless_insert([json.dumps(input_json1)], 2, None) + input_json2 = self.genFullTypeJson(col_value=self.genTsColValue(value=i, t_type="bool"))[0] + self._conn.schemaless_insert([json.dumps(input_json2)], 2, None) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # i8 i16 i32 i64 f32 f64 + for input_json in [ + self.genFullTypeJson(tag_value=self.genTagValue(t1_value="1s2"))[0], + self.genFullTypeJson(tag_value=self.genTagValue(t2_value="1s2"))[0], + self.genFullTypeJson(tag_value=self.genTagValue(t3_value="1s2"))[0], + self.genFullTypeJson(tag_value=self.genTagValue(t4_value="1s2"))[0], + self.genFullTypeJson(tag_value=self.genTagValue(t5_value="11.1s45"))[0], + self.genFullTypeJson(tag_value=self.genTagValue(t6_value="11.1s45"))[0], + ]: + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # check binary and nchar blank + input_sql1 = self.genFullTypeJson(col_value=self.genTsColValue(value="abc aaa", t_type="binary", value_type=value_type))[0] + input_sql2 = self.genFullTypeJson(col_value=self.genTsColValue(value="abc aaa", t_type="nchar", value_type=value_type))[0] + input_sql3 = self.genFullTypeJson(tag_value=self.genTagValue(t7_value="abc aaa", value_type=value_type))[0] + input_sql4 = self.genFullTypeJson(tag_value=self.genTagValue(t8_value="abc aaa", value_type=value_type))[0] + for input_json in [input_sql1, input_sql2, input_sql3, input_sql4]: + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + # check accepted binary and nchar symbols + # # * ~!@#$¥%^&*()-+={}|[]、「」:; + for symbol in list('~!@#$¥%^&*()-+={}|[]、「」:;'): + input_json1 = self.genFullTypeJson(col_value=self.genTsColValue(value=f"abc{symbol}aaa", t_type="binary", value_type=value_type))[0] + input_json2 = self.genFullTypeJson(tag_value=self.genTagValue(t8_value=f"abc{symbol}aaa", value_type=value_type))[0] + self._conn.schemaless_insert([json.dumps(input_json1)], TDSmlProtocolType.JSON.value, None) + self._conn.schemaless_insert([json.dumps(input_json2)], TDSmlProtocolType.JSON.value, None) + + def duplicateIdTagColInsertCheckCase(self, value_type="obj"): + """ + check duplicate Id Tag Col + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(id_double_tag=True, value_type=value_type)[0] + print(input_json) + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + input_json = self.genFullTypeJson(tag_value=self.genTagValue(t5_value=11.12345027923584, t6_type="float", t6_value=22.12345027923584, value_type=value_type))[0] + try: + self._conn.schemaless_insert([json.dumps(input_json).replace("t6", "t5")], 2, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + ##### stb exist ##### + def noIdStbExistCheckCase(self, value_type="obj"): + """ + case no id when stb exist + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(tb_name="sub_table_0123456", col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type)) + self.resCmp(input_json, stb_name) + input_json, stb_name = self.genFullTypeJson(stb_name=stb_name, id_noexist_tag=True, col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type)) + self.resCmp(input_json, stb_name, condition='where tbname like "t_%"') + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(1) + + def duplicateInsertExistCheckCase(self, value_type="obj"): + """ + check duplicate insert when stb exist + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(value_type=value_type) + self.resCmp(input_json, stb_name) + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + self.resCmp(input_json, stb_name) + + def tagColBinaryNcharLengthCheckCase(self, value_type="obj"): + """ + check length increase + """ + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(value_type=value_type) + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + self.resCmp(input_json, stb_name) + tb_name = tdCom.getLongName(5, "letters") + input_json, stb_name = self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name, tag_value=self.genTagValue(t7_value="binaryTagValuebinaryTagValue", t8_value="ncharTagValuencharTagValue", value_type=value_type)) + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + self.resCmp(input_json, stb_name, condition=f'where tbname like "{tb_name}"') + + def lengthIcreaseCrashCheckCase(self): + """ + check length increase + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + stb_name = "test_crash" + input_json = self.genFullTypeJson(stb_name=stb_name)[0] + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + os.system('python3 query/schemalessQueryCrash.py &') + time.sleep(2) + tb_name = tdCom.getLongName(5, "letters") + input_json, stb_name = self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name, tag_value=self.genTagValue(t7_value="binaryTagValuebinaryTagValue", t8_value="ncharTagValuencharTagValue")) + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + time.sleep(3) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + + def tagColAddDupIDCheckCase(self, value_type="obj"): + """ + check tag count add, stb and tb duplicate + * tag: alter table ... + * col: when update==0 and ts is same, unchange + * so this case tag&&value will be added, + * col is added without value when update==0 + * col is added with value when update==1 + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + for db_update_tag in [0, 1]: + if db_update_tag == 1 : + self.createDb("test_update", db_update_tag=db_update_tag) + input_json, stb_name = self.genFullTypeJson(tb_name=tb_name, col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type)) + self.resCmp(input_json, stb_name) + input_json, stb_name = self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name, col_value=self.genTsColValue(value=False, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type), t_add_tag=True) + if db_update_tag == 1 : + self.resCmp(input_json, stb_name, condition=f'where tbname like "{tb_name}"', none_check_tag=True) + tdSql.query(f'select * from {stb_name} where tbname like "{tb_name}"') + tdSql.checkData(0, 11, None) + tdSql.checkData(0, 12, None) + else: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + tdSql.query(f'select * from {stb_name} where tbname like "{tb_name}"') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 11, None) + tdSql.checkData(0, 12, None) + self.createDb() + + def tagAddCheckCase(self, value_type="obj"): + """ + check tag count add + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_json, stb_name = self.genFullTypeJson(tb_name=tb_name, col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type)) + self.resCmp(input_json, stb_name) + tb_name_1 = tdCom.getLongName(7, "letters") + input_json, stb_name = self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name_1, col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type), t_add_tag=True) + self.resCmp(input_json, stb_name, condition=f'where tbname like "{tb_name_1}"') + res_row_list = self.resHandle(f"select t10,t11 from {tb_name}", True)[0] + tdSql.checkEqual(res_row_list[0], ['None', 'None']) + self.resCmp(input_json, stb_name, condition=f'where tbname like "{tb_name}"', none_check_tag=True) + + def tagMd5Check(self, value_type="obj"): + """ + condition: stb not change + insert two table, keep tag unchange, change col + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type), id_noexist_tag=True) + self.resCmp(input_json, stb_name) + tb_name1 = self.getNoIdTbName(stb_name) + input_json, stb_name = self.genFullTypeJson(stb_name=stb_name, col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type), id_noexist_tag=True) + self.resCmp(input_json, stb_name) + tb_name2 = self.getNoIdTbName(stb_name) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(1) + tdSql.checkEqual(tb_name1, tb_name2) + input_json, stb_name = self.genFullTypeJson(stb_name=stb_name, col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type), id_noexist_tag=True, t_add_tag=True) + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + tb_name3 = self.getNoIdTbName(stb_name) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + tdSql.checkNotEqual(tb_name1, tb_name3) + + # * tag binary max is 16384, col+ts binary max 49151 + def tagColBinaryMaxLengthCheckCase(self, value_type="obj"): + """ + every binary and nchar must be length+2 + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + stb_name = tdCom.getLongName(7, "letters") + tb_name = f'{stb_name}_1' + tag_value = {"t0": {"value": True, "type": "bool"}} + tag_value["id"] = tb_name + col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type) + input_json = {"metric": stb_name, "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": col_value, "tags": tag_value} + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + + # * every binary and nchar must be length+2, so here is two tag, max length could not larger than 16384-2*2 + if value_type == "obj": + tag_value["t1"] = {"value": tdCom.getLongName(16374, "letters"), "type": "binary"} + tag_value["t2"] = {"value": tdCom.getLongName(5, "letters"), "type": "binary"} + elif value_type == "default": + if tdSql.getVariable("defaultJSONStrType")[0].lower() == "binary": + tag_value["t1"] = tdCom.getLongName(16374, "letters") + tag_value["t2"] = tdCom.getLongName(5, "letters") + elif tdSql.getVariable("defaultJSONStrType")[0].lower() == "nchar": + tag_value["t1"] = tdCom.getLongName(4093, "letters") + tag_value["t2"] = tdCom.getLongName(1, "letters") + tag_value.pop('id') + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + if value_type == "obj": + tag_value["t2"] = {"value": tdCom.getLongName(6, "letters"), "type": "binary"} + elif value_type == "default": + if tdSql.getVariable("defaultJSONStrType")[0].lower() == "binary": + tag_value["t2"] = tdCom.getLongName(6, "letters") + elif tdSql.getVariable("defaultJSONStrType")[0].lower() == "nchar": + tag_value["t2"] = tdCom.getLongName(2, "letters") + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + + # * tag nchar max is 16374/4, col+ts nchar max 49151 + def tagColNcharMaxLengthCheckCase(self, value_type="obj"): + """ + check nchar length limit + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + stb_name = tdCom.getLongName(7, "letters") + tb_name = f'{stb_name}_1' + tag_value = {"t0": True} + tag_value["id"] = tb_name + col_value= True + input_json = {"metric": stb_name, "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": col_value, "tags": tag_value} + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + + # * legal nchar could not be larger than 16374/4 + if value_type == "obj": + tag_value["t1"] = {"value": tdCom.getLongName(4093, "letters"), "type": "nchar"} + tag_value["t2"] = {"value": tdCom.getLongName(1, "letters"), "type": "nchar"} + elif value_type == "default": + if tdSql.getVariable("defaultJSONStrType")[0].lower() == "binary": + tag_value["t1"] = tdCom.getLongName(16374, "letters") + tag_value["t2"] = tdCom.getLongName(5, "letters") + elif tdSql.getVariable("defaultJSONStrType")[0].lower() == "nchar": + tag_value["t1"] = tdCom.getLongName(4093, "letters") + tag_value["t2"] = tdCom.getLongName(1, "letters") + tag_value.pop('id') + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + if value_type == "obj": + tag_value["t2"] = {"value": tdCom.getLongName(2, "letters"), "type": "binary"} + elif value_type == "default": + if tdSql.getVariable("defaultJSONStrType")[0].lower() == "binary": + tag_value["t2"] = tdCom.getLongName(6, "letters") + elif tdSql.getVariable("defaultJSONStrType")[0].lower() == "nchar": + tag_value["t2"] = tdCom.getLongName(2, "letters") + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + + def batchInsertCheckCase(self, value_type="obj"): + """ + test batch insert + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + stb_name = "stb_name" + tdSql.execute(f'create stable {stb_name}(ts timestamp, f int) tags(t1 bigint)') + input_json = [{"metric": "st123456", "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": {"value": 1, "type": "bigint"}, "tags": {"t1": {"value": 3, "type": "bigint"}, "t2": {"value": 4, "type": "double"}, "t3": {"value": "t3", "type": "binary"}}}, + {"metric": "st123456", "timestamp": {"value": 1626006833640000000, "type": "ns"}, "value": {"value": 2, "type": "bigint"}, "tags": {"t1": {"value": 4, "type": "bigint"}, "t3": {"value": "t4", "type": "binary"}, "t2": {"value": 5, "type": "double"}, "t4": {"value": 5, "type": "double"}}}, + {"metric": "stb_name", "timestamp": {"value": 1626056811823316532, "type": "ns"}, "value": {"value": 3, "type": "bigint"}, "tags": {"t2": {"value": 5, "type": "double"}, "t3": {"value": "ste", "type": "nchar"}}}, + {"metric": "stf567890", "timestamp": {"value": 1626006933640000000, "type": "ns"}, "value": {"value": 4, "type": "bigint"}, "tags": {"t1": {"value": 4, "type": "bigint"}, "t3": {"value": "t4", "type": "binary"}, "t2": {"value": 5, "type": "double"}, "t4": {"value": 5, "type": "double"}}}, + {"metric": "st123456", "timestamp": {"value": 1626006833642000000, "type": "ns"}, "value": {"value": 5, "type": "bigint"}, "tags": {"t1": {"value": 4, "type": "bigint"}, "t2": {"value": 5, "type": "double"}, "t3": {"value": "t4", "type": "binary"}}}, + {"metric": "stb_name", "timestamp": {"value": 1626056811843316532, "type": "ns"}, "value": {"value": 6, "type": "bigint"}, "tags": {"t2": {"value": 5, "type": "double"}, "t3": {"value": "ste2", "type": "nchar"}}}, + {"metric": "stb_name", "timestamp": {"value": 1626056812843316532, "type": "ns"}, "value": {"value": 7, "type": "bigint"}, "tags": {"t2": {"value": 5, "type": "double"}, "t3": {"value": "ste2", "type": "nchar"}}}, + {"metric": "st123456", "timestamp": {"value": 1626006933640000000, "type": "ns"}, "value": {"value": 8, "type": "bigint"}, "tags": {"t1": {"value": 4, "type": "bigint"}, "t3": {"value": "t4", "type": "binary"}, "t2": {"value": 5, "type": "double"}, "t4": {"value": 5, "type": "double"}}}, + {"metric": "st123456", "timestamp": {"value": 1626006933641000000, "type": "ns"}, "value": {"value": 9, "type": "bigint"}, "tags": {"t1": {"value": 4, "type": "bigint"}, "t3": {"value": "t4", "type": "binary"}, "t2": {"value": 5, "type": "double"}, "t4": {"value": 5, "type": "double"}}}] + if value_type != "obj": + input_json = [{"metric": "st123456", "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": 1, "tags": {"t1": 3, "t2": {"value": 4, "type": "double"}, "t3": {"value": "t3", "type": "binary"}}}, + {"metric": "st123456", "timestamp": {"value": 1626006833640000000, "type": "ns"}, "value": 2, "tags": {"t1": {"value": 4, "type": "double"}, "t3": {"value": "t4", "type": "binary"}, "t2": {"value": 5, "type": "double"}, "t4": {"value": 5, "type": "double"}}}, + {"metric": "stb_name", "timestamp": {"value": 1626056811823316532, "type": "ns"}, "value": 3, "tags": {"t2": {"value": 5, "type": "double"}, "t3": {"value": "ste", "type": "nchar"}}}, + {"metric": "stf567890", "timestamp": {"value": 1626006933640000000, "type": "ns"}, "value": 4, "tags": {"t1": {"value": 4, "type": "bigint"}, "t3": {"value": "t4", "type": "binary"}, "t2": {"value": 5, "type": "double"}, "t4": {"value": 5, "type": "double"}}}, + {"metric": "st123456", "timestamp": {"value": 1626006833642000000, "type": "ns"}, "value": {"value": 5, "type": "double"}, "tags": {"t1": {"value": 4, "type": "double"}, "t2": 5.0, "t3": {"value": "t4", "type": "binary"}}}, + {"metric": "stb_name", "timestamp": {"value": 1626056811843316532, "type": "ns"}, "value": {"value": 6, "type": "double"}, "tags": {"t2": 5.0, "t3": {"value": "ste2", "type": "nchar"}}}, + {"metric": "stb_name", "timestamp": {"value": 1626056812843316532, "type": "ns"}, "value": {"value": 7, "type": "double"}, "tags": {"t2": {"value": 5, "type": "double"}, "t3": {"value": "ste2", "type": "nchar"}}}, + {"metric": "st123456", "timestamp": {"value": 1626006933640000000, "type": "ns"}, "value": {"value": 8, "type": "double"}, "tags": {"t1": {"value": 4, "type": "double"}, "t3": {"value": "t4", "type": "binary"}, "t2": {"value": 5, "type": "double"}, "t4": {"value": 5, "type": "double"}}}, + {"metric": "st123456", "timestamp": {"value": 1626006933641000000, "type": "ns"}, "value": {"value": 9, "type": "double"}, "tags": {"t1": 4, "t3": {"value": "t4", "type": "binary"}, "t2": {"value": 5, "type": "double"}, "t4": {"value": 5, "type": "double"}}}] + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + tdSql.query('show stables') + tdSql.checkRows(3) + tdSql.query('show tables') + tdSql.checkRows(6) + tdSql.query('select * from st123456') + tdSql.checkRows(5) + + def multiInsertCheckCase(self, count, value_type="obj"): + """ + test multi insert + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + sql_list = list() + stb_name = tdCom.getLongName(8, "letters") + tdSql.execute(f'create stable {stb_name}(ts timestamp, f int) tags(t1 bigint)') + for i in range(count): + input_json = self.genFullTypeJson(stb_name=stb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), id_noexist_tag=True)[0] + sql_list.append(input_json) + self._conn.schemaless_insert([json.dumps(sql_list)], TDSmlProtocolType.JSON.value, None) + tdSql.query('show tables') + tdSql.checkRows(count) + + def batchErrorInsertCheckCase(self): + """ + test batch error insert + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = [{"metric": "st123456", "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": {"value": "tt", "type": "bool"}, "tags": {"t1": {"value": 3, "type": "bigint"}, "t2": {"value": 4, "type": "double"}, "t3": {"value": "t3", "type": "binary"}}}, + {"metric": "st123456", "timestamp": {"value": 1626006933641000000, "type": "ns"}, "value": {"value": 9, "type": "bigint"}, "tags": {"t1": {"value": 4, "type": "bigint"}, "t3": {"value": "t4", "type": "binary"}, "t2": {"value": 5, "type": "double"}, "t4": {"value": 5, "type": "double"}}}] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def multiColsInsertCheckCase(self, value_type="obj"): + """ + test multi cols insert + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(c_multi_tag=True, value_type=value_type)[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def blankColInsertCheckCase(self, value_type="obj"): + """ + test blank col insert + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(c_blank_tag=True, value_type=value_type)[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def blankTagInsertCheckCase(self, value_type="obj"): + """ + test blank tag insert + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(t_blank_tag=True, value_type=value_type)[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def chineseCheckCase(self): + """ + check nchar ---> chinese + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(chinese_tag=True) + self.resCmp(input_json, stb_name) + + def multiFieldCheckCase(self, value_type="obj"): + ''' + multi_field + ''' + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(multi_field_tag=True, value_type=value_type)[0] + try: + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + raise Exception("should not reach here") + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + def spellCheckCase(self): + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + stb_name = tdCom.getLongName(8, "letters") + input_json_list = [{"metric": f'{stb_name}_1', "timestamp": {"value": 1626006833639000000, "type": "Ns"}, "value": {"value": 1, "type": "Bigint"}, "tags": {"t1": {"value": 127, "type": "tinYint"}}}, + {"metric": f'{stb_name}_2', "timestamp": {"value": 1626006833639000001, "type": "nS"}, "value": {"value": 32767, "type": "smallInt"}, "tags": {"t1": {"value": 32767, "type": "smallInt"}}}, + {"metric": f'{stb_name}_3', "timestamp": {"value": 1626006833639000002, "type": "NS"}, "value": {"value": 2147483647, "type": "iNt"}, "tags": {"t1": {"value": 2147483647, "type": "iNt"}}}, + {"metric": f'{stb_name}_4', "timestamp": {"value": 1626006833639019, "type": "Us"}, "value": {"value": 9223372036854775807, "type": "bigInt"}, "tags": {"t1": {"value": 9223372036854775807, "type": "bigInt"}}}, + {"metric": f'{stb_name}_5', "timestamp": {"value": 1626006833639018, "type": "uS"}, "value": {"value": 11.12345027923584, "type": "flOat"}, "tags": {"t1": {"value": 11.12345027923584, "type": "flOat"}}}, + {"metric": f'{stb_name}_6', "timestamp": {"value": 1626006833639017, "type": "US"}, "value": {"value": 22.123456789, "type": "douBle"}, "tags": {"t1": {"value": 22.123456789, "type": "douBle"}}}, + {"metric": f'{stb_name}_7', "timestamp": {"value": 1626006833640, "type": "Ms"}, "value": {"value": "vozamcts", "type": "binaRy"}, "tags": {"t1": {"value": "vozamcts", "type": "binaRy"}}}, + {"metric": f'{stb_name}_8', "timestamp": {"value": 1626006833641, "type": "mS"}, "value": {"value": "vozamcts", "type": "nchAr"}, "tags": {"t1": {"value": "vozamcts", "type": "nchAr"}}}, + {"metric": f'{stb_name}_9', "timestamp": {"value": 1626006833642, "type": "MS"}, "value": {"value": "vozamcts", "type": "nchAr"}, "tags": {"t1": {"value": "vozamcts", "type": "nchAr"}}}, + {"metric": f'{stb_name}_10', "timestamp": {"value": 1626006834, "type": "S"}, "value": {"value": "vozamcts", "type": "nchAr"}, "tags": {"t1": {"value": "vozamcts", "type": "nchAr"}}}] + + for input_sql in input_json_list: + stb_name = input_sql["metric"] + self.resCmp(input_sql, stb_name) + + def tbnameTagsColsNameCheckCase(self): + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = {'metric': 'rFa$sta', 'timestamp': {'value': 1626006834, 'type': 's'}, 'value': {'value': True, 'type': 'bool'}, 'tags': {'Tt!0': {'value': False, 'type': 'bool'}, 'tT@1': {'value': 127, 'type': 'tinyint'}, 't@2': {'value': 32767, 'type': 'smallint'}, 't$3': {'value': 2147483647, 'type': 'int'}, 't%4': {'value': 9223372036854775807, 'type': 'bigint'}, 't^5': {'value': 11.12345027923584, 'type': 'float'}, 't&6': {'value': 22.123456789, 'type': 'double'}, 't*7': {'value': 'binaryTagValue', 'type': 'binary'}, 't!@#$%^&*()_+[];:<>?,9': {'value': 'ncharTagValue', 'type': 'nchar'}, 'id': 'rFas$ta_1'}} + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + query_sql = 'select * from `rFa$sta`' + query_res = tdSql.query(query_sql, True) + tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), True, 'rFas$ta_1', 'ncharTagValue', 2147483647, 9223372036854775807, 22.123456789, 'binaryTagValue', 32767, 11.12345027923584, False, 127)]) + col_tag_res = tdSql.getColNameList(query_sql) + tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'id', 't!@#$%^&*()_+[];:<>?,9', 't$3', 't%4', 't&6', 't*7', 't@2', 't^5', 'Tt!0', 'tT@1']) + tdSql.execute('drop table `rFa$sta`') + + def pointTransCheckCase(self, value_type="obj"): + """ + metric value "." trans to "_" + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genFullTypeJson(point_trans_tag=True, value_type=value_type)[0] + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + tdSql.execute("drop table `.point.trans.test`") + + def genSqlList(self, count=5, stb_name="", tb_name="", value_type="obj"): + """ + stb --> supertable + tb --> table + ts --> timestamp, same default + col --> column, same default + tag --> tag, same default + d --> different + s --> same + a --> add + m --> minus + """ + d_stb_d_tb_list = list() + s_stb_s_tb_list = list() + s_stb_s_tb_a_tag_list = list() + s_stb_s_tb_m_tag_list = list() + s_stb_d_tb_list = list() + s_stb_d_tb_m_tag_list = list() + s_stb_d_tb_a_tag_list = list() + s_stb_s_tb_d_ts_list = list() + s_stb_s_tb_d_ts_m_tag_list = list() + s_stb_s_tb_d_ts_a_tag_list = list() + s_stb_d_tb_d_ts_list = list() + s_stb_d_tb_d_ts_m_tag_list = list() + s_stb_d_tb_d_ts_a_tag_list = list() + for i in range(count): + d_stb_d_tb_list.append(self.genFullTypeJson(col_value=self.genTsColValue(value=True, t_type="bool", value_type=value_type), tag_value=self.genTagValue(t0_value=True, value_type=value_type))) + s_stb_s_tb_list.append(self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type))) + s_stb_s_tb_a_tag_list.append(self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), t_add_tag=True)) + s_stb_s_tb_m_tag_list.append(self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), t_mul_tag=True)) + s_stb_d_tb_list.append(self.genFullTypeJson(stb_name=stb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), id_noexist_tag=True)) + s_stb_d_tb_m_tag_list.append(self.genFullTypeJson(stb_name=stb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), id_noexist_tag=True, t_mul_tag=True)) + s_stb_d_tb_a_tag_list.append(self.genFullTypeJson(stb_name=stb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), id_noexist_tag=True, t_add_tag=True)) + s_stb_s_tb_d_ts_list.append(self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), ts_value = self.genTsColValue(1626006833639000000, "ns"))) + s_stb_s_tb_d_ts_m_tag_list.append(self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), ts_value = self.genTsColValue(1626006833639000000, "ns"), t_mul_tag=True)) + s_stb_s_tb_d_ts_a_tag_list.append(self.genFullTypeJson(stb_name=stb_name, tb_name=tb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), ts_value = self.genTsColValue(1626006833639000000, "ns"), t_add_tag=True)) + s_stb_d_tb_d_ts_list.append(self.genFullTypeJson(stb_name=stb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), id_noexist_tag=True, ts_value = self.genTsColValue(1626006833639000000, "ns"))) + s_stb_d_tb_d_ts_m_tag_list.append(self.genFullTypeJson(stb_name=stb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), id_noexist_tag=True, ts_value = self.genTsColValue(0, "ns"), t_mul_tag=True)) + s_stb_d_tb_d_ts_a_tag_list.append(self.genFullTypeJson(stb_name=stb_name, col_value=self.genTsColValue(value=tdCom.getLongName(8, "letters"), t_type="binary", value_type=value_type), tag_value=self.genTagValue(t7_value=tdCom.getLongName(8, "letters"), value_type=value_type), id_noexist_tag=True, ts_value = self.genTsColValue(0, "ns"), t_add_tag=True)) + + return d_stb_d_tb_list, s_stb_s_tb_list, s_stb_s_tb_a_tag_list, s_stb_s_tb_m_tag_list, \ + s_stb_d_tb_list, s_stb_d_tb_m_tag_list, s_stb_d_tb_a_tag_list, s_stb_s_tb_d_ts_list, \ + s_stb_s_tb_d_ts_m_tag_list, s_stb_s_tb_d_ts_a_tag_list, s_stb_d_tb_d_ts_list, \ + s_stb_d_tb_d_ts_m_tag_list, s_stb_d_tb_d_ts_a_tag_list + + def genMultiThreadSeq(self, sql_list): + tlist = list() + for insert_sql in sql_list: + t = threading.Thread(target=self._conn.schemaless_insert,args=([json.dumps(insert_sql[0])], TDSmlProtocolType.JSON.value, None)) + tlist.append(t) + return tlist + + def multiThreadRun(self, tlist): + for t in tlist: + t.start() + for t in tlist: + t.join() + + def stbInsertMultiThreadCheckCase(self, value_type="obj"): + """ + thread input different stb + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json = self.genSqlList(value_type=value_type)[0] + self.multiThreadRun(self.genMultiThreadSeq(input_json)) + tdSql.query(f"show tables;") + tdSql.checkRows(5) + + def sStbStbDdataInsertMultiThreadCheckCase(self, value_type="obj"): + """ + thread input same stb tb, different data, result keep first data + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_json, stb_name = self.genFullTypeJson(tb_name=tb_name, col_value=self.genTsColValue(value="binaryTagValue", t_type="binary", value_type=value_type)) + self.resCmp(input_json, stb_name) + s_stb_s_tb_list = self.genSqlList(stb_name=stb_name, tb_name=tb_name, value_type=value_type)[1] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + expected_tb_name = self.getNoIdTbName(stb_name)[0] + tdSql.checkEqual(tb_name, expected_tb_name) + tdSql.query(f"select * from {stb_name};") + tdSql.checkRows(1) + + def sStbStbDdataAtInsertMultiThreadCheckCase(self, value_type="obj"): + """ + thread input same stb tb, different data, add columes and tags, result keep first data + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_json, stb_name = self.genFullTypeJson(tb_name=tb_name, col_value=self.genTsColValue(value="binaryTagValue", t_type="binary", value_type=value_type)) + self.resCmp(input_json, stb_name) + s_stb_s_tb_a_tag_list = self.genSqlList(stb_name=stb_name, tb_name=tb_name, value_type=value_type)[2] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_a_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + expected_tb_name = self.getNoIdTbName(stb_name)[0] + tdSql.checkEqual(tb_name, expected_tb_name) + tdSql.query(f"select * from {stb_name};") + tdSql.checkRows(1) + + def sStbStbDdataMtInsertMultiThreadCheckCase(self, value_type="obj"): + """ + thread input same stb tb, different data, minus columes and tags, result keep first data + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_json, stb_name = self.genFullTypeJson(tb_name=tb_name, col_value=self.genTsColValue(value="binaryTagValue", t_type="binary", value_type=value_type)) + self.resCmp(input_json, stb_name) + s_stb_s_tb_m_tag_list = self.genSqlList(stb_name=stb_name, tb_name=tb_name, value_type=value_type)[3] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_m_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + expected_tb_name = self.getNoIdTbName(stb_name)[0] + tdSql.checkEqual(tb_name, expected_tb_name) + tdSql.query(f"select * from {stb_name};") + tdSql.checkRows(1) + + def sStbDtbDdataInsertMultiThreadCheckCase(self, value_type="obj"): + """ + thread input same stb, different tb, different data + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value="binaryTagValue", t_type="binary", value_type=value_type)) + self.resCmp(input_json, stb_name) + s_stb_d_tb_list = self.genSqlList(stb_name=stb_name, value_type=value_type)[4] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(6) + + def sStbDtbDdataMtInsertMultiThreadCheckCase(self): + """ + thread input same stb, different tb, different data, add col, mul tag + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value="binaryTagValue", t_type="binary")) + self.resCmp(input_json, stb_name) + s_stb_d_tb_m_tag_list = [({"metric": stb_name, "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": "omfdhyom", "tags": {"t0": {"value": False, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}}}, 'yzwswz'), + ({"metric": stb_name, "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": "vqowydbc", "tags": {"t0": {"value": False, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}}}, 'yzwswz'), + ({"metric": stb_name, "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": "plgkckpv", "tags": {"t0": {"value": False, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}}}, 'yzwswz'), + ({"metric": stb_name, "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": "cujyqvlj", "tags": {"t0": {"value": False, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}}}, 'yzwswz'), + ({"metric": stb_name, "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": "twjxisat", "tags": {"t0": {"value": False, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}}}, 'yzwswz')] + + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_m_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(2) + + def sStbDtbDdataAtInsertMultiThreadCheckCase(self, value_type="obj"): + """ + thread input same stb, different tb, different data, add tag, mul col + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value="binaryTagValue", t_type="binary", value_type=value_type)) + self.resCmp(input_json, stb_name) + s_stb_d_tb_a_tag_list = self.genSqlList(stb_name=stb_name, value_type=value_type)[6] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_a_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(6) + + def sStbStbDdataDtsInsertMultiThreadCheckCase(self): + """ + thread input same stb tb, different ts + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_json, stb_name = self.genFullTypeJson(tb_name=tb_name, col_value=self.genTsColValue(value="binaryTagValue", t_type="binary")) + self.resCmp(input_json, stb_name) + s_stb_s_tb_d_ts_list = [({"metric": stb_name, "timestamp": {"value": 0, "type": "ns"}, "value": "hkgjiwdj", "tags": {"id": tb_name, "t0": {"value": False, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}, "t7": {"value": "vozamcts", "type": "binary"}, "t8": {"value": "ncharTagValue", "type": "nchar"}}}, 'yzwswz'), + ({"metric": stb_name, "timestamp": {"value": 0, "type": "ns"}, "value": "rljjrrul", "tags": {"id": tb_name, "t0": {"value": False, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}, "t7": {"value": "bmcanhbs", "type": "binary"}, "t8": {"value": "ncharTagValue", "type": "nchar"}}}, 'yzwswz'), + ({"metric": stb_name, "timestamp": {"value": 0, "type": "ns"}, "value": "basanglx", "tags": {"id": tb_name, "t0": {"value": False, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}, "t7": {"value": "enqkyvmb", "type": "binary"}, "t8": {"value": "ncharTagValue", "type": "nchar"}}}, 'yzwswz'), + ({"metric": stb_name, "timestamp": {"value": 0, "type": "ns"}, "value": "clsajzpp", "tags": {"id": tb_name, "t0": {"value": False, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}, "t7": {"value": "eivaegjk", "type": "binary"}, "t8": {"value": "ncharTagValue", "type": "nchar"}}}, 'yzwswz'), + ({"metric": stb_name, "timestamp": {"value": 0, "type": "ns"}, "value": "jitwseso", "tags": {"id": tb_name, "t0": {"value": True, "type": "bool"}, "t1": {"value": 127, "type": "tinyint"}, "t2": {"value": 32767, "type": "smallint"}, "t3": {"value": 2147483647, "type": "int"}, "t4": {"value": 9223372036854775807, "type": "bigint"}, "t5": {"value": 11.12345, "type": "float"}, "t6": {"value": 22.123456789, "type": "double"}, "t7": {"value": "yhlwkddq", "type": "binary"}, "t8": {"value": "ncharTagValue", "type": "nchar"}}}, 'yzwswz')] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_d_ts_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(6) + + def sStbStbDdataDtsMtInsertMultiThreadCheckCase(self): + """ + thread input same stb tb, different ts, add col, mul tag + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_json, stb_name = self.genFullTypeJson(tb_name=tb_name, col_value=self.genTsColValue(value="binaryTagValue", t_type="binary")) + self.resCmp(input_json, stb_name) + s_stb_s_tb_d_ts_m_tag_list = [({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'pjndapjb', 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'tuzsfrom', 'type': 'binary'}, 'id': tb_name}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'llqzvgvw', 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'nttjdzgi', 'type': 'binary'}, 'id': tb_name}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'tclbosqc', 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'uatpzgpi', 'type': 'binary'}, 'id': tb_name}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'rlpuzodt', 'tags': {'t0': {'value': True, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'cwnpdnng', 'type': 'binary'}, 'id': tb_name}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'rhnikvfq', 'tags': {'t0': {'value': True, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'afcibyeb', 'type': 'binary'}, 'id': tb_name}}, 'punftb')] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_d_ts_m_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(6) + tdSql.query(f"select * from {stb_name} where t8 is not NULL") + tdSql.checkRows(6) + + def sStbStbDdataDtsAtInsertMultiThreadCheckCase(self): + """ + thread input same stb tb, different ts, add tag, mul col + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + tb_name = tdCom.getLongName(7, "letters") + input_json, stb_name = self.genFullTypeJson(tb_name=tb_name, col_value=self.genTsColValue(value="binaryTagValue", t_type="binary")) + self.resCmp(input_json, stb_name) + s_stb_s_tb_d_ts_a_tag_list = [({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'pjndapjb', 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'tuzsfrom', 'type': 'binary'}, 't8': {'value': 'ncharTagValue', 'type': 'nchar'}, 't11': {'value': 127, 'type': 'tinyint'}, 't10': {'value': 'ncharTagValue', 'type': 'nchar'}, 'id': tb_name}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'llqzvgvw', 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'nttjdzgi', 'type': 'binary'}, 't8': {'value': 'ncharTagValue', 'type': 'nchar'}, 't11': {'value': 127, 'type': 'tinyint'}, 't10': {'value': 'ncharTagValue', 'type': 'nchar'}, 'id': tb_name}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': {'value': 'tclbosqc', 'type': 'binary'}, 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'uatpzgpi', 'type': 'binary'}, 't8': {'value': 'ncharTagValue', 'type': 'nchar'}, 't11': {'value': 127, 'type': 'tinyint'}, 't10': {'value': 'ncharTagValue', 'type': 'nchar'}, 'id': tb_name}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'rlpuzodt', 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'cwnpdnng', 'type': 'binary'}, 't8': {'value': 'ncharTagValue', 'type': 'nchar'}, 't11': {'value': 127, 'type': 'tinyint'}, 't10': {'value': 'ncharTagValue', 'type': 'nchar'}, 'id': tb_name}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': {'value': 'rhnikvfq', 'type': 'binary'}, 'tags': {'t0': {'value': True, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'afcibyeb', 'type': 'binary'}, 't8': {'value': 'ncharTagValue', 'type': 'nchar'}, 't11': {'value': 127, 'type': 'tinyint'}, 't10': {'value': 'ncharTagValue', 'type': 'nchar'}, 'id': tb_name}}, 'punftb')] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_s_tb_d_ts_a_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(1) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(6) + for t in ["t10", "t11"]: + tdSql.query(f"select * from {stb_name} where {t} is not NULL;") + tdSql.checkRows(0) + + def sStbDtbDdataDtsInsertMultiThreadCheckCase(self, value_type="obj"): + """ + thread input same stb, different tb, data, ts + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value="binaryTagValue", t_type="binary", value_type=value_type)) + self.resCmp(input_json, stb_name) + s_stb_d_tb_d_ts_list = self.genSqlList(stb_name=stb_name, value_type=value_type)[10] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_d_ts_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(6) + + def sStbDtbDdataDtsMtInsertMultiThreadCheckCase(self): + """ + thread input same stb, different tb, data, ts, add col, mul tag + """ + tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') + tdCom.cleanTb() + input_json, stb_name = self.genFullTypeJson(col_value=self.genTsColValue(value="binaryTagValue", t_type="binary")) + self.resCmp(input_json, stb_name) + s_stb_d_tb_d_ts_m_tag_list = [({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'pjndapjb', 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': {'value': 'llqzvgvw', 'type': 'binary'}, 'tags': {'t0': {'value': True, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': 'tclbosqc', 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': {'value': 'rlpuzodt', 'type': 'binary'}, 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}}}, 'punftb'), + ({'metric': stb_name, 'timestamp': {'value': 0, 'type': 'ns'}, 'value': {'value': 'rhnikvfq', 'type': 'binary'}, 'tags': {'t0': {'value': False, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {"value": 9223372036854775807, "type": "bigint"}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}}}, 'punftb')] + self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_d_ts_m_tag_list)) + tdSql.query(f"show tables;") + tdSql.checkRows(3) + + def test(self): + try: + input_json = f'test_nchar 0 L"涛思数据" t0=f,t1=L"涛思数据",t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64' + self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) + # input_json, stb_name = self.genFullTypeJson() + # self.resCmp(input_json, stb_name) + except SchemalessError as err: + print(err.errno) + + def runAll(self): + for value_type in ["obj", "default"]: + self.initCheckCase(value_type) + self.symbolsCheckCase(value_type) + # self.tsCheckCase(value_type) + self.idSeqCheckCase(value_type) + self.idLetterCheckCase(value_type) + self.noIdCheckCase(value_type) + self.maxColTagCheckCase(value_type) + self.idIllegalNameCheckCase(value_type) + self.idStartWithNumCheckCase(value_type) + self.nowTsCheckCase(value_type) + self.dateFormatTsCheckCase(value_type) + self.illegalTsCheckCase(value_type) + self.tbnameCheckCase(value_type) + # self.tagValueLengthCheckCase(value_type) + self.colValueLengthCheckCase(value_type) + self.tagColIllegalValueCheckCase(value_type) + # self.duplicateIdTagColInsertCheckCase(value_type) + self.noIdStbExistCheckCase(value_type) + self.duplicateInsertExistCheckCase(value_type) + # self.tagColBinaryNcharLengthCheckCase(value_type) + # self.tagColAddDupIDCheckCase(value_type) + # self.tagAddCheckCase(value_type) + # self.tagMd5Check(value_type) + # self.tagColBinaryMaxLengthCheckCase(value_type) + # self.tagColNcharMaxLengthCheckCase(value_type) + # self.batchInsertCheckCase(value_type) + # self.multiInsertCheckCase(10, value_type) + self.multiColsInsertCheckCase(value_type) + self.blankColInsertCheckCase(value_type) + self.blankTagInsertCheckCase(value_type) + self.multiFieldCheckCase(value_type) + # self.stbInsertMultiThreadCheckCase(value_type) + self.pointTransCheckCase(value_type) + self.tagNameLengthCheckCase() + self.boolTypeCheckCase() + self.batchErrorInsertCheckCase() + self.chineseCheckCase() + # self.spellCheckCase() + self.tbnameTagsColsNameCheckCase() + # # MultiThreads + # self.sStbStbDdataInsertMultiThreadCheckCase() + # self.sStbStbDdataAtInsertMultiThreadCheckCase() + # self.sStbStbDdataMtInsertMultiThreadCheckCase() + # self.sStbDtbDdataInsertMultiThreadCheckCase() + # self.sStbDtbDdataAtInsertMultiThreadCheckCase() + # self.sStbDtbDdataDtsInsertMultiThreadCheckCase() + # self.sStbDtbDdataMtInsertMultiThreadCheckCase() + # self.sStbStbDdataDtsInsertMultiThreadCheckCase() + # self.sStbStbDdataDtsMtInsertMultiThreadCheckCase() + # self.sStbDtbDdataDtsMtInsertMultiThreadCheckCase() + # self.lengthIcreaseCrashCheckCase() + + def run(self): + print("running {}".format(__file__)) + self.createDb() + try: + self.runAll() + except Exception as err: + print(''.join(traceback.format_exception(None, err, err.__traceback__))) + raise err + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 89ad138fb1621a69d455d2a1d960e3874a863deb Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 31 May 2022 19:00:06 +0800 Subject: [PATCH 09/57] test: finish 1-insert/opentsdb_json_taosc_insert.py --- tests/system-test/fulltest.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 2725cf3d13..509e39d1f8 100644 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -14,7 +14,9 @@ python3 ./test.py -f 0-others/udf_restart_taosd.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py +python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py From 061a7996a3bc9f2e038e798f65a93b2e3e3501bf Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 31 May 2022 19:07:38 +0800 Subject: [PATCH 10/57] test: update schemaless-insert 1-insert/opentsdb_telnet_line_taosc_insert.py to CI --- .../system-test/1-insert/opentsdb_telnet_line_taosc_insert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index 4c56511d27..6d50dd4fda 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -36,10 +36,10 @@ class TDTestCase: if db_update_tag == 0: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms'") + tdSql.execute(f"create database if not exists {name} precision 'us'") else: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms' update 1") + tdSql.execute(f"create database if not exists {name} precision 'ns' update 1") tdSql.execute(f'use {name}') def timeTrans(self, time_value, ts_type): From 1013b06107fa060fe86eb2b6e0348042f767a3e4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 31 May 2022 19:24:26 +0800 Subject: [PATCH 11/57] refactor: make trans support global conflict --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndConsumer.c | 6 +++--- source/dnode/mnode/impl/src/mndOffset.c | 2 +- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndSubscribe.c | 13 ++++++------- source/dnode/mnode/impl/src/mndTopic.c | 4 ++-- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index d1fd98137d..9608a84a10 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -122,7 +122,7 @@ typedef struct { int64_t dbUid; char dbname[TSDB_DB_FNAME_LEN]; char lastError[TSDB_TRANS_ERROR_LEN]; - char desc[TSDB_TRANS_DESC_LEN]; + char o[TSDB_TRANS_DESC_LEN]; int32_t startFunc; int32_t stopFunc; int32_t paramLen; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 470b7ac1bf..0314891d59 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -97,7 +97,7 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { mndReleaseConsumer(pMnode, pConsumer); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg); if (pTrans == NULL) goto FAIL; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; @@ -121,7 +121,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { mndReleaseConsumer(pMnode, pConsumer); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg); if (pTrans == NULL) goto FAIL; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; @@ -403,7 +403,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t newTopicNum = taosArrayGetSize(newSub); // check topic existance - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg); if (pTrans == NULL) goto SUBSCRIBE_OVER; for (int32_t i = 0; i < newTopicNum; i++) { diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 8dfcbfbf3c..00c8bb30d0 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -179,7 +179,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) { tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg); for (int32_t i = 0; i < commitOffsetReq.num; i++) { SMqOffset *pOffset = &commitOffsetReq.offsets[i]; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 246d5dc951..5ee5b06a57 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -402,7 +402,7 @@ static int32_t mndCreateStream(SMnode *pMnode, SRpcMsg *pReq, SCMCreateStreamReq tstrncpy(streamObj.targetDb, pDb->name, TSDB_DB_FNAME_LEN); } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 7f422e2919..e7ce1cbe78 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -393,8 +393,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR mInfo("rebalance calculation completed, rebalanced vg:"); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("vg: %d moved from consumer %ld to consumer %ld", pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, - pOutputRebVg->newConsumerId); + mInfo("vgId:%d moved from consumer %" PRId64 " to consumer %" PRId64, pOutputRebVg->pVgEp->vgId, + pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } // 9. clear @@ -404,10 +404,9 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pMsg); - if (pTrans == NULL) { - return -1; - } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg); + if (pTrans == NULL) return -1; + // make txn: // 1. redo action: action to all vg const SArray *rebVgs = pOutput->rebVgs; @@ -623,7 +622,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 28f7ae2607..74d8f9c2b6 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -377,7 +377,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.withSchema = 1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); taosMemoryFreeClear(topicObj.ast); @@ -545,7 +545,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } #endif - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); return -1; From d68ff9641fcb5fc97930d027b4a9f6f57f56e20a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 31 May 2022 19:27:12 +0800 Subject: [PATCH 12/57] fix(parser): extract db name --- source/libs/parser/src/parTranslater.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index af2c13946d..66cc24e39c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3310,6 +3310,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS if ('\0' != pStmt->subSTbName[0]) { pReq->subType = TOPIC_SUB_TYPE__TABLE; toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); + tNameGetFullDbName(&name, pReq->subDbName); tNameExtractFullName(&name, pReq->subStbName); } else if ('\0' != pStmt->subDbName[0]) { pReq->subType = TOPIC_SUB_TYPE__DB; From a15f46d0f5a2aae2de3340f831cbdad3974084e0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 31 May 2022 19:40:34 +0800 Subject: [PATCH 13/57] refactor: make trans support global conflict --- source/dnode/mnode/impl/inc/mndDef.h | 3 +-- tests/test/c/sdbDump.c | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9608a84a10..5e889c91f8 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -119,10 +119,9 @@ typedef struct { SArray* commitActions; int64_t createdTime; int64_t lastExecTime; - int64_t dbUid; char dbname[TSDB_DB_FNAME_LEN]; char lastError[TSDB_TRANS_ERROR_LEN]; - char o[TSDB_TRANS_DESC_LEN]; + char desc[TSDB_TRANS_DESC_LEN]; int32_t startFunc; int32_t stopFunc; int32_t paramLen; diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c index b2f66a7e76..1781dd3183 100644 --- a/tests/test/c/sdbDump.c +++ b/tests/test/c/sdbDump.c @@ -282,7 +282,6 @@ void dumpTrans(SSdb *pSdb, SJson *json) { tjsonAddIntegerToObject(item, "conflict", pObj->conflict); tjsonAddIntegerToObject(item, "exec", pObj->exec); tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid)); tjsonAddStringToObject(item, "dbname", pObj->dbname); tjsonAddIntegerToObject(item, "commitLogNum", taosArrayGetSize(pObj->commitActions)); tjsonAddIntegerToObject(item, "redoActionNum", taosArrayGetSize(pObj->redoActions)); From 77757f3ef9188128ad4ed369285d0bf28bae86b7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 31 May 2022 19:52:40 +0800 Subject: [PATCH 14/57] feat(tmq): support drop cgroup --- include/common/tmsgdef.h | 1 - source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 ++ source/dnode/mnode/impl/src/mndSubscribe.c | 13 ++++++++++--- source/libs/parser/src/parTranslater.c | 2 +- source/util/src/terror.c | 2 +- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 51a15c1489..fe722c031d 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -144,7 +144,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_CGROUP, "mnode-drop-cgroup", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, NULL) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index f6350ba279..34576d5441 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -197,6 +197,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 2f0e2cdd8f..2dbc0cbbc1 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -78,6 +78,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP, mndProcessDropCgroupReq); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP_RSP, mndProcessSubscribeInternalRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SUBSCRIPTIONS, mndRetrieveSubscribe); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextSubscribe); @@ -596,8 +597,8 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { } static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - /*SSdb *pSdb = pMnode->pSdb;*/ + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; SMDropCgroupReq dropReq = {0}; if (tDeserializeSMDropCgroupReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { @@ -617,15 +618,17 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { } } - if (taosHashGetSize(pSub->consumerHash) == 0) { + if (taosHashGetSize(pSub->consumerHash) != 0) { terrno = TSDB_CODE_MND_CGROUP_USED; mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); + mndReleaseSubscribe(pMnode, pSub); return -1; } STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_CGROUP, pReq); if (pTrans == NULL) { mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); + mndReleaseSubscribe(pMnode, pSub); return -1; } @@ -633,14 +636,18 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { if (mndDropOffsetBySubKey(pMnode, pTrans, pSub->key) < 0) { ASSERT(0); + mndReleaseSubscribe(pMnode, pSub); return -1; } if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) { mError("cgroup %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); + mndReleaseSubscribe(pMnode, pSub); return -1; } + mndTransPrepare(pMnode, pTrans); + mndReleaseSubscribe(pMnode, pSub); return TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index af2c13946d..2b3a1762a7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3380,7 +3380,7 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt dropReq.igNotExists = pStmt->ignoreNotExists; strcpy(dropReq.cgroup, pStmt->cgroup); - return buildCmdMsg(pCxt, TDMT_MND_DROP_CGROUP, (FSerializeFunc)tSerializeSMDropCgroupReq, &dropReq); + return buildCmdMsg(pCxt, TDMT_MND_MQ_DROP_CGROUP, (FSerializeFunc)tSerializeSMDropCgroupReq, &dropReq); } static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 178d6e8d2b..575ae10bbb 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -270,7 +270,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC, "Invalid topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_QUERY, "Topic with invalid query") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION, "Topic with invalid option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_EXIST, "Consumer not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer waiting for rebalance") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED, "Topic subscribed cannot be dropped") TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_ALREADY_EXIST, "Stream already exists") From 13694d0352562a1260eead306c6f365e32043bdc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 May 2022 19:55:12 +0800 Subject: [PATCH 15/57] add tag filter sim --- tests/script/tsim/stable/tag_filter.sim | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/script/tsim/stable/tag_filter.sim diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim new file mode 100644 index 0000000000..c8edfb1ee3 --- /dev/null +++ b/tests/script/tsim/stable/tag_filter.sim @@ -0,0 +1,59 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ========== prepare stb and ctb +sql create database db vgroups 1 +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd" + + +sql create table db.ctb1 using db.stb tags(1, "102") +sql insert into db.ctb1 values(now, 1, "2") + +sql create table db.ctb2 using db.stb tags(2, "102") +sql insert into db.ctb2 values(now, 2, "2") + +sql create table db.ctb3 using db.stb tags(3, "102") +sql insert into db.ctb3 values(now, 3, "2") + +sql create table db.ctb4 using db.stb tags(4, "102") +sql insert into db.ctb4 values(now, 4, "2") + +sql create table db.ctb5 using db.stb tags(5, "102") +sql insert into db.ctb5 values(now, 5, "2") + +sql create table db.ctb6 using db.stb tags(6, "102") +sql insert into db.ctb6 values(now, 6, "2") + +sql select * from db.stb where t1 = 1 +if $rows != 1 then + return -1 +endi + +sql select * from db.stb where t1 < 1 +if $rows != 0 then + return -=1 +endi + +sql select * from db.stb where t1 < 2 +if $rows != 1 then + return -1 +endi + +sql select * from db.stb where t1 <= 2 +if $rows != 2 then + return -1 +endi + +sql select * from db.stb where t1 >= 1 +if $rows != 6 then + return -1 +endi + +sql select * from db.stb where t1 > 1 +if $rows != 5 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 89ca8c08000e12bef22e225b7489a35ffdfda7c4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 May 2022 20:06:42 +0800 Subject: [PATCH 16/57] add tag filter sim --- tests/script/jenkins/basic.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index b2ffe83b0b..ca0db9e32a 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -108,6 +108,7 @@ ./test.sh -f tsim/stable/alter_insert1.sim ./test.sh -f tsim/stable/alter_insert2.sim ./test.sh -f tsim/stable/alter_import.sim +./test.sh -f tsim/stable/tag_filter.sim # --- for multi process mode ./test.sh -f tsim/user/basic1.sim -m From f89ce48e664acf95d9caca6fdd7723a212867c90 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 20:18:02 +0800 Subject: [PATCH 17/57] feat:add new logic for new tag format --- source/libs/parser/src/parInsert.c | 72 +++++++++++-------- source/libs/parser/src/parTranslater.c | 6 +- source/libs/scalar/src/sclvector.c | 2 +- .../libs/scalar/test/scalar/scalarTests.cpp | 40 ++++------- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d670f5ad38..05e5d46c7e 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -958,6 +958,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint SArray *pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); SToken sToken; bool isParseBindParam = false; + bool isJson = false; for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); @@ -994,6 +995,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint if(code != TSDB_CODE_SUCCESS){ goto end; } + isJson = true; }else{ STagVal val = {0}; code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, &val, &pCxt->msg); @@ -1014,7 +1016,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } STag* pTag = NULL; - code = tTagNew(pTagVals, 1, false, &pTag); + code = tTagNew(pTagVals, 1, isJson, &pTag); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -1404,7 +1406,6 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), .pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), .totalNum = 0, - .pTagVals = NULL, .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), .pStmtCb = pContext->pStmtCb}; @@ -1526,6 +1527,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN int32_t code = TSDB_CODE_SUCCESS; SSchema* pSchema = pDataBlock->pTableMeta->schema; + bool isJson = false; for (int c = 0; c < tags->numOfBound; ++c) { if (bind[c].is_null && bind[c].is_null[0]) { continue; @@ -1537,42 +1539,56 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; } - - STagVal val = {0}; - if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ - val.pData = (uint8_t*)bind[c].buffer; - val.nData = colLen; - }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ - int32_t output = 0; - void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE); - if(p == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + if (colLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pBuf, "json string too long than 4095", bind[c].buffer); goto end; } - if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { - if (errno == E2BIG) { - taosMemoryFree(p); - code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + + isJson = true; + char *tmp = taosMemoryCalloc(1, colLen + 1); + memcpy(tmp, bind[c].buffer, colLen); + code = parseJsontoTagData(tmp, pTagArray, &pBuf); + taosMemoryFree(tmp); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } + }else{ + STagVal val = {0}; + if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ + val.pData = (uint8_t*)bind[c].buffer; + val.nData = colLen; + }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ + int32_t output = 0; + void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE); + if(p == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); - taosMemoryFree(p); - code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer); - goto end; + if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (errno == E2BIG) { + taosMemoryFree(p); + code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + goto end; + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + taosMemoryFree(p); + code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer); + goto end; + } + val.pData = p; + val.nData = output; + }else{ + memcpy(&val.i64, bind[c].buffer, colLen); } - val.pData = p; - val.nData = output; - }else{ - memcpy(&val.i64, bind[c].buffer, colLen); + taosArrayPush(pTagArray, &val); } - taosArrayPush(pTagArray, &val); } STag* pTag = NULL; - // TODO: stmt support json - if (0 != tTagNew(pTagArray, 1, false, &pTag)) { + if (0 != tTagNew(pTagArray, 1, isJson, &pTag)) { code = buildInvalidOperationMsg(&pBuf, "out of memory"); goto end; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 96ee694e1f..bd31c16f54 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4235,7 +4235,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } - code = tTagNew(pTagArray, 1, false, ppTag); + code = tTagNew(pTagArray, 1, isJson, ppTag); end: if(isJson){ @@ -4302,7 +4302,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } ++index; } - code = tTagNew(pTagArray, 1, false, ppTag); + code = tTagNew(pTagArray, 1, isJson, ppTag); end: if(isJson){ @@ -4584,7 +4584,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS if (TSDB_CODE_SUCCESS != code) { break; } - code = tTagNew(pTagVals, 1, false, &pTag); + code = tTagNew(pTagVals, 1, true, &pTag); }while(0); for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c4b3809035..f84f564b8d 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -951,7 +951,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO char *data = tTagValToData(value, true); colDataAppend(pOutputCol, i, data, data == NULL); if(value && IS_VAR_DATA_TYPE(value->type) && data){ - taosMemoryFree(data) + taosMemoryFree(data); } } } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 3a16e0a969..2081c6a3f0 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1105,24 +1105,16 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do nodesDestroyNode(opNode); } -#if 0 TEST(columnTest, json_column_arith_op) { scltInitLogFile(); char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); - SKVRowBuilder kvRowBuilder; - tdInitKVRowBuilder(&kvRowBuilder); - parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1); - if(tmp == NULL){ - ASSERT_TRUE(0); - } - memmove(tmp+1, tmp, kvRowLen(tmp)); - *tmp = TSDB_DATA_TYPE_JSON; - row = tmp; + SArray *tags = taosArrayInit(1, sizeof(STagVal)); + parseJsontoTagData(rightv, tags, NULL); + STag* row = NULL; + tTagNew(tags, 1, true, &row); const int32_t len = 8; EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, @@ -1176,10 +1168,10 @@ TEST(columnTest, json_column_arith_op) { makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); } - tdDestroyKVRowBuilder(&kvRowBuilder); + taosArrayDestroy(tags); taosMemoryFree(row); } -#endif + void *prepareNchar(char* rightData){ int32_t len = 0; int32_t inputLen = strlen(rightData); @@ -1189,24 +1181,17 @@ void *prepareNchar(char* rightData){ varDataSetLen(t, len); return t; } -#if 0 + TEST(columnTest, json_column_logic_op) { scltInitLogFile(); char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}"; char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); - SKVRowBuilder kvRowBuilder; - tdInitKVRowBuilder(&kvRowBuilder); - parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1); - if(tmp == NULL){ - ASSERT_TRUE(0); - } - memmove(tmp+1, tmp, kvRowLen(tmp)); - *tmp = TSDB_DATA_TYPE_JSON; - row = tmp; + SArray *tags = taosArrayInit(1, sizeof(STagVal)); + parseJsontoTagData(rightv, tags, NULL); + STag* row = NULL; + tTagNew(tags, 1, true, &row); const int32_t len = 9; const int32_t len1 = 4; @@ -1306,10 +1291,9 @@ TEST(columnTest, json_column_logic_op) { taosMemoryFree(rightData); } - tdDestroyKVRowBuilder(&kvRowBuilder); + taosArrayDestroy(tags); taosMemoryFree(row); } -#endif TEST(columnTest, smallint_value_add_int_column) { scltInitLogFile(); From 8fb3bf538448b3bd17699fa5b55f2b06a0350737 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 31 May 2022 20:20:19 +0800 Subject: [PATCH 18/57] add case for stateduration --- .../2-query/function_stateduration.py | 431 ++++++++++++++++++ 1 file changed, 431 insertions(+) create mode 100644 tests/system-test/2-query/function_stateduration.py diff --git a/tests/system-test/2-query/function_stateduration.py b/tests/system-test/2-query/function_stateduration.py new file mode 100644 index 0000000000..b25a658469 --- /dev/null +++ b/tests/system-test/2-query/function_stateduration.py @@ -0,0 +1,431 @@ +from math import floor +from random import randint, random +from numpy import equal +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def test_errors(self): + error_sql_lists = [ + # "select stateduration(c1,'GT',5,1s) from t1" + "select stateduration from t1", + "select stateduration(123--123)==1 from t1", + "select stateduration(123,123) from t1", + "select stateduration(c1,ts) from t1", + "select stateduration(c1,c1,ts) from t1", + "select stateduration(c1 ,c2 ) from t1", + "select stateduration(c1 ,NULL) from t1", + #"select stateduration(c1 ,'NULL',1.0,1s) from t1", + "select stateduration(c1 ,'GT','1',1s) from t1", + "select stateduration(c1 ,'GT','tbname',1s) from t1", + "select stateduration(c1 ,'GT','*',1s) from t1", + "select stateduration(c1 ,'GT',ts,1s) from t1", + "select stateduration(c1 ,'GT',max(c1),1s) from t1", + "select stateduration(abs(c1) ,'GT',1,1s) from t1", + "select stateduration(c1+2 ,'GT',1,1s) from t1", + "select stateduration(c1 ,'GT',1,1u) from t1", + "select stateduration(c1 ,'GT',1,now) from t1", + "select stateduration(c1 ,'GT','1',1s) from t1", + "select stateduration(c1 ,'GT','1',True) from t1", + "select stateduration(stateduration(c1) ab from t1)", + "select stateduration(c1 ,'GT',1,,)int from t1", + "select stateduration('c1','GT',1) from t1", + "select stateduration('c1','GT', 1 , NULL) from t1", + "select stateduration('c1','GT', 1 , '') from t1", + "select stateduration('c1','GT', 1 ,c%) from t1", + "select stateduration(c1 ,'GT',1,t1) from t1", + "select stateduration(c1 ,'GT',1,True) from t1", + "select stateduration(c1 ,'GT',1,1s) , count(c1) from t1", + "select stateduration(c1 ,'GT',1,1s) , avg(c1) from t1", + "select stateduration(c1 ,'GT',1,1s) , min(c1) from t1", + "select stateduration(c1 ,'GT',1,1s) , spread(c1) from t1", + "select stateduration(c1 ,'GT',1,1s) , diff(c1) from t1", + "select stateduration(c1 ,'GT',1,1s) , abs(c1) from t1", + "select stateduration(c1 ,'GT',1,1s) , c1 from t1", + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + pass + + def support_types(self): + other_no_value_types = [ + "select stateduration(ts,'GT',1,1s) from t1" , + "select stateduration(c7,'GT',1,1s) from t1", + "select stateduration(c8,'GT',1,1s) from t1", + "select stateduration(c9,'GT',1,1s) from t1", + "select stateduration(ts,'GT',1,1s) from ct1" , + "select stateduration(c7,'GT',1,1s) from ct1", + "select stateduration(c8,'GT',1,1s) from ct1", + "select stateduration(c9,'GT',1,1s) from ct1", + "select stateduration(ts,'GT',1,1s) from ct3" , + "select stateduration(c7,'GT',1,1s) from ct3", + "select stateduration(c8,'GT',1,1s) from ct3", + "select stateduration(c9,'GT',1,1s) from ct3", + "select stateduration(ts,'GT',1,1s) from ct4" , + "select stateduration(c7,'GT',1,1s) from ct4", + "select stateduration(c8,'GT',1,1s) from ct4", + "select stateduration(c9,'GT',1,1s) from ct4", + "select stateduration(ts,'GT',1,1s) from stb1 partition by tbname" , + "select stateduration(c7,'GT',1,1s) from stb1 partition by tbname", + "select stateduration(c8,'GT',1,1s) from stb1 partition by tbname", + "select stateduration(c9,'GT',1,1s) from stb1 partition by tbname" + ] + + for type_sql in other_no_value_types: + tdSql.error(type_sql) + tdLog.info("support type ok , sql is : %s"%type_sql) + + type_sql_lists = [ + "select stateduration(c1,'GT',1,1s) from t1", + "select stateduration(c2,'GT',1,1s) from t1", + "select stateduration(c3,'GT',1,1s) from t1", + "select stateduration(c4,'GT',1,1s) from t1", + "select stateduration(c5,'GT',1,1s) from t1", + "select stateduration(c6,'GT',1,1s) from t1", + + "select stateduration(c1,'GT',1,1s) from ct1", + "select stateduration(c2,'GT',1,1s) from ct1", + "select stateduration(c3,'GT',1,1s) from ct1", + "select stateduration(c4,'GT',1,1s) from ct1", + "select stateduration(c5,'GT',1,1s) from ct1", + "select stateduration(c6,'GT',1,1s) from ct1", + + "select stateduration(c1,'GT',1,1s) from ct3", + "select stateduration(c2,'GT',1,1s) from ct3", + "select stateduration(c3,'GT',1,1s) from ct3", + "select stateduration(c4,'GT',1,1s) from ct3", + "select stateduration(c5,'GT',1,1s) from ct3", + "select stateduration(c6,'GT',1,1s) from ct3", + + "select stateduration(c1,'GT',1,1s) from stb1 partition by tbname", + "select stateduration(c2,'GT',1,1s) from stb1 partition by tbname", + "select stateduration(c3,'GT',1,1s) from stb1 partition by tbname", + "select stateduration(c4,'GT',1,1s) from stb1 partition by tbname", + "select stateduration(c5,'GT',1,1s) from stb1 partition by tbname", + "select stateduration(c6,'GT',1,1s) from stb1 partition by tbname", + + "select stateduration(c6,'GT',1,1s) as alisb from stb1 partition by tbname", + "select stateduration(c6,'GT',1,1s) alisb from stb1 partition by tbname", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def support_opers(self): + oper_lists = ['LT','lt','Lt','lT','GT','gt','Gt','gT','LE','le','Le','lE','GE','ge','Ge','gE','NE','ne','Ne','nE','EQ','eq','Eq','eQ'] + + oper_errors = [",","*","NULL","tbname","ts","sum","_c0"] + + for oper in oper_lists: + tdSql.query(f"select stateduration(c1 ,'{oper}',1,1s) as col from t1") + tdSql.checkRows(12) + + for oper in oper_errors: + tdSql.error(f"select stateduration(c1 ,'{oper}',1,1s) as col from t1") + + + def basic_stateduration_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select stateduration(c6,'GT',1,1s) from ct3") + tdSql.checkRows(0) + tdSql.query("select stateduration(c6,'GT',1,1s) from ct3") + tdSql.checkRows(0) + tdSql.query("select stateduration(c6,'GT',1,1s) from ct3") + tdSql.checkRows(0) + tdSql.query("select stateduration(c6,'GT',1,1s) from ct3") + tdSql.checkRows(0) + tdSql.query("select stateduration(c6,'GT',1,1s) from ct3") + tdSql.checkRows(0) + tdSql.query("select stateduration(c6,'GT',1,1s) from ct3") + + # will support _rowts mix with + # tdSql.query("select (c6,'GT',1,1s),_rowts from ct3") + + # auto check for t1 table + # used for regular table + tdSql.query("select stateduration(c6,'GT',1,1s) from t1") + + # unique with super tags + + tdSql.query("select stateduration(c6,'GT',1,1s) from ct1") + tdSql.checkRows(13) + + tdSql.query("select stateduration(c6,'GT',1,1s) from ct4") + tdSql.checkRows(12) + + tdSql.error("select stateduration(c6,'GT',1,1s),tbname from ct1") + tdSql.error("select stateduration(c6,'GT',1,1s),t1 from ct1") + + # unique with common col + tdSql.error("select stateduration(c6,'GT',1,1s) ,ts from ct1") + tdSql.error("select stateduration(c6,'GT',1,1s) ,c1 from ct1") + + # unique with scalar function + tdSql.error("select stateduration(c6,'GT',1,1s) ,abs(c1) from ct1") + tdSql.error("select stateduration(c6,'GT',1,1s) , unique(c2) from ct1") + tdSql.error("select stateduration(c6,'GT',1,1s) , abs(c2)+2 from ct1") + + + # unique with aggregate function + tdSql.error("select stateduration(c6,'GT',1,1s) ,sum(c1) from ct1") + tdSql.error("select stateduration(c6,'GT',1,1s) ,max(c1) from ct1") + tdSql.error("select stateduration(c6,'GT',1,1s) ,csum(c1) from ct1") + tdSql.error("select stateduration(c6,'GT',1,1s) ,count(c1) from ct1") + + # unique with filter where + tdSql.query("select stateduration(c6,'GT',1,1s) from ct4 where c1 is null") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + + tdSql.query("select stateduration(c1,'GT',1,1s) from t1 where c1 >2 ") + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 10886404) + tdSql.checkData(2, 0, 23500810) + tdSql.checkData(4, 0, 57456020) + tdSql.checkData(5, 0, 60393624) + + tdSql.query("select stateduration(c2,'GT',1,1s) from t1 where c2 between 0 and 99999") + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 6134400) + tdSql.checkData(6, 0, -1) + + + # unique with union all + tdSql.query("select stateduration(c1,'GT',1,1s) from ct4 union all select stateduration(c1,'GT',1,1s) from ct1") + tdSql.checkRows(25) + tdSql.query("select stateduration(c1,'GT',1,1s) from ct4 union all select distinct(c1) from ct4") + tdSql.checkRows(22) + + # unique with join + # prepare join datas with same ts + + tdSql.execute(" use db ") + tdSql.execute(" create stable st1 (ts timestamp , num int) tags(ind int)") + tdSql.execute(" create table tb1 using st1 tags(1)") + tdSql.execute(" create table tb2 using st1 tags(2)") + + tdSql.execute(" create stable st2 (ts timestamp , num int) tags(ind int)") + tdSql.execute(" create table ttb1 using st2 tags(1)") + tdSql.execute(" create table ttb2 using st2 tags(2)") + + start_ts = 1622369635000 # 2021-05-30 18:13:55 + + for i in range(10): + ts_value = start_ts+i*1000 + tdSql.execute(f" insert into tb1 values({ts_value} , {i})") + tdSql.execute(f" insert into tb2 values({ts_value} , {i})") + + tdSql.execute(f" insert into ttb1 values({ts_value} , {i})") + tdSql.execute(f" insert into ttb2 values({ts_value} , {i})") + + tdSql.query("select stateduration(tb1.num,'GT',1,1s) from tb1, tb2 where tb1.ts=tb2.ts ") + tdSql.checkRows(10) + tdSql.checkData(0,0,-1) + tdSql.checkData(1,0,-1) + tdSql.checkData(2,0,0) + tdSql.checkData(9,0,7) + + tdSql.query("select stateduration(tb1.num,'GT',1,1s) from tb1, tb2 where tb1.ts=tb2.ts union all select stateduration(tb2.num,'GT',1,1s) from tb1, tb2 where tb1.ts=tb2.ts ") + tdSql.checkRows(20) + + # nest query + # tdSql.query("select unique(c1) from (select c1 from ct1)") + tdSql.query("select c1 from (select stateduration(c1,'GT',1,1s) c1 from t1)") + tdSql.checkRows(12) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, -1) + tdSql.checkData(2, 0, 0) + tdSql.checkData(10, 0, 63072035) + + tdSql.query("select sum(c1) from (select stateduration(c1,'GT',1,1d) c1 from t1)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 2893) + + tdSql.query("select sum(c1) from (select distinct(c1) c1 from ct1) union all select sum(c1) from (select stateduration(c1,'GT',1,1s) c1 from ct1)") + tdSql.checkRows(2) + + tdSql.query("select 1-abs(c1) from (select stateduration(c1,'GT',1,1s) c1 from t1)") + tdSql.checkRows(12) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 0.000000000) + tdSql.checkData(3, 0, -86404.000000000) + + + # bug for stable + #partition by tbname + # tdSql.query(" select unique(c1) from stb1 partition by tbname ") + # tdSql.checkRows(21) + + # tdSql.query(" select unique(c1) from stb1 partition by tbname ") + # tdSql.checkRows(21) + + # group by + tdSql.error("select stateduration(c1,'GT',1,1s) from ct1 group by c1") + tdSql.error("select stateduration(c1,'GT',1,1s) from ct1 group by tbname") + + # super table + + def check_unit_time(self): + tdSql.execute(" use db ") + tdSql.error("select stateduration(c1,'GT',1,1b) from ct1") + tdSql.error("select stateduration(c1,'GT',1,1u) from ct1") + tdSql.query("select stateduration(c1,'GT',1,1s) from t1") + tdSql.checkData(10,0,63072035) + tdSql.query("select stateduration(c1,'GT',1,1000s) from t1") + tdSql.checkData(10,0,int(63072035/1000)) + tdSql.query("select stateduration(c1,'GT',1,1m) from t1") + tdSql.checkData(10,0,int(63072035/60)) + tdSql.query("select stateduration(c1,'GT',1,1h) from t1") + tdSql.checkData(10,0,int(63072035/60/60)) + tdSql.query("select stateduration(c1,'GT',1,1d) from t1") + tdSql.checkData(10,0,int(63072035/60/24/60)) + tdSql.query("select stateduration(c1,'GT',1,1w) from t1") + tdSql.checkData(10,0,int(63072035/60/7/24/60)) + + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.query("select stateduration(c1,'GT',1,1s) from sub1_bound") + tdSql.checkRows(5) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4:support opers ============") + self.support_opers() + + tdLog.printNoPrefix("==========step5: stateduration basic query ============") + + self.basic_stateduration_function() + + tdLog.printNoPrefix("==========step6: stateduration boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: stateduration unit time test ============") + + self.check_unit_time() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 7095efa5eed2f9ad7648374897c0efc4b9fb7c7a Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 31 May 2022 20:36:14 +0800 Subject: [PATCH 19/57] add test case for statecount --- tests/system-test/2-query/statecount.py | 431 ++++++++++++++++++++++++ tests/system-test/fulltest.sh | 2 + 2 files changed, 433 insertions(+) create mode 100644 tests/system-test/2-query/statecount.py diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py new file mode 100644 index 0000000000..2634d9a9ab --- /dev/null +++ b/tests/system-test/2-query/statecount.py @@ -0,0 +1,431 @@ +from math import floor +from random import randint, random +from numpy import equal +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def test_errors(self): + error_sql_lists = [ + # "select statecount(c1,'GT',5) from t1" + "select statecount from t1", + "select statecount(123--123)==1 from t1", + "select statecount(123,123) from t1", + "select statecount(c1,ts) from t1", + "select statecount(c1,c1,ts) from t1", + "select statecount(c1 ,c2 ) from t1", + "select statecount(c1 ,NULL) from t1", + #"select statecount(c1 ,'NULL',1.0) from t1", + "select statecount(c1 ,'GT','1') from t1", + "select statecount(c1 ,'GT','tbname') from t1", + "select statecount(c1 ,'GT','*') from t1", + "select statecount(c1 ,'GT',ts) from t1", + "select statecount(c1 ,'GT',max(c1)) from t1", + "select statecount(abs(c1) ,'GT',1) from t1", + "select statecount(c1+2 ,'GT',1) from t1", + "select statecount(c1 ,'GT',1,1u) from t1", + "select statecount(c1 ,'GT',1,now) from t1", + "select statecount(c1 ,'GT','1') from t1", + "select statecount(c1 ,'GT','1',True) from t1", + "select statecount(statecount(c1) ab from t1)", + "select statecount(c1 ,'GT',1,,)int from t1", + "select statecount('c1','GT',1) from t1", + "select statecount('c1','GT' , NULL) from t1", + "select statecount('c1','GT', 1 , '') from t1", + "select statecount('c1','GT', 1 ,c%) from t1", + "select statecount(c1 ,'GT',1,t1) from t1", + "select statecount(c1 ,'GT',1,True) from t1", + "select statecount(c1 ,'GT',1) , count(c1) from t1", + "select statecount(c1 ,'GT',1) , avg(c1) from t1", + "select statecount(c1 ,'GT',1) , min(c1) from t1", + "select statecount(c1 ,'GT',1) , spread(c1) from t1", + "select statecount(c1 ,'GT',1) , diff(c1) from t1", + "select statecount(c1 ,'GT',1) , abs(c1) from t1", + "select statecount(c1 ,'GT',1) , c1 from t1", + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + pass + + def support_types(self): + other_no_value_types = [ + "select statecount(ts,'GT',1) from t1" , + "select statecount(c7,'GT',1) from t1", + "select statecount(c8,'GT',1) from t1", + "select statecount(c9,'GT',1) from t1", + "select statecount(ts,'GT',1) from ct1" , + "select statecount(c7,'GT',1) from ct1", + "select statecount(c8,'GT',1) from ct1", + "select statecount(c9,'GT',1) from ct1", + "select statecount(ts,'GT',1) from ct3" , + "select statecount(c7,'GT',1) from ct3", + "select statecount(c8,'GT',1) from ct3", + "select statecount(c9,'GT',1) from ct3", + "select statecount(ts,'GT',1) from ct4" , + "select statecount(c7,'GT',1) from ct4", + "select statecount(c8,'GT',1) from ct4", + "select statecount(c9,'GT',1) from ct4", + "select statecount(ts,'GT',1) from stb1 partition by tbname" , + "select statecount(c7,'GT',1) from stb1 partition by tbname", + "select statecount(c8,'GT',1) from stb1 partition by tbname", + "select statecount(c9,'GT',1) from stb1 partition by tbname" + ] + + for type_sql in other_no_value_types: + tdSql.error(type_sql) + tdLog.info("support type ok , sql is : %s"%type_sql) + + type_sql_lists = [ + "select statecount(c1,'GT',1) from t1", + "select statecount(c2,'GT',1) from t1", + "select statecount(c3,'GT',1) from t1", + "select statecount(c4,'GT',1) from t1", + "select statecount(c5,'GT',1) from t1", + "select statecount(c6,'GT',1) from t1", + + "select statecount(c1,'GT',1) from ct1", + "select statecount(c2,'GT',1) from ct1", + "select statecount(c3,'GT',1) from ct1", + "select statecount(c4,'GT',1) from ct1", + "select statecount(c5,'GT',1) from ct1", + "select statecount(c6,'GT',1) from ct1", + + "select statecount(c1,'GT',1) from ct3", + "select statecount(c2,'GT',1) from ct3", + "select statecount(c3,'GT',1) from ct3", + "select statecount(c4,'GT',1) from ct3", + "select statecount(c5,'GT',1) from ct3", + "select statecount(c6,'GT',1) from ct3", + + "select statecount(c1,'GT',1) from stb1 partition by tbname", + "select statecount(c2,'GT',1) from stb1 partition by tbname", + "select statecount(c3,'GT',1) from stb1 partition by tbname", + "select statecount(c4,'GT',1) from stb1 partition by tbname", + "select statecount(c5,'GT',1) from stb1 partition by tbname", + "select statecount(c6,'GT',1) from stb1 partition by tbname", + + "select statecount(c6,'GT',1) as alisb from stb1 partition by tbname", + "select statecount(c6,'GT',1) alisb from stb1 partition by tbname", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def support_opers(self): + oper_lists = ['LT','lt','Lt','lT','GT','gt','Gt','gT','LE','le','Le','lE','GE','ge','Ge','gE','NE','ne','Ne','nE','EQ','eq','Eq','eQ'] + + oper_errors = [",","*","NULL","tbname","ts","sum","_c0"] + + for oper in oper_lists: + tdSql.query(f"select statecount(c1 ,'{oper}',1) as col from t1") + tdSql.checkRows(12) + + for oper in oper_errors: + tdSql.error(f"select statecount(c1 ,'{oper}',1) as col from t1") + + + def basic_statecount_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select statecount(c6,'GT',1) from ct3") + tdSql.checkRows(0) + tdSql.query("select statecount(c6,'GT',1) from ct3") + tdSql.checkRows(0) + tdSql.query("select statecount(c6,'GT',1) from ct3") + tdSql.checkRows(0) + tdSql.query("select statecount(c6,'GT',1) from ct3") + tdSql.checkRows(0) + tdSql.query("select statecount(c6,'GT',1) from ct3") + tdSql.checkRows(0) + tdSql.query("select statecount(c6,'GT',1) from ct3") + + # will support _rowts mix with + # tdSql.query("select (c6,'GT',1),_rowts from ct3") + + # auto check for t1 table + # used for regular table + tdSql.query("select statecount(c6,'GT',1) from t1") + + # unique with super tags + + tdSql.query("select statecount(c6,'GT',1) from ct1") + tdSql.checkRows(13) + + tdSql.query("select statecount(c6,'GT',1) from ct4") + tdSql.checkRows(12) + + tdSql.error("select statecount(c6,'GT',1),tbname from ct1") + tdSql.error("select statecount(c6,'GT',1),t1 from ct1") + + # unique with common col + tdSql.error("select statecount(c6,'GT',1) ,ts from ct1") + tdSql.error("select statecount(c6,'GT',1) ,c1 from ct1") + + # unique with scalar function + tdSql.error("select statecount(c6,'GT',1) ,abs(c1) from ct1") + tdSql.error("select statecount(c6,'GT',1) , unique(c2) from ct1") + tdSql.error("select statecount(c6,'GT',1) , abs(c2)+2 from ct1") + + + # unique with aggregate function + tdSql.error("select statecount(c6,'GT',1) ,sum(c1) from ct1") + tdSql.error("select statecount(c6,'GT',1) ,max(c1) from ct1") + tdSql.error("select statecount(c6,'GT',1) ,csum(c1) from ct1") + tdSql.error("select statecount(c6,'GT',1) ,count(c1) from ct1") + + # unique with filter where + tdSql.query("select statecount(c6,'GT',1) from ct4 where c1 is null") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + + tdSql.query("select statecount(c1,'GT',1) from t1 where c1 >2 ") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 3) + tdSql.checkData(4, 0, 5) + tdSql.checkData(5, 0, 6) + + tdSql.query("select statecount(c2,'GT',1) from t1 where c2 between 0 and 99999") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 2) + tdSql.checkData(6, 0, -1) + + + # unique with union all + tdSql.query("select statecount(c1,'GT',1) from ct4 union all select statecount(c1,'GT',1) from ct1") + tdSql.checkRows(25) + tdSql.query("select statecount(c1,'GT',1) from ct4 union all select distinct(c1) from ct4") + tdSql.checkRows(22) + + # unique with join + # prepare join datas with same ts + + tdSql.execute(" use db ") + tdSql.execute(" create stable st1 (ts timestamp , num int) tags(ind int)") + tdSql.execute(" create table tb1 using st1 tags(1)") + tdSql.execute(" create table tb2 using st1 tags(2)") + + tdSql.execute(" create stable st2 (ts timestamp , num int) tags(ind int)") + tdSql.execute(" create table ttb1 using st2 tags(1)") + tdSql.execute(" create table ttb2 using st2 tags(2)") + + start_ts = 1622369635000 # 2021-05-30 18:13:55 + + for i in range(10): + ts_value = start_ts+i*1000 + tdSql.execute(f" insert into tb1 values({ts_value} , {i})") + tdSql.execute(f" insert into tb2 values({ts_value} , {i})") + + tdSql.execute(f" insert into ttb1 values({ts_value} , {i})") + tdSql.execute(f" insert into ttb2 values({ts_value} , {i})") + + tdSql.query("select statecount(tb1.num,'GT',1) from tb1, tb2 where tb1.ts=tb2.ts ") + tdSql.checkRows(10) + tdSql.checkData(0,0,-1) + tdSql.checkData(1,0,-1) + tdSql.checkData(2,0,1) + tdSql.checkData(9,0,8) + + tdSql.query("select statecount(tb1.num,'GT',1) from tb1, tb2 where tb1.ts=tb2.ts union all select statecount(tb2.num,'GT',1) from tb1, tb2 where tb1.ts=tb2.ts ") + tdSql.checkRows(20) + + # nest query + # tdSql.query("select unique(c1) from (select c1 from ct1)") + tdSql.query("select c1 from (select statecount(c1,'GT',1) c1 from t1)") + tdSql.checkRows(12) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, -1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(10, 0, 8) + + tdSql.query("select sum(c1) from (select statecount(c1,'GT',1) c1 from t1)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 35) + + tdSql.query("select sum(c1) from (select distinct(c1) c1 from ct1) union all select sum(c1) from (select statecount(c1,'GT',1) c1 from ct1)") + tdSql.checkRows(2) + + tdSql.query("select 1-abs(c1) from (select statecount(c1,'GT',1) c1 from t1)") + tdSql.checkRows(12) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 0.000000000) + tdSql.checkData(3, 0, -1.000000000) + + + # bug for stable + #partition by tbname + # tdSql.query(" select unique(c1) from stb1 partition by tbname ") + # tdSql.checkRows(21) + + # tdSql.query(" select unique(c1) from stb1 partition by tbname ") + # tdSql.checkRows(21) + + # group by + tdSql.query("select statecount(c1,'GT',1) from ct1 group by c1") + tdSql.error("select statecount(c1,'GT',1) from ct1 group by tbname") + + # super table + + def check_unit_time(self): + tdSql.execute(" use db ") + tdSql.error("select stateduration(c1,'GT',1,1b) from ct1") + tdSql.error("select stateduration(c1,'GT',1,1u) from ct1") + tdSql.query("select stateduration(c1,'GT',1,1s) from t1") + tdSql.checkData(10,0,63072035) + tdSql.query("select stateduration(c1,'GT',1,1000s) from t1") + tdSql.checkData(10,0,int(63072035/1000)) + tdSql.query("select stateduration(c1,'GT',1,1m) from t1") + tdSql.checkData(10,0,int(63072035/60)) + tdSql.query("select stateduration(c1,'GT',1,1h) from t1") + tdSql.checkData(10,0,int(63072035/60/60)) + tdSql.query("select stateduration(c1,'GT',1,1d) from t1") + tdSql.checkData(10,0,int(63072035/60/24/60)) + tdSql.query("select stateduration(c1,'GT',1,1w) from t1") + tdSql.checkData(10,0,int(63072035/60/7/24/60)) + + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.query("select statecount(c1,'GT',1) from sub1_bound") + tdSql.checkRows(5) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4:support opers ============") + self.support_opers() + + tdLog.printNoPrefix("==========step5: statecount basic query ============") + + self.basic_statecount_function() + + tdLog.printNoPrefix("==========step6: statecount boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: statecount unit time test ============") + + self.check_unit_time() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index a3770b381c..0adbc366b2 100644 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -85,6 +85,8 @@ python3 ./test.py -f 2-query/sample.py python3 ./test.py -f 2-query/function_diff.py python3 ./test.py -f 2-query/unique.py python3 ./test.py -f 2-query/stateduration.py +python3 ./test.py -f 2-query/function_stateduration.py +python3 ./test.py -f 2-query/statecount.py python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py From 5f74daa5e44bbbbf0cc188ff5f0ec3449256b2bd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 31 May 2022 20:36:54 +0800 Subject: [PATCH 20/57] enh: display the ertransaction error reason --- include/util/tdef.h | 3 +- source/common/src/systable.c | 1 - source/dnode/mnode/impl/inc/mndDef.h | 6 ++- source/dnode/mnode/impl/src/mndTrans.c | 55 ++++++++++++++++++++------ tests/script/tsim/trans/create_db.sim | 18 +-------- 5 files changed, 48 insertions(+), 35 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 839fd0b263..9e8ed7cea3 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -253,8 +253,7 @@ typedef enum ELogicConditionType { #define TSDB_TRANS_STAGE_LEN 12 #define TSDB_TRANS_TYPE_LEN 16 -#define TSDB_TRANS_ERROR_LEN 64 -#define TSDB_TRANS_DESC_LEN 128 +#define TSDB_TRANS_ERROR_LEN 512 #define TSDB_STEP_NAME_LEN 32 #define TSDB_STEP_DESC_LEN 128 diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 38a6bafe9a..8207ffb22f 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -215,7 +215,6 @@ static const SSysDbTableSchema transSchema[] = { {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "stage", .bytes = TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "type", .bytes = TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "failed_times", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "last_exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "last_error", .bytes = (TSDB_TRANS_ERROR_LEN - 1) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 5e889c91f8..e7b87ebda1 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -119,9 +119,11 @@ typedef struct { SArray* commitActions; int64_t createdTime; int64_t lastExecTime; + int32_t lastErrorAction; + int32_t lastErrorNo; + tmsg_t lastErrorMsgType; + SEpSet lastErrorEpset; char dbname[TSDB_DB_FNAME_LEN]; - char lastError[TSDB_TRANS_ERROR_LEN]; - char desc[TSDB_TRANS_DESC_LEN]; int32_t startFunc; int32_t stopFunc; int32_t paramLen; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 84a1308271..02a30a91d3 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -813,9 +813,6 @@ void mndTransProcessRsp(SRpcMsg *pRsp) { if (pAction != NULL) { pAction->msgReceived = 1; pAction->errCode = pRsp->code; - if (pAction->errCode != 0) { - tstrncpy(pTrans->lastError, tstrerror(pAction->errCode), TSDB_TRANS_ERROR_LEN); - } } mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x", transId, mndTransStr(pAction->stage), action, @@ -924,24 +921,36 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA return -1; } - int32_t numOfExecuted = 0; - int32_t errCode = 0; + int32_t numOfExecuted = 0; + int32_t errCode = 0; + STransAction *pErrAction = NULL; for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pArray, action); if (pAction->msgReceived || pAction->rawWritten) { numOfExecuted++; if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { errCode = pAction->errCode; + pErrAction = pAction; } } } if (numOfExecuted == numOfActions) { if (errCode == 0) { + pTrans->lastErrorAction = 0; + pTrans->lastErrorNo = 0; + pTrans->lastErrorMsgType = 0; + memset(&pTrans->lastErrorEpset, 0, sizeof(pTrans->lastErrorEpset)); mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions); return 0; } else { mError("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode & 0XFFFF); + if (pErrAction != NULL) { + pTrans->lastErrorMsgType = pErrAction->msgType; + pTrans->lastErrorAction = pErrAction->id; + pTrans->lastErrorNo = pErrAction->errCode; + pTrans->lastErrorEpset = pErrAction->epSet; + } mndTransResetActions(pMnode, pTrans, pArray); terrno = errCode; return errCode; @@ -976,7 +985,7 @@ static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans) { return code; } -static int32_t mndTransExecuteRedoActionsNoParallel(SMnode *pMnode, STrans *pTrans) { +static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; @@ -1003,6 +1012,18 @@ static int32_t mndTransExecuteRedoActionsNoParallel(SMnode *pMnode, STrans *pTra } } + if (code == 0) { + pTrans->lastErrorAction = 0; + pTrans->lastErrorNo = 0; + pTrans->lastErrorMsgType = 0; + memset(&pTrans->lastErrorEpset, 0, sizeof(pTrans->lastErrorEpset)); + } else { + pTrans->lastErrorMsgType = pAction->msgType; + pTrans->lastErrorAction = action; + pTrans->lastErrorNo = pAction->errCode; + pTrans->lastErrorEpset = pAction->epSet; + } + if (code == 0) { pTrans->redoActionPos++; mDebug("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage), @@ -1037,7 +1058,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; if (pTrans->exec == TRN_EXEC_SERIAL) { - code = mndTransExecuteRedoActionsNoParallel(pMnode, pTrans); + code = mndTransExecuteRedoActionsSerial(pMnode, pTrans); } else { code = mndTransExecuteRedoActions(pMnode, pTrans); } @@ -1347,11 +1368,6 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)dbname, false); - char type[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(type, "todo", pShow->pMeta->pSchemas[cols].bytes); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)type, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->failedTimes, false); @@ -1359,7 +1375,20 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->lastExecTime, false); char lastError[TSDB_TRANS_ERROR_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(lastError, pTrans->lastError, pShow->pMeta->pSchemas[cols].bytes); + char detail[TSDB_TRANS_ERROR_LEN] = {0}; + if (pTrans->lastErrorNo != 0) { + int32_t len = snprintf(detail, sizeof(detail), "action:%d errno:0x%x(%s) ", pTrans->lastErrorAction, + pTrans->lastErrorNo & 0xFFFF, tstrerror(pTrans->lastErrorNo)); + SEpSet epset = pTrans->lastErrorEpset; + if (epset.numOfEps > 0) { + len += snprintf(detail + len, sizeof(detail) - len, "msgType:%s numOfEps:%d inUse:%d ", + TMSG_INFO(pTrans->lastErrorMsgType), epset.numOfEps, epset.inUse); + } + for (int32_t i = 0; i < pTrans->lastErrorEpset.numOfEps; ++i) { + len += snprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port); + } + } + STR_WITH_MAXSIZE_TO_VARSTR(lastError, detail, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)lastError, false); diff --git a/tests/script/tsim/trans/create_db.sim b/tests/script/tsim/trans/create_db.sim index 14415c7193..158a6b9f92 100644 --- a/tests/script/tsim/trans/create_db.sim +++ b/tests/script/tsim/trans/create_db.sim @@ -76,14 +76,6 @@ if $data[0][3] != d1 then return -1 endi -if $data[0][4] != todo then - return -1 -endi - -if $data[0][7] != @Unable to establish connection@ then - return -1 -endi - sql_error create database d1 vgroups 2; print =============== start dnode2 @@ -125,15 +117,7 @@ endi if $data[0][3] != d2 then return -1 endi - -if $data[0][4] != todo then - return -1 -endi - -if $data[0][7] != @Unable to establish connection@ then - return -1 -endi - +return sql_error create database d2 vgroups 2; print =============== kill transaction From 719cebdd87d0904dd57cbf6297ce8789324cb974 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 31 May 2022 21:03:47 +0800 Subject: [PATCH 21/57] stmt new api --- include/client/taos.h | 10 +++ include/libs/parser/parser.h | 4 +- source/client/inc/clientStmt.h | 3 + source/client/src/clientMain.c | 41 ++++++++++ source/client/src/clientStmt.c | 121 +++++++++++++++++++++++++++-- source/libs/parser/src/parInsert.c | 23 +++--- 6 files changed, 184 insertions(+), 18 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 0b8c67aa79..94e6221b31 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -85,6 +85,14 @@ typedef struct taosField { int32_t bytes; } TAOS_FIELD; +typedef struct TAOS_FIELD_E { + char name[65]; + int8_t type; + uint8_t precision; + uint8_t scale; + int32_t bytes; +} TAOS_FIELD_E; + #ifdef WINDOWS #define DLL_EXPORT __declspec(dllexport) #else @@ -135,6 +143,8 @@ DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsign DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags); DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int* fieldNum, TAOS_FIELD_E** fields); +DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int* fieldNum, TAOS_FIELD_E** fields); DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 06272b8151..ca825b9e2f 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -77,8 +77,8 @@ int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); -int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields); -int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields); +int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields); +int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tName, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); void destroyBoundColumnInfo(void* pBoundInfo); diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index f0c9dcd67d..936fb92fc4 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -116,8 +116,11 @@ int stmtAffectedRowsOnce(TAOS_STMT *stmt); int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length); int stmtSetTbName(TAOS_STMT *stmt, const char *tbName); int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags); +int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields); +int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields); int stmtIsInsert(TAOS_STMT *stmt, int *insert); int stmtGetParamNum(TAOS_STMT *stmt, int *nums); +int stmtGetParam(TAOS_STMT *stmt, int idx, int *type, int *bytes); int stmtAddBatch(TAOS_STMT *stmt); TAOS_RES *stmtUseResult(TAOS_STMT *stmt); int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 53eb443b36..e144885e9e 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -666,8 +666,39 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) { return stmtSetTbName(stmt, name); } +int taos_stmt_set_tags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) { + if (stmt == NULL || tags == NULL) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtSetTbTags(stmt, tags); +} + + int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { return taos_stmt_set_tbname(stmt, name); } +int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int* fieldNum, TAOS_FIELD_E** fields) { + if (stmt == NULL || NULL == fieldNum) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtGetTagFields(stmt, fieldNum, fields); +} + +int taos_stmt_get_col_fields(TAOS_STMT *stmt, int* fieldNum, TAOS_FIELD_E** fields) { + if (stmt == NULL || NULL == fieldNum) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtGetColFields(stmt, fieldNum, fields); +} + int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); @@ -772,6 +803,16 @@ int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) { return stmtGetParamNum(stmt, nums); } +int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes) { + if (stmt == NULL || type == NULL || NULL == bytes || idx < 0) { + tscError("invalid parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtGetParam(stmt, idx, type, bytes); +} + TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 01d785ef73..3adb3684da 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -17,7 +17,7 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { } break; case STMT_SETTAGS: - if (STMT_STATUS_NE(SETTBNAME)) { + if (STMT_STATUS_NE(SETTBNAME) && STMT_STATUS_NE(FETCH_FIELDS)) { code = TSDB_CODE_TSC_STMT_API_ERROR; } break; @@ -540,6 +540,8 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { if (pStmt->bInfo.needParse) { strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; + + STMT_ERR_RET(stmtParseSql(pStmt)); } return TSDB_CODE_SUCCESS; @@ -550,10 +552,6 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); - if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); - } - if (pStmt->bInfo.inExecCache) { return TSDB_CODE_SUCCESS; } @@ -571,7 +569,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { return TSDB_CODE_SUCCESS; } -int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fields) { +int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) { if (STMT_TYPE_QUERY == pStmt->sql.type) { tscError("invalid operation to get query tag fileds"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); @@ -589,7 +587,7 @@ int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fiel return TSDB_CODE_SUCCESS; } -int32_t stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fields) { +int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) { if (STMT_TYPE_QUERY == pStmt->sql.type) { tscError("invalid operation to get query column fileds"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); @@ -852,6 +850,71 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) { return TSDB_CODE_SUCCESS; } +int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { + STscStmt* pStmt = (STscStmt*)stmt; + + if (STMT_TYPE_QUERY == pStmt->sql.type) { + STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); + } + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && + STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + + if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + } + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + STMT_ERR_RET(stmtFetchTagFields(stmt, nums, fields)); + + return TSDB_CODE_SUCCESS; +} + +int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { + STscStmt* pStmt = (STscStmt*)stmt; + + if (STMT_TYPE_QUERY == pStmt->sql.type) { + STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); + } + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && + STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + + if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + } + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + STMT_ERR_RET(stmtFetchColFields(stmt, nums, fields)); + + return TSDB_CODE_SUCCESS; +} + + int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { STscStmt* pStmt = (STscStmt*)stmt; @@ -884,6 +947,50 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { return TSDB_CODE_SUCCESS; } +int stmtGetParam(TAOS_STMT *stmt, int idx, int *type, int *bytes) { + STscStmt* pStmt = (STscStmt*)stmt; + + if (STMT_TYPE_QUERY == pStmt->sql.type) { + STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); + } + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && + STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + + if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + } + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + int32_t nums = 0; + TAOS_FIELD_E *pField = NULL; + STMT_ERR_RET(stmtFetchColFields(stmt, &nums, &pField)); + if (idx >= nums) { + tscError("idx %d is too big", idx); + taosMemoryFree(pField); + STMT_ERR_RET(TSDB_CODE_INVALID_PARA); + } + + *type = pField[idx].type; + *bytes = pField[idx].bytes; + + taosMemoryFree(pField); + + return TSDB_CODE_SUCCESS; +} + TAOS_RES* stmtUseResult(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 047c2d1504..a6e2473c4f 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1724,18 +1724,23 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu return TSDB_CODE_SUCCESS; } -int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD** fields) { +int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, uint8_t timePrec) { if (fields) { *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); if (NULL == *fields) { return TSDB_CODE_OUT_OF_MEMORY; } + SSchema* pSchema = &pSchema[boundInfo->boundColumns[0]]; + if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type) { + (*fields)[0].precision = timePrec; + } + for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i]]; - strcpy((*fields)[i].name, pTagSchema->name); - (*fields)[i].type = pTagSchema->type; - (*fields)[i].bytes = pTagSchema->bytes; + SSchema* pSchema = &pSchema[boundInfo->boundColumns[i]]; + strcpy((*fields)[i].name, pSchema->name); + (*fields)[i].type = pSchema->type; + (*fields)[i].bytes = pSchema->bytes; } } @@ -1744,7 +1749,7 @@ int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_ return TSDB_CODE_SUCCESS; } -int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields) { +int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields) { STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { @@ -1759,12 +1764,12 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields)); + CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields, 0)); return TSDB_CODE_SUCCESS; } -int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD** fields) { +int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fields) { STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); if (pDataBlock->boundColumnInfo.numOfBound <= 0) { @@ -1776,7 +1781,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD** fields return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields)); + CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields, pDataBlock->pTableMeta->tableInfo.precision)); return TSDB_CODE_SUCCESS; } From d95ec693c00f06215cc4625195a6f4fb72243901 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 21:05:43 +0800 Subject: [PATCH 22/57] feat:add new logic for new tag format --- include/common/tdataformat.h | 4 ++-- source/common/src/tdataformat.c | 3 +++ source/libs/parser/inc/parUtil.h | 2 +- source/libs/parser/src/parInsert.c | 16 +++++++--------- source/libs/parser/src/parTranslater.c | 11 +++++------ source/libs/parser/src/parUtil.c | 16 +++++++++++----- source/libs/scalar/test/scalar/scalarTests.cpp | 6 ++---- 7 files changed, 31 insertions(+), 27 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 38bbee1f1d..18932119ff 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -136,8 +136,8 @@ struct STagVal { }; #pragma pack(push, 1) -#define TD_TAG_JSON ((int8_t)0x80) // distinguish JSON string and JSON value with the highest bit -#define TD_TAG_LARGE ((int8_t)0x40) +#define TD_TAG_JSON ((int8_t)0x40) // distinguish JSON string and JSON value with the highest bit +#define TD_TAG_LARGE ((int8_t)0x20) struct STag { int8_t flags; int16_t len; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 722fa57bad..71842655ec 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -567,6 +567,9 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c case TSDB_DATA_TYPE_UBIGINT: printf("%s:%d type:%d vlen:%d, val:%" PRIu64 "\n", tag, ln, (int32_t)type, vlen, *(uint64_t *)val); break; + case TSDB_DATA_TYPE_NULL: + printf("%s:%d type:%d vlen:%d, val:%" PRIi8 "\n", tag, ln, (int32_t)type, vlen, *(int8_t *)val); + break; default: ASSERT(0); break; diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 35cab7db36..00a8767e0c 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -55,7 +55,7 @@ int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* errMsg); +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 05e5d46c7e..df1e597b86 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -959,6 +959,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint SToken sToken; bool isParseBindParam = false; bool isJson = false; + STag* pTag = NULL; for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); @@ -990,7 +991,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint taosMemoryFree(tmpTokenBuf); goto end; } - code = parseJsontoTagData(sToken.z, pTagVals, &pCxt->msg); + code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); taosMemoryFree(tmpTokenBuf); if(code != TSDB_CODE_SUCCESS){ goto end; @@ -1015,9 +1016,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint goto end; } - STag* pTag = NULL; - code = tTagNew(pTagVals, 1, isJson, &pTag); - if (code != TSDB_CODE_SUCCESS) { + if(!isJson && (code = tTagNew(pTagVals, 1, false, &pTag)) != TSDB_CODE_SUCCESS) { goto end; } @@ -1528,6 +1527,8 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN SSchema* pSchema = pDataBlock->pTableMeta->schema; bool isJson = false; + STag* pTag = NULL; + for (int c = 0; c < tags->numOfBound; ++c) { if (bind[c].is_null && bind[c].is_null[0]) { continue; @@ -1548,7 +1549,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN isJson = true; char *tmp = taosMemoryCalloc(1, colLen + 1); memcpy(tmp, bind[c].buffer, colLen); - code = parseJsontoTagData(tmp, pTagArray, &pBuf); + code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf); taosMemoryFree(tmp); if(code != TSDB_CODE_SUCCESS){ goto end; @@ -1586,10 +1587,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN } } - STag* pTag = NULL; - - if (0 != tTagNew(pTagArray, 1, isJson, &pTag)) { - code = buildInvalidOperationMsg(&pBuf, "out of memory"); + if (!isJson && (code = tTagNew(pTagArray, 1, false, &pTag)) != TSDB_CODE_SUCCESS) { goto end; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bd31c16f54..16c35fce84 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4218,7 +4218,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } isJson = true; - code = parseJsontoTagData(pVal->literal, pTagArray, &pCxt->msgBuf); + code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); if(code != TSDB_CODE_SUCCESS){ goto end; } @@ -4235,7 +4235,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } - code = tTagNew(pTagArray, 1, isJson, ppTag); + if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag); end: if(isJson){ @@ -4285,7 +4285,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } isJson = true; - code = parseJsontoTagData(pVal->literal, pTagArray, &pCxt->msgBuf); + code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); if(code != TSDB_CODE_SUCCESS){ goto end; } @@ -4302,7 +4302,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } ++index; } - code = tTagNew(pTagArray, 1, isJson, ppTag); + if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag); end: if(isJson){ @@ -4580,11 +4580,10 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS int32_t code = TSDB_CODE_SUCCESS; STag* pTag = NULL; do{ - code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pCxt->msgBuf); + code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); if (TSDB_CODE_SUCCESS != code) { break; } - code = tTagNew(pTagVals, 1, true, &pTag); }while(0); for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 726028355f..18fb5322b1 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -328,24 +328,27 @@ static bool isValidateTag(char* input) { return true; } -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* pMsgBuf) { +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf) { + int32_t retCode = TSDB_CODE_SUCCESS; // set json NULL data if (!json || strtrim((char*)json) == 0 || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) { - return TSDB_CODE_SUCCESS; + retCode = TSDB_CODE_SUCCESS; + goto end; } // set json real data cJSON* root = cJSON_Parse(json); if (root == NULL) { - return buildSyntaxErrMsg(pMsgBuf, "json parse error", json); + retCode = buildSyntaxErrMsg(pMsgBuf, "json parse error", json); + goto end; } int32_t size = cJSON_GetArraySize(root); if (!cJSON_IsObject(root)) { - return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); + retCode = buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); + goto end; } - int32_t retCode = TSDB_CODE_SUCCESS; SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); for (int32_t i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); @@ -413,6 +416,9 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* pMsgBuf) end: taosHashCleanup(keyHash); + if(retCode == TSDB_CODE_SUCCESS){ + tTagNew(pTagVals, 1, true, ppTag); + } cJSON_Delete(root); return retCode; } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 2081c6a3f0..6ddf4b12b9 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1112,9 +1112,8 @@ TEST(columnTest, json_column_arith_op) { char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); SArray *tags = taosArrayInit(1, sizeof(STagVal)); - parseJsontoTagData(rightv, tags, NULL); STag* row = NULL; - tTagNew(tags, 1, true, &row); + parseJsontoTagData(rightv, tags, &row, NULL); const int32_t len = 8; EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, @@ -1189,9 +1188,8 @@ TEST(columnTest, json_column_logic_op) { char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); SArray *tags = taosArrayInit(1, sizeof(STagVal)); - parseJsontoTagData(rightv, tags, NULL); STag* row = NULL; - tTagNew(tags, 1, true, &row); + parseJsontoTagData(rightv, tags, &row, NULL); const int32_t len = 9; const int32_t len1 = 4; From d8405d82dcbe87a23fbd8ba8766c2caca36bad1a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 31 May 2022 21:07:52 +0800 Subject: [PATCH 23/57] feat: tsma refactor --- include/common/tmsg.h | 1 + include/util/tdef.h | 2 +- include/util/tencode.h | 54 +++++++++++---------- source/common/src/tmsg.c | 11 +++++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 3 +- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndSma.c | 10 +++- source/dnode/mnode/impl/src/mndVgroup.c | 2 + source/dnode/vnode/inc/vnode.h | 3 +- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/vnd/vnodeCfg.c | 3 ++ source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- 12 files changed, 62 insertions(+), 32 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 12d1c85879..3f0752ea59 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1012,6 +1012,7 @@ typedef struct { // for tsma int8_t isTsma; + void* pTsma; } SCreateVnodeReq; diff --git a/include/util/tdef.h b/include/util/tdef.h index 1e1770d1b9..61cd335dea 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -342,7 +342,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF #define TSDB_MIN_ROLLUP_FILE_FACTOR 0 -#define TSDB_MAX_ROLLUP_FILE_FACTOR 1 +#define TSDB_MAX_ROLLUP_FILE_FACTOR 10 #define TSDB_DEFAULT_ROLLUP_FILE_FACTOR 0.1 #define TSDB_MIN_TABLE_TTL 0 #define TSDB_DEFAULT_TABLE_TTL 0 diff --git a/include/util/tencode.h b/include/util/tencode.h index cbacd59fa7..05e4c013c4 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -179,23 +179,23 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \ return 0; -#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \ - int32_t i = 0; \ - *(PVAL) = 0; \ - for (;;) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ - TYPE tval = TD_CODER_CURRENT(CODER)[0]; \ - if (tval < ENCODE_LIMIT) { \ - *(PVAL) |= (tval << (7 * i)); \ - TD_CODER_MOVE_POS(pCoder, 1); \ - break; \ - } else { \ - *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \ - i++; \ - TD_CODER_MOVE_POS(pCoder, 1); \ - } \ - } \ - \ +#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \ + int32_t i = 0; \ + if (PVAL) *(PVAL) = 0; \ + for (;;) { \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ + TYPE tval = TD_CODER_CURRENT(CODER)[0]; \ + if (tval < ENCODE_LIMIT) { \ + if (PVAL) *(PVAL) |= (tval << (7 * i)); \ + TD_CODER_MOVE_POS(pCoder, 1); \ + break; \ + } else { \ + if (PVAL) *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \ + i++; \ + TD_CODER_MOVE_POS(pCoder, 1); \ + } \ + } \ + \ return 0; // 8 @@ -378,14 +378,16 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { } static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { - if (tDecodeU32v(pCoder, len) < 0) return -1; + uint32_t length = 0; + if (tDecodeU32v(pCoder, &length) < 0) return -1; + if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; if (val) { *val = (uint8_t*)TD_CODER_CURRENT(pCoder); } - TD_CODER_MOVE_POS(pCoder, *len); + TD_CODER_MOVE_POS(pCoder, length); return 0; } @@ -410,14 +412,16 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { } static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) { - if (tDecodeU64v(pCoder, len) < 0) return -1; + uint64_t length = 0; + if (tDecodeU64v(pCoder, &length) < 0) return -1; + if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1; - *val = taosMemoryMalloc(*len); + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; + *val = taosMemoryMalloc(length); if (*val == NULL) return -1; - memcpy(*val, TD_CODER_CURRENT(pCoder), *len); + memcpy(*val, TD_CODER_CURRENT(pCoder), length); - TD_CODER_MOVE_POS(pCoder, *len); + TD_CODER_MOVE_POS(pCoder, length); return 0; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7615f7b070..05b90092e5 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2921,6 +2921,11 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR } if (tEncodeI8(&encoder, pReq->isTsma) < 0) return -1; + if (pReq->isTsma) { + uint32_t tsmaLen = (uint32_t)(htonl(((SMsgHead *)pReq->pTsma)->contLen)); + if (tEncodeBinary(&encoder, (const uint8_t *)pReq->pTsma, tsmaLen) < 0) return -1; + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2984,6 +2989,9 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * } if (tDecodeI8(&decoder, &pReq->isTsma) < 0) return -1; + if (pReq->isTsma) { + if (tDecodeBinaryAlloc(&decoder, &pReq->pTsma, NULL) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2993,6 +3001,9 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) { taosArrayDestroy(pReq->pRetensions); pReq->pRetensions = NULL; + if(pReq->isTsma) { + taosMemoryFreeClear(pReq->pTsma); + } return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 8374db129f..27ea19a5dc 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -140,6 +140,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->szCache = pCreate->pages; pCfg->szBuf = (uint64_t)pCreate->buffer * 1024 * 1024; pCfg->isWeak = true; + pCfg->isTsma = pCreate->isTsma; pCfg->tsdbCfg.compression = pCreate->compression; pCfg->tsdbCfg.precision = pCreate->precision; pCfg->tsdbCfg.days = pCreate->daysPerFile; @@ -209,7 +210,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { - dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); + dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); code = terrno; goto _OVER; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 2164c98c83..29a3992682 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -345,6 +345,7 @@ typedef struct { int8_t isTsma; int8_t replica; SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; + void* pTsma; } SVgObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 7b5d1b6c32..388b0d0c22 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -409,7 +409,8 @@ static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, + SSmaObj *pSma) { SVnodeGid *pVgid = pVgroup->vnodeGid + 0; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode == NULL) return -1; @@ -419,9 +420,14 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, mndReleaseDnode(pMnode, pDnode); // todo add sma info here + int32_t smaContLen = 0; + void *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen); + if (pSmaReq == NULL) return -1; + pVgroup->pTsma = pSmaReq; int32_t contLen = 0; void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + taosMemoryFreeClear(pSmaReq); if (pReq == NULL) return -1; action.pCont = pReq; @@ -514,7 +520,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; - if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg) != 0) goto _OVER; + if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, STREAM_TRIGGER_AT_ONCE, 0, pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e05b38a7c0..dc1d292830 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -218,6 +218,8 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.hashMethod = pDb->cfg.hashMethod; createReq.numOfRetensions = pDb->cfg.numOfRetensions; createReq.pRetensions = pDb->cfg.pRetensions; + createReq.isTsma = pVgroup->isTsma; + createReq.pTsma = pVgroup->pTsma; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e4343e3bbf..bc493135ac 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -159,12 +159,13 @@ struct SVnodeCfg { uint64_t szBuf; bool isHeap; bool isWeak; + int8_t isTsma; + int8_t hashMethod; STsdbCfg tsdbCfg; SWalCfg walCfg; SSyncCfg syncCfg; uint32_t hashBegin; uint32_t hashEnd; - int8_t hashMethod; }; typedef struct { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index a1e397f11d..42c215ee7c 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -414,7 +414,7 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 } taosMemoryFreeClear(pReq); } else { - smaWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno)); + smaDebug("vgId:%d no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno)); } taosArrayDestroy(pResult); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index a66ecc493d..56cc3d6374 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -56,6 +56,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; @@ -130,6 +131,8 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if(code < 0) return -1; tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code); if(code < 0) return -1; + tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code); + if(code < 0) return -1; tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code); if(code < 0) return -1; tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index a90bb7afcb..178ef28e5d 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -97,7 +97,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { } // open tsdb - if (!vnodeIsRollup(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_TYPE_TSDB) < 0) { + if (!vnodeIsRollup(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL) < 0) { vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } From c5120e68ba47d53258764ac21873ab4eff0c953e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 31 May 2022 21:15:45 +0800 Subject: [PATCH 24/57] enh: code optimization --- include/util/tencode.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index 05e4c013c4..ceef2a5980 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -181,16 +181,16 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); #define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \ int32_t i = 0; \ - if (PVAL) *(PVAL) = 0; \ + *(PVAL) = 0; \ for (;;) { \ if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ TYPE tval = TD_CODER_CURRENT(CODER)[0]; \ if (tval < ENCODE_LIMIT) { \ - if (PVAL) *(PVAL) |= (tval << (7 * i)); \ + *(PVAL) |= (tval << (7 * i)); \ TD_CODER_MOVE_POS(pCoder, 1); \ break; \ } else { \ - if (PVAL) *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \ + *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \ i++; \ TD_CODER_MOVE_POS(pCoder, 1); \ } \ From 1e7196037614a3b8e9273f2c4271fd45db387b09 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 31 May 2022 21:16:51 +0800 Subject: [PATCH 25/57] enh: format optimization --- include/util/tencode.h | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index ceef2a5980..ffd9ee53dd 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -179,23 +179,23 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \ return 0; -#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \ - int32_t i = 0; \ +#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \ + int32_t i = 0; \ *(PVAL) = 0; \ - for (;;) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ - TYPE tval = TD_CODER_CURRENT(CODER)[0]; \ - if (tval < ENCODE_LIMIT) { \ + for (;;) { \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ + TYPE tval = TD_CODER_CURRENT(CODER)[0]; \ + if (tval < ENCODE_LIMIT) { \ *(PVAL) |= (tval << (7 * i)); \ - TD_CODER_MOVE_POS(pCoder, 1); \ - break; \ - } else { \ + TD_CODER_MOVE_POS(pCoder, 1); \ + break; \ + } else { \ *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \ - i++; \ - TD_CODER_MOVE_POS(pCoder, 1); \ - } \ - } \ - \ + i++; \ + TD_CODER_MOVE_POS(pCoder, 1); \ + } \ + } \ + \ return 0; // 8 From 34d8d054498131695d1e3f61ff68b426a49b2d90 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 31 May 2022 09:31:30 -0400 Subject: [PATCH 26/57] fix(query): fix leastsquares c0+null and also group by output has empty result TD-16136 --- source/libs/function/src/builtinsimpl.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 8707946253..3f6f13a572 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1754,6 +1754,11 @@ int32_t leastSQRFunction(SqlFunctionCtx* pCtx) { } break; } + case TSDB_DATA_TYPE_NULL: { + GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElem = 1; + break; + } default: break; @@ -1797,7 +1802,7 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { size_t len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%.6lf, intercept:%.6lf}", param[0][2], param[1][2]); varDataSetLen(buf, len); - colDataAppend(pCol, currentRow, buf, false); + colDataAppend(pCol, currentRow, buf, pResInfo->isNullRes); return pResInfo->numOfRes; } From 4ba4f5391206329e560e9d159a4ecd517f106f30 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 31 May 2022 09:49:30 -0400 Subject: [PATCH 27/57] fix(query): fix leastsquares function param type error TD-16187 --- source/libs/function/src/builtinsimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3f6f13a572..a7e93246b7 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1646,8 +1646,8 @@ bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInf pInfo->startVal = IS_FLOAT_TYPE(pCtx->param[1].param.nType) ? pCtx->param[1].param.d : (double)pCtx->param[1].param.i; - pInfo->stepVal = IS_FLOAT_TYPE(pCtx->param[1].param.nType) ? pCtx->param[2].param.d : - (double)pCtx->param[1].param.i; + pInfo->stepVal = IS_FLOAT_TYPE(pCtx->param[2].param.nType) ? pCtx->param[2].param.d : + (double)pCtx->param[2].param.i; return true; } From 55c7d1c1e9430c73d8ac303730feddf0720384b9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 May 2022 22:17:18 +0800 Subject: [PATCH 28/57] fix: close cause flush result error --- source/libs/index/inc/indexCache.h | 1 + source/libs/index/src/index.c | 9 +++++++-- source/libs/index/src/indexCache.c | 7 +++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 1046a04db3..6e68163d74 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -36,6 +36,7 @@ typedef struct MemTable { typedef struct IndexCache { T_REF_DECLARE() MemTable *mem, *imm; + int32_t merging; SIndex* index; char* colName; int64_t version; diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 4814cc14f7..3d905303d1 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -462,7 +462,10 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { int64_t st = taosGetTimestampUs(); - IndexCache* pCache = (IndexCache*)cache; + IndexCache* pCache = (IndexCache*)cache; + + while (sIdx->quit && atomic_load_32(&pCache->merging) == 1) { + } TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); if (pReader == NULL) { indexWarn("empty tfile reader found"); @@ -475,9 +478,9 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { tfileReaderUnRef(pReader); if (sIdx->quit) { indexPost(sIdx); - // indexCacheBroadcast(pCache); } indexReleaseRef(sIdx->refId); + atomic_store_32(&pCache->merging, 0); return 0; } @@ -539,6 +542,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { if (sIdx->quit) { indexPost(sIdx); } + atomic_store_32(&pCache->merging, 0); indexReleaseRef(sIdx->refId); return ret; @@ -605,6 +609,7 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { taosThreadMutexLock(&tf->mtx); tfileCachePut(tf->cache, &key, reader); taosThreadMutexUnlock(&tf->mtx); + return ret; END: if (tw != NULL) { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 3b33006452..586a3ae573 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -494,16 +494,19 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { // TODO: wake up by condition variable indexCacheWait(cache); } else { - bool notifyQuit = cache->occupiedMem >= MEM_SIGNAL_QUIT ? true : false; + bool quit = cache->occupiedMem >= MEM_SIGNAL_QUIT ? true : false; indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); cache->mem->pCache = cache; cache->occupiedMem = 0; + if (quit == false) { + atomic_store_32(&cache->merging, 1); + } // sched to merge // unref cache in bgwork - indexCacheSchedToMerge(cache, notifyQuit); + indexCacheSchedToMerge(cache, quit); } } } From f6b700ff4581c49aaf740816b8b48cf4b237dbac Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 22:30:08 +0800 Subject: [PATCH 29/57] feat:add new logic for new tag format --- source/common/src/tdataformat.c | 2 +- source/libs/scalar/src/sclvector.c | 22 +++++++++++-------- .../libs/scalar/test/scalar/scalarTests.cpp | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 71842655ec..f946aa3b01 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -672,8 +672,8 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); } else { - n += tDataTypes[pTagVal->type].bytes; memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); + n += tDataTypes[pTagVal->type].bytes; } return n; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f84f564b8d..e844b3cdb6 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -922,12 +922,13 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } -STagVal *getJsonValue(char *json, STagVal *tagVal) { - bool find = tTagGet(((const STag *)json), tagVal); // json value is null and not exist is different - if(!find){ - return NULL; +STagVal getJsonValue(char *json, char *key, bool *isExist) { + STagVal val = {.pKey = key}; + bool find = tTagGet(((const STag *)json), &val); // json value is null and not exist is different + if(isExist){ + *isExist = find; } - return tagVal; + return val; } void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { @@ -939,6 +940,8 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); char *pRightData = colDataGetVarData(pRight->columnData, 0); + char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1); + memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData)); for (; i >= 0 && i < pLeft->numOfRows; i += step) { if (colDataIsNull_var(pLeft->columnData, i)) { colDataSetNull_var(pOutputCol, i); @@ -946,14 +949,15 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO continue; } char *pLeftData = colDataGetVarData(pLeft->columnData, i); - STagVal val = {.pKey = pRightData}; - STagVal *value = getJsonValue(pLeftData, &val); - char *data = tTagValToData(value, true); + bool isExist = false; + STagVal value = getJsonValue(pLeftData, jsonKey, &isExist); + char *data = isExist ? tTagValToData(&value, true) : NULL; colDataAppend(pOutputCol, i, data, data == NULL); - if(value && IS_VAR_DATA_TYPE(value->type) && data){ + if(isExist && IS_VAR_DATA_TYPE(value.type) && data){ taosMemoryFree(data); } } + taosMemoryFree(jsonKey); } void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 6ddf4b12b9..e971055a66 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -217,7 +217,7 @@ void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode SOperatorNode *onode = (SOperatorNode *)node; onode->node.resType.type = resType; onode->node.resType.bytes = tDataTypes[resType].bytes; - + onode->opType = opType; onode->pLeft = pLeft; onode->pRight = pRight; From 51b5e359333ec62ed294b9bdcb98f9e0f4429fa2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 23:38:21 +0800 Subject: [PATCH 30/57] feat:add new logic for new tag format --- include/common/tdataformat.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 16 +++++++++------- source/dnode/vnode/src/meta/metaTable.c | 14 ++++++++++---- source/libs/parser/src/parUtil.c | 9 ++++++--- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 18932119ff..53197d2d7b 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -120,6 +120,7 @@ struct SColVal { uint8_t *pData; }; +#pragma pack(push, 1) struct STagVal { union { int16_t cid; @@ -135,7 +136,6 @@ struct STagVal { }; }; -#pragma pack(push, 1) #define TD_TAG_JSON ((int8_t)0x40) // distinguish JSON string and JSON value with the highest bit #define TD_TAG_LARGE ((int8_t)0x20) struct STag { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index dead85f854..24aaeec406 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -567,14 +567,16 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { ASSERT(pEntry->type == TSDB_CHILD_TABLE); STag *tag = (STag *)pEntry->ctbEntry.pTags; - tTagGet(tag, val); + if (type == TSDB_DATA_TYPE_JSON){ + if(tag->nTag == 0){ + return NULL; + } + return tag; + } + bool find = tTagGet(tag, val); - if(val->type == TSDB_DATA_TYPE_NULL){ + if(!find){ return NULL; } - if (type == TSDB_DATA_TYPE_JSON){ - return tag; - }else{ - return val; - } + return val; } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 984688fc19..5ac13d478d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -787,10 +787,16 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; - STagVal tagVal = {.cid = pTagColumn->colId}; - tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; + if(pTagColumn->type != TSDB_DATA_TYPE_JSON){ + STagVal tagVal = {.cid = pTagColumn->colId}; + tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + }else{ + //pTagData = pCtbEntry->ctbEntry.pTags; + //nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; + } + // update tag index #ifdef USE_INVERTED_INDEX diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 18fb5322b1..dfd89a230d 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -330,6 +330,9 @@ static bool isValidateTag(char* input) { int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf) { int32_t retCode = TSDB_CODE_SUCCESS; + cJSON* root = NULL; + SHashObj* keyHash = NULL; + int32_t size = 0; // set json NULL data if (!json || strtrim((char*)json) == 0 || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) { retCode = TSDB_CODE_SUCCESS; @@ -337,19 +340,19 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMs } // set json real data - cJSON* root = cJSON_Parse(json); + root = cJSON_Parse(json); if (root == NULL) { retCode = buildSyntaxErrMsg(pMsgBuf, "json parse error", json); goto end; } - int32_t size = cJSON_GetArraySize(root); + size = cJSON_GetArraySize(root); if (!cJSON_IsObject(root)) { retCode = buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); goto end; } - SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); for (int32_t i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); if (!item) { From 1035312f1cc17018f29258d4a4b3cb7cb6d8435f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 09:25:45 +0800 Subject: [PATCH 31/57] refactor: make trans support db conflict --- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/inc/mndTrans.h | 2 +- source/dnode/mnode/impl/src/mndStb.c | 6 +-- source/dnode/mnode/impl/src/mndTrans.c | 66 +++++++++++++------------- 4 files changed, 39 insertions(+), 36 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e7b87ebda1..e284f52955 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -57,6 +57,7 @@ typedef enum { TRN_CONFLICT_NOTHING = 0, TRN_CONFLICT_GLOBAL = 1, TRN_CONFLICT_DB = 2, + TRN_CONFLICT_DB_INSIDE = 3, } ETrnConflct; typedef enum { diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index c70c6c4bd9..9b063fb44f 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -34,7 +34,7 @@ typedef struct { int32_t errCode; int32_t acceptableCode; int8_t stage; - int8_t isRaw; + int8_t actionType; // 0-msg, 1-raw int8_t rawWritten; int8_t msgSent; int8_t msgReceived; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9a8cabad92..9ca7613519 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -735,7 +735,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); @@ -1257,7 +1257,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p if (code != 0) goto _OVER; code = -1; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name); @@ -1403,7 +1403,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 02a30a91d3..90afec4d7d 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -88,12 +88,12 @@ static int32_t mndTransGetActionsSize(SArray *pArray) { for (int32_t i = 0; i < actionNum; ++i) { STransAction *pAction = taosArrayGet(pArray, i); - if (pAction->isRaw) { + if (pAction->actionType) { rawDataLen += (sdbGetRawTotalSize(pAction->pRaw) + sizeof(int32_t)); } else { rawDataLen += (sizeof(STransAction) + pAction->contLen); } - rawDataLen += sizeof(pAction->isRaw); + rawDataLen += sizeof(pAction->actionType); } return rawDataLen; @@ -135,9 +135,9 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->isRaw, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - if (pAction->isRaw) { + if (pAction->actionType) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) SDB_SET_INT32(pRaw, dataPos, len, _OVER) @@ -157,9 +157,9 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->isRaw, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - if (pAction->isRaw) { + if (pAction->actionType) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) SDB_SET_INT32(pRaw, dataPos, len, _OVER) @@ -179,9 +179,9 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->isRaw, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - if (pAction->isRaw) { + if (pAction->actionType) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) SDB_SET_INT32(pRaw, dataPos, len, _OVER) @@ -279,9 +279,9 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &action.isRaw, _OVER) + SDB_GET_INT8(pRaw, dataPos, &action.actionType, _OVER) SDB_GET_INT8(pRaw, dataPos, &action.stage, _OVER) - if (action.isRaw) { + if (action.actionType) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); @@ -308,9 +308,9 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &action.isRaw, _OVER) + SDB_GET_INT8(pRaw, dataPos, &action.actionType, _OVER) SDB_GET_INT8(pRaw, dataPos, &action.stage, _OVER) - if (action.isRaw) { + if (action.actionType) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); @@ -337,9 +337,9 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &action.isRaw, _OVER) + SDB_GET_INT8(pRaw, dataPos, &action.actionType, _OVER) SDB_GET_INT8(pRaw, dataPos, &action.stage, _OVER) - if (action.isRaw) { + if (action.actionType) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); @@ -552,7 +552,7 @@ static void mndTransDropActions(SArray *pArray) { int32_t size = taosArrayGetSize(pArray); for (int32_t i = 0; i < size; ++i) { STransAction *pAction = taosArrayGet(pArray, i); - if (pAction->isRaw) { + if (pAction->actionType) { taosMemoryFreeClear(pAction->pRaw); } else { taosMemoryFreeClear(pAction->pCont); @@ -583,17 +583,17 @@ static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) { } int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_REDO_ACTION, .isRaw = true, .pRaw = pRaw}; + STransAction action = {.stage = TRN_STAGE_REDO_ACTION, .actionType = true, .pRaw = pRaw}; return mndTransAppendAction(pTrans->redoActions, &action); } int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_UNDO_ACTION, .isRaw = true, .pRaw = pRaw}; + STransAction action = {.stage = TRN_STAGE_UNDO_ACTION, .actionType = true, .pRaw = pRaw}; return mndTransAppendAction(pTrans->undoActions, &action); } int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_COMMIT_ACTION, .isRaw = true, .pRaw = pRaw}; + STransAction action = {.stage = TRN_STAGE_COMMIT_ACTION, .actionType = true, .pRaw = pRaw}; return mndTransAppendAction(pTrans->commitActions, &action); } @@ -646,36 +646,38 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { return 0; } -static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNewTrans) { +static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { STrans *pTrans = NULL; void *pIter = NULL; bool conflict = false; - if (pNewTrans->conflict == TRN_CONFLICT_NOTHING) return conflict; + if (pNew->conflict == TRN_CONFLICT_NOTHING) return conflict; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); if (pIter == NULL) break; - if (pNewTrans->conflict == TRN_CONFLICT_GLOBAL) { - mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - conflict = true; - } else if (pNewTrans->conflict == TRN_CONFLICT_DB) { - if (pTrans->conflict == TRN_CONFLICT_GLOBAL || strcmp(pNewTrans->dbname, pTrans->dbname) == 0) { - mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - conflict = true; - } - } else { + if (pNew->conflict == TRN_CONFLICT_GLOBAL) conflict = true; + if (pNew->conflict == TRN_CONFLICT_DB) { + if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; + if (pTrans->conflict == TRN_CONFLICT_DB && strcmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; + if (pTrans->conflict == TRN_CONFLICT_DB_INSIDE && strcmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; + } + if (pNew->conflict == TRN_CONFLICT_DB_INSIDE) { + if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; + if (pTrans->conflict == TRN_CONFLICT_DB && strcmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; } - sdbRelease(pMnode->pSdb, pTrans); } + if (conflict) { + mError("trans:%d, can't execute since conflict with trans:%d, db:%s", pNew->id, pTrans->id, pTrans->dbname); + } return conflict; } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if (pTrans->conflict == TRN_CONFLICT_DB) { + if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { if (strlen(pTrans->dbname) == 0) { terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("trans:%d, failed to prepare conflict db not set", pTrans->id); @@ -893,7 +895,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio } static int32_t mndTransExecSingleAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { - if (pAction->isRaw) { + if (pAction->actionType) { return mndTransWriteSingleLog(pMnode, pTrans, pAction); } else { return mndTransSendSingleMsg(pMnode, pTrans, pAction); From 7c18d2f453165cf23d29fdacf25011ed389d0e93 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 1 Jun 2022 09:30:33 +0800 Subject: [PATCH 32/57] fix(os): delete two win ctest --- source/dnode/mnode/impl/test/user/CMakeLists.txt | 10 ++++++---- source/libs/scalar/test/scalar/CMakeLists.txt | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/test/user/CMakeLists.txt b/source/dnode/mnode/impl/test/user/CMakeLists.txt index b39ea0e73f..ed4d964617 100644 --- a/source/dnode/mnode/impl/test/user/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/user/CMakeLists.txt @@ -5,7 +5,9 @@ target_link_libraries( PUBLIC sut ) -add_test( - NAME userTest - COMMAND userTest -) +if(NOT TD_WINDOWS) + add_test( + NAME userTest + COMMAND userTest + ) +endif(NOT TD_WINDOWS) diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt index 86b936d93a..672cb5a3de 100644 --- a/source/libs/scalar/test/scalar/CMakeLists.txt +++ b/source/libs/scalar/test/scalar/CMakeLists.txt @@ -17,7 +17,9 @@ TARGET_INCLUDE_DIRECTORIES( PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc" ) -add_test( - NAME scalarTest - COMMAND scalarTest -) +if(NOT TD_WINDOWS) + add_test( + NAME scalarTest + COMMAND scalarTest + ) +endif(NOT TD_WINDOWS) From 8baf32fc23ab172ed04e7887bac448a37507af91 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 09:37:10 +0800 Subject: [PATCH 33/57] fix: the max number of mnodes is limited to 3 --- include/util/taoserror.h | 5 +++-- source/dnode/mnode/impl/src/mndMnode.c | 5 +++++ source/util/src/terror.c | 1 + tests/script/tsim/mnode/basic3.sim | 7 ++++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 65cfe8de0b..3669348a18 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -182,8 +182,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_BNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0356) #define TSDB_CODE_MND_BNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0357) #define TSDB_CODE_MND_TOO_FEW_MNODES TAOS_DEF_ERROR_CODE(0, 0x0358) -#define TSDB_CODE_MND_MNODE_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0359) -#define TSDB_CODE_MND_CANT_DROP_MASTER TAOS_DEF_ERROR_CODE(0, 0x035A) +#define TSDB_CODE_MND_TOO_MANY_MNODES TAOS_DEF_ERROR_CODE(0, 0x0359) +#define TSDB_CODE_MND_MNODE_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x035A) +#define TSDB_CODE_MND_CANT_DROP_MASTER TAOS_DEF_ERROR_CODE(0, 0x035B) // mnode-acct #define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 5b8ba6deaa..808eac82ac 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -396,6 +396,11 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { mDebug("mnode:%d, start to create", createReq.dnodeId); + if (sdbGetSize(pMnode->pSdb, SDB_MNODE) > 3) { + terrno = TSDB_CODE_MND_TOO_MANY_MNODES; + goto _OVER; + } + pObj = mndAcquireMnode(pMnode, createReq.dnodeId); if (pObj != NULL) { terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 575ae10bbb..a54d8c2c03 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -187,6 +187,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_NOT_EXIST, "Snode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_ALREADY_EXIST, "Bnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_NOT_EXIST, "Bnode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_FEW_MNODES, "Too few mnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "Too many mnodes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_DEPLOYED, "Mnode deployed in this dnode") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CANT_DROP_MASTER, "Can't drop mnode which is master") diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index b0ee23cd8c..ab3193e085 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -2,14 +2,17 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start sql connect print =============== step1: create dnodes sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 $x = 0 step1: @@ -32,6 +35,7 @@ endi print =============== step2: create mnode 2 sql create mnode on dnode 2 sql create mnode on dnode 3 +sql_error create mnode on dnode 4 $x = 0 step2: @@ -134,4 +138,5 @@ endi system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop -system sh/exec.sh -n dnode3 -s stop \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop +system sh/exec.sh -n dnode4 -s stop \ No newline at end of file From e610237e5bc2d6a6ba7300b0b07b4926f73706be Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 09:53:25 +0800 Subject: [PATCH 34/57] fix: the max number of mnodes is limited to 3 --- source/dnode/mnode/impl/src/mndMnode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 5f22b25c97..9eda87bdf1 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -18,9 +18,9 @@ #include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" +#include "mndSync.h" #include "mndTrans.h" #include "mndUser.h" -#include "mndSync.h" #define MNODE_VER_NUMBER 1 #define MNODE_RESERVE_SIZE 64 @@ -392,9 +392,9 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { mDebug("mnode:%d, start to create", createReq.dnodeId); - if (sdbGetSize(pMnode->pSdb, SDB_MNODE) > 3) { - terrno = TSDB_CODE_MND_TOO_MANY_MNODES; - goto _OVER; + if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) { + terrno = TSDB_CODE_MND_TOO_MANY_MNODES; + goto _OVER; } pObj = mndAcquireMnode(pMnode, createReq.dnodeId); From a6fe2f686217ecb96588f3d84ed41c053e1edca1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 09:55:25 +0800 Subject: [PATCH 35/57] refactor: make trans support db conflict --- source/dnode/mnode/impl/src/mndTrans.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 90afec4d7d..bad513a89d 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -667,12 +667,10 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_DB && strcmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; } + mError("trans:%d, can't execute since conflict with trans:%d, db:%s", pNew->id, pTrans->id, pTrans->dbname); sdbRelease(pMnode->pSdb, pTrans); } - if (conflict) { - mError("trans:%d, can't execute since conflict with trans:%d, db:%s", pNew->id, pTrans->id, pTrans->dbname); - } return conflict; } From 172834bc59c6ae1b3a4f2c3197588cbf6ef3a16e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 09:58:58 +0800 Subject: [PATCH 36/57] feat:fix conflict merge from 3.0 --- source/dnode/vnode/src/inc/meta.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 82 +++++++++++++++++++++++++ source/dnode/vnode/src/meta/metaTable.c | 2 +- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 9da603f894..52e576ee52 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -117,7 +117,7 @@ typedef struct { } SSmaIdxKey; // metaTable ================== -int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int8_t type, tb_uid_t uid, +int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey); #ifndef META_REFACT diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index c4c232ff34..65a61707a3 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -589,3 +589,85 @@ const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { } return val; } + +typedef struct { + SMeta * pMeta; + TBC * pCur; + tb_uid_t suid; + int16_t cid; + int16_t type; + void * pKey; + void * pVal; + int32_t kLen; + int32_t vLen; +} SIdxCursor; + +int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { + SIdxCursor *pCursor = NULL; + + char *tagData = param->val; + + int32_t ret = 0, valid = 0; + pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor)); + pCursor->pMeta = pMeta; + pCursor->suid = param->suid; + pCursor->cid = param->cid; + pCursor->type = param->type; + + metaRLock(pMeta); + ret = tdbTbcOpen(pMeta->pTagIdx, &pCursor->pCur, NULL); + if (ret < 0) { + goto END; + } + STagIdxKey *pKey = NULL; + int32_t nKey = 0; + + int32_t nTagData = 0; + if(IS_VAR_DATA_TYPE(param->type)){ + nTagData = strlen(param->val); + }else{ + nTagData = tDataTypes[param->type].bytes + } + ret = metaCreateTagIdxKey(pCursor->suid, pCursor->cid, param->val, nTagData, pCursor->type, + param->reverse ? INT64_MAX : INT64_MIN, &pKey, &nKey); + if (ret != 0) { + goto END; + } + int cmp = 0; + if (tdbTbcMoveTo(pCursor->pCur, pKey, nKey, &cmp) < 0) { + goto END; + } + void * entryKey = NULL, *entryVal = NULL; + int32_t nEntryKey, nEntryVal; + while (1) { + valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); + if (valid < 0) { + break; + } + STagIdxKey *p = entryKey; + if (p != NULL) { + int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); + if (cmp == 0) { + // match + tb_uid_t tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); + taosArrayPush(pUids, &tuid); + } else if (cmp == 1) { + // not match but should continue to iter + } else { + // not match and no more result + break; + } + } + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) { + break; + } + } +END: + if (pCursor->pMeta) metaULock(pCursor->pMeta); + if (pCursor->pCur) tdbTbcClose(pCursor->pCur); + + taosMemoryFree(pCursor); + + return ret; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0962de8b00..17ef6a270c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -731,7 +731,7 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } -static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, +int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { // int32_t nTagData = 0; From e10edb80ef320eeb51b7467ff22ba475272ddeaf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 10:21:27 +0800 Subject: [PATCH 37/57] fix: the role should be OFFLINE after mnode stopped --- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/dnode/mnode/impl/src/mndMnode.c | 8 +++++++- tests/script/tsim/mnode/basic3.sim | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9cec29eaa1..73d7eff04a 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -256,7 +256,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { int64_t interval = TABS(pDnode->lastAccessTime - curMs); - if (interval > 30000 * tsStatusInterval) { + if (interval > 5000 * tsStatusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 9eda87bdf1..4578d81efb 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -633,6 +633,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t cols = 0; SMnodeObj *pObj = NULL; char *pWrite; + int64_t curMs = taosGetTimestampMs(); while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj); @@ -648,11 +649,16 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); + bool online = mndIsDnodeOnline(pMnode, pObj->pDnode, curMs); const char *roles = NULL; if (pObj->id == pMnode->selfDnodeId) { roles = syncStr(TAOS_SYNC_STATE_LEADER); } else { - roles = syncStr(pObj->state); + if (!online) { + roles = "OFFLINE"; + } else { + roles = syncStr(pObj->state); + } } char *b2 = taosMemoryCalloc(1, 12 + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(b2, roles, pShow->pMeta->pSchemas[cols].bytes); diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index ab3193e085..3c69e6ed51 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -110,6 +110,10 @@ print $data(1)[0] $data(1)[1] $data(1)[2] print $data(2)[0] $data(2)[1] $data(2)[2] print $data(3)[0] $data(3)[1] $data(3)[2] +if $data(2)[2] != OFFLINE then + goto step5 +endi + sql show users if $rows != 2 then return -1 From 66899bd8d0d33b0ab8ff74f7ec9e571268531a57 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 1 Jun 2022 10:27:41 +0800 Subject: [PATCH 38/57] fix(tmq): auto unsubscribe when consumer close --- example/src/tmq.c | 41 ++---------- include/client/taos.h | 2 +- include/common/tmsg.h | 2 +- source/client/src/tmq.c | 34 +++++----- source/dnode/mnode/impl/src/mndMain.c | 6 +- source/dnode/vnode/src/inc/tq.h | 12 ++-- source/dnode/vnode/src/tq/tq.c | 89 ++++++++------------------- 7 files changed, 55 insertions(+), 131 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 7e4de21f2e..5eecb5e4cc 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -196,8 +196,9 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0); if (tmqmessage) { cnt++; + msg_process(tmqmessage); + if (cnt >= 2) break; /*printf("get data\n");*/ - /*msg_process(tmqmessage);*/ taos_free_result(tmqmessage); /*} else {*/ /*break;*/ @@ -253,39 +254,6 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { fprintf(stderr, "%% Consumer closed\n"); } -void perf_loop(tmq_t* tmq, tmq_list_t* topics) { - tmq_resp_err_t err; - - if ((err = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); - printf("subscribe err\n"); - return; - } - int32_t batchCnt = 0; - int32_t skipLogNum = 0; - clock_t startTime = clock(); - while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 500); - if (tmqmessage) { - batchCnt++; - /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/ - /*msg_process(tmqmessage);*/ - taos_free_result(tmqmessage); - } else { - break; - } - } - clock_t endTime = clock(); - printf("log batch cnt: %d, skip log cnt: %d, time used:%f s\n", batchCnt, skipLogNum, - (double)(endTime - startTime) / CLOCKS_PER_SEC); - - err = tmq_consumer_close(tmq); - if (err) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); - else - fprintf(stderr, "%% Consumer closed\n"); -} - int main(int argc, char* argv[]) { if (argc > 1) { printf("env init\n"); @@ -296,7 +264,6 @@ int main(int argc, char* argv[]) { } tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); - /*perf_loop(tmq, topic_list);*/ - /*basic_consume_loop(tmq, topic_list);*/ - sync_consume_loop(tmq, topic_list); + basic_consume_loop(tmq, topic_list); + /*sync_consume_loop(tmq, topic_list);*/ } diff --git a/include/client/taos.h b/include/client/taos.h index 0b8c67aa79..beeb3a104e 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -230,7 +230,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); -DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time); +DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 07605c735c..723d305ca5 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2412,7 +2412,7 @@ typedef struct { int32_t epoch; uint64_t reqId; int64_t consumerId; - int64_t waitTime; + int64_t timeout; int64_t currentOffset; } SMqPollReq; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index dfa56f80c4..416d1a6f26 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1243,7 +1243,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { return TMQ_RESP_ERR__FAIL; } -SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t waitTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { +SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { int64_t reqOffset; if (pVg->currentOffset >= 0) { reqOffset = pVg->currentOffset; @@ -1269,7 +1269,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t waitTime, SMqClientTopic* strcpy(pReq->subKey + tlen + 1, pTopic->topicName); pReq->withTbName = tmq->withTbName; - pReq->waitTime = waitTime; + pReq->timeout = timeout; pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; pReq->currentOffset = reqOffset; @@ -1297,7 +1297,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { return pRspObj; } -int32_t tmqPollImpl(tmq_t* tmq, int64_t waitTime) { +int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { /*printf("call poll\n");*/ for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); @@ -1318,7 +1318,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t waitTime) { #endif } atomic_store_32(&pVg->vgSkipCnt, 0); - SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, waitTime, pTopic, pVg); + SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, timeout, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); @@ -1388,7 +1388,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) return 0; } -SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t waitTime, bool pollIfReset) { +SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { while (1) { SMqRspWrapper* rspWrapper = NULL; taosGetQitem(tmq->qall, (void**)&rspWrapper); @@ -1428,17 +1428,17 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t waitTime, bool pollIfReset) { taosFreeQitem(rspWrapper); if (pollIfReset && reset) { tscDebug("consumer %ld reset and repoll", tmq->consumerId); - tmqPollImpl(tmq, waitTime); + tmqPollImpl(tmq, timeout); } } } } -TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) { +TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { SMqRspObj* rspObj; int64_t startTime = taosGetTimestampMs(); - rspObj = tmqHandleAllRsp(tmq, wait_time, false); + rspObj = tmqHandleAllRsp(tmq, timeout, false); if (rspObj) { return (TAOS_RES*)rspObj; } @@ -1450,16 +1450,16 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) { while (1) { tmqHandleAllDelayedTask(tmq); - if (tmqPollImpl(tmq, wait_time) < 0) return NULL; + if (tmqPollImpl(tmq, timeout) < 0) return NULL; - rspObj = tmqHandleAllRsp(tmq, wait_time, false); + rspObj = tmqHandleAllRsp(tmq, timeout, false); if (rspObj) { return (TAOS_RES*)rspObj; } - if (wait_time != 0) { + if (timeout != 0) { int64_t endTime = taosGetTimestampMs(); int64_t leftTime = endTime - startTime; - if (leftTime > wait_time) { + if (leftTime > timeout) { tscDebug("consumer %ld (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch); return NULL; } @@ -1474,10 +1474,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) { tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) { tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL); - if (rsp == TMQ_RESP_ERR__SUCCESS) { - // TODO: free resources - return TMQ_RESP_ERR__SUCCESS; - } else { + if (rsp == TMQ_RESP_ERR__FAIL) { return TMQ_RESP_ERR__FAIL; } @@ -1485,10 +1482,7 @@ tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); - if (rsp == TMQ_RESP_ERR__SUCCESS) { - // TODO: free resources - return TMQ_RESP_ERR__SUCCESS; - } else { + if (rsp == TMQ_RESP_ERR__FAIL) { return TMQ_RESP_ERR__FAIL; } } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 2a2a45a45d..3a3fd7ebdb 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -369,7 +369,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { mError("failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); return TAOS_SYNC_PROPOSE_OTHER_ERROR; } - + char logBuf[512] = {0}; char *syncNodeStr = sync2SimpleStr(pMgmt->sync); snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); @@ -472,7 +472,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { } else if (code == 0) { mTrace("msg:%p, successfully processed and response", pMsg); } else { - mDebug("msg:%p, failed to process since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle, + mError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); } @@ -686,4 +686,4 @@ void mndReleaseSyncRef(SMnode *pMnode) { int32_t ref = atomic_sub_fetch_32(&pMnode->syncRef, 1); mTrace("mnode sync is released, ref:%d", ref); taosThreadRwlockUnlock(&pMnode->lock); -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 72138926aa..34a7ff823a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -66,12 +66,12 @@ struct STqReadHandle { // tqPush typedef struct { - int64_t consumerId; - int32_t epoch; - int32_t skipLogNum; - int64_t reqOffset; - SRWLatch lock; - SRpcMsg* handle; + int64_t consumerId; + int32_t epoch; + int32_t skipLogNum; + int64_t reqOffset; + SRpcHandleInfo info; + SRWLatch lock; } STqPushHandle; #if 0 diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b4747f2264..4bce829b10 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -180,40 +180,6 @@ void tqClose(STQ* pTq) { // TODO } -#if 0 -int32_t tEncodeSTqExec(SEncoder* pEncoder, const STqExec* pExec) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeCStr(pEncoder, pExec->subKey) < 0) return -1; - if (tEncodeI64(pEncoder, pExec->consumerId) < 0) return -1; - if (tEncodeI32(pEncoder, pExec->epoch) < 0) return -1; - if (tEncodeI8(pEncoder, pExec->subType) < 0) return -1; - /*if (tEncodeI8(pEncoder, pExec->withTbName) < 0) return -1;*/ - /*if (tEncodeI8(pEncoder, pExec->withSchema) < 0) return -1;*/ - /*if (tEncodeI8(pEncoder, pExec->withTag) < 0) return -1;*/ - if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) { - if (tEncodeCStr(pEncoder, pExec->qmsg) < 0) return -1; - } - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tDecodeSTqExec(SDecoder* pDecoder, STqExec* pExec) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pExec->subKey) < 0) return -1; - if (tDecodeI64(pDecoder, &pExec->consumerId) < 0) return -1; - if (tDecodeI32(pDecoder, &pExec->epoch) < 0) return -1; - if (tDecodeI8(pDecoder, &pExec->subType) < 0) return -1; - /*if (tDecodeI8(pDecoder, &pExec->withTbName) < 0) return -1;*/ - /*if (tDecodeI8(pDecoder, &pExec->withSchema) < 0) return -1;*/ - /*if (tDecodeI8(pDecoder, &pExec->withTag) < 0) return -1;*/ - if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) { - if (tDecodeCStrAlloc(pDecoder, &pExec->qmsg) < 0) return -1; - } - tEndDecode(pDecoder); - return 0; -} -#endif - int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeCStr(pEncoder, pHandle->subKey) < 0) return -1; @@ -290,8 +256,8 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ taosWLockLatch(&pHandle->pushHandle.lock); - SRpcMsg* pMsg = atomic_load_ptr(&pHandle->pushHandle.handle); - ASSERT(pMsg); + /*SRpcHandleInfo* pInfo = atomic_load_ptr(&pHandle->pushHandle.pInfo);*/ + /*ASSERT(pInfo);*/ SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pHandle->pushHandle.reqOffset; @@ -318,7 +284,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); void* buf = rpcMallocCont(tlen); if (buf == NULL) { - pMsg->code = -1; + // todo free return -1; } @@ -329,10 +295,16 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqDataBlkRsp(&abuf, &rsp); - SRpcMsg resp = {.info = handleInfo, .pCont = buf, .contLen = tlen, .code = 0}; + SRpcMsg resp = { + .info = pHandle->pushHandle.info, + .pCont = buf, + .contLen = tlen, + .code = 0, + }; tmsgSendRsp(&resp); - atomic_store_ptr(&pHandle->pushHandle.handle, NULL); + /*atomic_store_ptr(&pHandle->pushHandle.pInfo, NULL);*/ + memset(&pHandle->pushHandle.info, 0, sizeof(SRpcHandleInfo)); taosWUnLockLatch(&pHandle->pushHandle.lock); tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", @@ -374,7 +346,7 @@ int tqCommit(STQ* pTq) { int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; - int64_t waitTime = pReq->waitTime; + int64_t waitTime = pReq->timeout; int32_t reqEpoch = pReq->epoch; int64_t fetchOffset; @@ -410,24 +382,22 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { rsp.blockData = taosArrayInit(0, sizeof(void*)); rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); - rsp.blockSchema = taosArrayInit(0, sizeof(void*)); - rsp.blockTbName = taosArrayInit(0, sizeof(void*)); rsp.withTbName = pReq->withTbName; + if (rsp.withTbName) { + rsp.blockTbName = taosArrayInit(0, sizeof(void*)); + } if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { rsp.withSchema = false; + rsp.withTag = false; } else { rsp.withSchema = true; + rsp.blockSchema = taosArrayInit(0, sizeof(void*)); + rsp.withTag = false; } - /*int8_t withTbName = pExec->withTbName;*/ - /*if (pReq->withTbName != -1) {*/ - /*withTbName = pReq->withTbName;*/ - /*}*/ - /*rsp.withTbName = withTbName;*/ - while (1) { consumerEpoch = atomic_load_32(&pHandle->epoch); if (consumerEpoch > reqEpoch) { @@ -443,15 +413,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SWalReadHead* pHead = &pHeadWithCkSum->head; -#if 0 - SWalReadHead* pHead; - if (walReadWithHandle_s(pExec->pWalReader, fetchOffset, &pHead) < 0) { - // TODO: no more log, set timer to wait blocking time - // if data inserted during waiting, launch query and - // response to user - tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), fetchOffset); - #if 0 // add to pushMgr taosWLockLatch(&pExec->pushHandle.lock); @@ -473,10 +434,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; #endif - break; - } -#endif - tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); @@ -533,8 +490,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO wrap in destroy func taosArrayDestroy(rsp.blockData); taosArrayDestroy(rsp.blockDataLen); - taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); - taosArrayDestroyP(rsp.blockTbName, (FDelete)taosMemoryFree); + + if (rsp.withSchema) { + taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + } + + if (rsp.withTbName) { + taosArrayDestroyP(rsp.blockTbName, (FDelete)taosMemoryFree); + } return 0; } From f99e66df3270ad18ba7ac96caa37a7ab60773b5b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 10:29:10 +0800 Subject: [PATCH 39/57] fix:fix error in new tag format --- source/dnode/vnode/src/meta/metaQuery.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 11 ++++++++--- source/libs/executor/src/scanoperator.c | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 65a61707a3..a57eb4e899 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -626,7 +626,7 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { if(IS_VAR_DATA_TYPE(param->type)){ nTagData = strlen(param->val); }else{ - nTagData = tDataTypes[param->type].bytes + nTagData = tDataTypes[param->type].bytes; } ret = metaCreateTagIdxKey(pCursor->suid, pCursor->cid, param->val, nTagData, pCursor->type, param->reverse ? INT64_MAX : INT64_MIN, &pKey, &nKey); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 17ef6a270c..9dd1543f9c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -787,11 +787,16 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + STagVal tagVal = {.cid = pTagColumn->colId}; if(pTagColumn->type != TSDB_DATA_TYPE_JSON){ - STagVal tagVal = {.cid = pTagColumn->colId}; tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; + if(IS_VAR_DATA_TYPE(pTagColumn->type)){ + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + }else{ + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } }else{ //pTagData = pCtbEntry->ctbEntry.pTags; //nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 75f125ff11..89340f2b85 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1594,7 +1594,8 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } colDataAppend(pDst, count, data, (data == NULL)); - if(pDst->info.type != TSDB_DATA_TYPE_JSON && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ + if(pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL + && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data != NULL){ taosMemoryFree(data); } } From 49b36c78367f5e6b0c3049450327c0c6b32ec859 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 1 Jun 2022 10:36:03 +0800 Subject: [PATCH 40/57] fix(os): win iconv find error --- source/os/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index b6e131d4cc..e15627fe66 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -10,7 +10,11 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/contrib/msvcregex" ) # iconv -find_path(IconvApiIncludes iconv.h PATHS) +if(TD_WINDOWS) + find_path(IconvApiIncludes iconv.h "${TD_SOURCE_DIR}/contrib/iconv") +else() + find_path(IconvApiIncludes iconv.h PATHS) +endif(TD_WINDOWS) if(NOT IconvApiIncludes) add_definitions(-DDISALLOW_NCHAR_WITHOUT_ICONV) endif () From 9879e69b2be7bd833314746bd81806d31f33c29e Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 1 Jun 2022 10:44:53 +0800 Subject: [PATCH 41/57] docs: reorganize and refine functions page in 3.0 --- docs-cn/12-taos-sql/07-function.md | 2446 ++++++++++------------------ 1 file changed, 905 insertions(+), 1541 deletions(-) diff --git a/docs-cn/12-taos-sql/07-function.md b/docs-cn/12-taos-sql/07-function.md index b924aad042..d2cd89f152 100644 --- a/docs-cn/12-taos-sql/07-function.md +++ b/docs-cn/12-taos-sql/07-function.md @@ -1,966 +1,90 @@ --- sidebar_label: SQL 函数 title: SQL 函数 +toc_max_heading_level: 4 --- -## 聚合函数 +## 单行函数 -TDengine 支持针对数据的聚合查询。提供支持的聚合和选择函数如下： +单行函数为查询结果中的每一行返回一个结果行。 -### COUNT +### 数学函数 -``` -SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]; -``` - -**功能说明**：统计表/超级表中记录行数或某列的非空值个数。 - -**返回数据类型**：长整型 INT64。 - -**应用字段**：应用全部字段。 - -**适用于**：表、超级表。 - -**使用说明**: - -- 可以使用星号(\*)来替代具体的字段，使用星号(\*)返回全部记录数量。 -- 针对同一表的（不包含 NULL 值）字段查询结果均相同。 -- 如果统计对象是具体的列，则返回该列中非 NULL 值的记录数量。 - -**示例**： - -``` -taos> SELECT COUNT(*), COUNT(voltage) FROM meters; - count(*) | count(voltage) | -================================================ - 9 | 9 | -Query OK, 1 row(s) in set (0.004475s) - -taos> SELECT COUNT(*), COUNT(voltage) FROM d1001; - count(*) | count(voltage) | -================================================ - 3 | 3 | -Query OK, 1 row(s) in set (0.001075s) -``` - -### AVG - -``` -SELECT AVG(field_name) FROM tb_name [WHERE clause]; -``` - -**功能说明**：统计表/超级表中某列的平均值。 - -**返回数据类型**：双精度浮点数 Double。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 字段。 - -**适用于**：表、超级表。 - -**示例**： - -``` -taos> SELECT AVG(current), AVG(voltage), AVG(phase) FROM meters; - avg(current) | avg(voltage) | avg(phase) | -==================================================================================== - 11.466666751 | 220.444444444 | 0.293333333 | -Query OK, 1 row(s) in set (0.004135s) - -taos> SELECT AVG(current), AVG(voltage), AVG(phase) FROM d1001; - avg(current) | avg(voltage) | avg(phase) | -==================================================================================== - 11.733333588 | 219.333333333 | 0.316666673 | -Query OK, 1 row(s) in set (0.000943s) -``` - -### TWA - -``` -SELECT TWA(field_name) FROM tb_name WHERE clause; -``` - -**功能说明**：时间加权平均函数。统计表中某列在一段时间内的时间加权平均。 - -**返回数据类型**：双精度浮点数 Double。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表。 - -**使用说明**： - -- 从 2.1.3.0 版本开始，TWA 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 - -### IRATE - -``` -SELECT IRATE(field_name) FROM tb_name WHERE clause; -``` - -**功能说明**：计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率；如果这两个值呈递减关系，那么只取最后一个数用于计算，而不是使用二者差值。 - -**返回数据类型**：双精度浮点数 Double。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表。 - -**使用说明**： - -- 从 2.1.3.0 版本开始此函数可用，IRATE 可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 - -### SUM - -``` -SELECT SUM(field_name) FROM tb_name [WHERE clause]; -``` - -**功能说明**：统计表/超级表中某列的和。 - -**返回数据类型**：双精度浮点数 Double 和长整型 INT64。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表。 - -**示例**： - -``` -taos> SELECT SUM(current), SUM(voltage), SUM(phase) FROM meters; - sum(current) | sum(voltage) | sum(phase) | -================================================================================ - 103.200000763 | 1984 | 2.640000001 | -Query OK, 1 row(s) in set (0.001702s) - -taos> SELECT SUM(current), SUM(voltage), SUM(phase) FROM d1001; - sum(current) | sum(voltage) | sum(phase) | -================================================================================ - 35.200000763 | 658 | 0.950000018 | -Query OK, 1 row(s) in set (0.000980s) -``` - -### STDDEV +#### ABS +```sql + SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` -SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; -``` - -**功能说明**：统计表中某列的均方差。 - -**返回数据类型**：双精度浮点数 Double。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表（从 2.0.15.1 版本开始） - -**示例**： - -``` -taos> SELECT STDDEV(current) FROM d1001; - stddev(current) | -============================ - 1.020892909 | -Query OK, 1 row(s) in set (0.000915s) -``` - -### LEASTSQUARES - -``` -SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]; -``` - -**功能说明**：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val 是自变量初始值，step_val 是自变量的步长值。 - -**返回数据类型**：字符串表达式（斜率, 截距）。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表。 - -**示例**： - -``` -taos> SELECT LEASTSQUARES(current, 1, 1) FROM d1001; - leastsquares(current, 1, 1) | -===================================================== -{slop:1.000000, intercept:9.733334} | -Query OK, 1 row(s) in set (0.000921s) -``` - -### MODE - -``` -SELECT MODE(field_name) FROM tb_name [WHERE clause]; -``` - -**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，输出空。不能匹配标签、时间戳输出。 - -**返回数据类型**：同应用的字段。 - -**应用字段**：适合于除时间主列外的任何类型字段。 - -**使用说明**：由于返回数据量未知，考虑到内存因素，为了函数可以正常返回结果，建议不重复的数据量在 10 万级别，否则会报错。 - -**支持的版本**：2.6.0.0 及以后的版本。 - -**示例**： - -``` -taos> select voltage from d002; - voltage | -======================== - 1 | - 1 | - 2 | - 19 | -Query OK, 4 row(s) in set (0.003545s) - -taos> select mode(voltage) from d002; - mode(voltage) | -======================== - 1 | -Query OK, 1 row(s) in set (0.019393s) -``` - -### HYPERLOGLOG - -``` -SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**： - - 采用 hyperloglog 算法，返回某列的基数。该算法在数据量很大的情况下，可以明显降低内存的占用，但是求出来的基数是个估算值，标准误差（标准误差是多次实验，每次的平均数的标准差，不是与真实结果的误差）为 0.81%。 - - 在数据量较少的时候该算法不是很准确，可以使用 select count(data) from (select unique(col) as data from table) 的方法。 - -**返回结果类型**：整形。 - -**应用字段**：适合于任何类型字段。 - -**支持的版本**：2.6.0.0 及以后的版本。 - -**示例**： - -``` -taos> select dbig from shll; - dbig | -======================== - 1 | - 1 | - 1 | - NULL | - 2 | - 19 | - NULL | - 9 | -Query OK, 8 row(s) in set (0.003755s) - -taos> select hyperloglog(dbig) from shll; - hyperloglog(dbig)| -======================== - 4 | -Query OK, 1 row(s) in set (0.008388s) -``` - -### HISTOGRAM - -``` -SELECT HISTOGRAM(field_name，bin_type, bin_description, normalized) FROM tb_name [WHERE clause]; -``` - -**功能说明**：统计数据按照用户指定区间的分布。 - -**返回结果类型**：如归一化参数 normalized 设置为 1，返回结果为双精度浮点类型 DOUBLE，否则为长整形 INT64。 - -**应用字段**：数值型字段。 - -**支持的版本**：2.6.0.0 及以后的版本。 - -**适用于**: 表和超级表。 - -**说明**： -1. bin_type 用户指定的分桶类型, 有效输入类型为"user_input“, ”linear_bin", "log_bin"。 -2. bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)： - - "user_input": "[1, 3, 5, 7]" - 用户指定 bin 的具体数值。 - - - "linear_bin": "{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" - "start" 表示数据起始点，"width" 表示每次 bin 偏移量, "count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点跟终点， - 生成区间为[-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]。 - - - "log_bin": "{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" - "start" 表示数据起始点，"factor" 表示按指数递增的因子，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点跟终点， - 生成区间为[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]。 -3. normalized 是否将返回结果归一化到 0~1 之间 。有效输入为 0 和 1。 - -**示例**： - -```mysql -taos> SELECT HISTOGRAM(voltage, "user_input", "[1,3,5,7]", 1) FROM meters; - histogram(voltage, "user_input", "[1,3,5,7]", 1) | - ======================================================= - {"lower_bin":1, "upper_bin":3, "count":0.333333} | - {"lower_bin":3, "upper_bin":5, "count":0.333333} | - {"lower_bin":5, "upper_bin":7, "count":0.333333} | - Query OK, 3 row(s) in set (0.004273s) - -taos> SELECT HISTOGRAM(voltage, 'linear_bin', '{"start": 1, "width": 3, "count": 3, "infinity": false}', 0) FROM meters; - histogram(voltage, 'linear_bin', '{"start": 1, "width": 3, " | - =================================================================== - {"lower_bin":1, "upper_bin":4, "count":3} | - {"lower_bin":4, "upper_bin":7, "count":3} | - {"lower_bin":7, "upper_bin":10, "count":3} | - Query OK, 3 row(s) in set (0.004887s) - -taos> SELECT HISTOGRAM(voltage, 'log_bin', '{"start": 1, "factor": 3, "count": 3, "infinity": true}', 0) FROM meters; - histogram(voltage, 'log_bin', '{"start": 1, "factor": 3, "count" | - =================================================================== - {"lower_bin":-inf, "upper_bin":1, "count":3} | - {"lower_bin":1, "upper_bin":3, "count":2} | - {"lower_bin":3, "upper_bin":9, "count":6} | - {"lower_bin":9, "upper_bin":27, "count":3} | - {"lower_bin":27, "upper_bin":inf, "count":1} | -``` - -### ELAPSED - -```mysql -SELECT ELAPSED(field_name[, time_unit]) FROM { tb_name | stb_name } [WHERE clause] [INTERVAL(interval [, offset]) [SLIDING sliding]]; -``` - -**功能说明**：elapsed函数表达了统计周期内连续的时间长度，和twa函数配合使用可以计算统计曲线下的面积。在通过INTERVAL子句指定窗口的情况下，统计在给定时间范围内的每个窗口内有数据覆盖的时间范围；如果没有INTERVAL子句，则返回整个给定时间范围内的有数据覆盖的时间范围。注意，ELAPSED返回的并不是时间范围的绝对值，而是绝对值除以time_unit所得到的单位个数。 - -**返回结果类型**：Double - -**应用字段**：Timestamp类型 - -**支持的版本**：2.6.0.0 及以后的版本。 - -**适用于**: 表，超级表，嵌套查询的外层查询 - -**说明**： -- field_name参数只能是表的第一列，即timestamp主键列。 -- 按time_unit参数指定的时间单位返回，最小是数据库的时间分辨率。time_unit参数未指定时，以数据库的时间分辨率为时间单位。 -- 可以和interval组合使用，返回每个时间窗口的时间戳差值。需要特别注意的是，除第一个时间窗口和最后一个时间窗口外，中间窗口的时间戳差值均为窗口长度。 -- order by asc/desc不影响差值的计算结果。 -- 对于超级表，需要和group by tbname子句组合使用，不可以直接使用。 -- 对于普通表，不支持和group by子句组合使用。 -- 对于嵌套查询，仅当内层查询会输出隐式时间戳列时有效。例如select elapsed(ts) from (select diff(value) from sub1)语句，diff函数会让内层查询输出隐式时间戳列，此为主键列，可以用于elapsed函数的第一个参数。相反，例如select elapsed(ts) from (select * from sub1) 语句，ts列输出到外层时已经没有了主键列的含义，无法使用elapsed函数。此外，elapsed函数作为一个与时间线强依赖的函数，形如select elapsed(ts) from (select diff(value) from st group by tbname)尽管会返回一条计算结果，但并无实际意义，这种用法后续也将被限制。 -- 不支持与leastsquares、diff、derivative、top、bottom、last_row、interp等函数混合使用。 - -## 选择函数 - -在使用所有的选择函数的时候，可以同时指定输出 ts 列或标签列（包括 tbname），这样就可以方便地知道被选出的值是源于哪个数据行的。 - -### MIN - -``` -SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; -``` - -**功能说明**：统计表/超级表中某列的值最小值。 - -**返回数据类型**：同应用的字段。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表。 - -**示例**： - -``` -taos> SELECT MIN(current), MIN(voltage) FROM meters; - min(current) | min(voltage) | -====================================== - 10.20000 | 218 | -Query OK, 1 row(s) in set (0.001765s) - -taos> SELECT MIN(current), MIN(voltage) FROM d1001; - min(current) | min(voltage) | -====================================== - 10.30000 | 218 | -Query OK, 1 row(s) in set (0.000950s) -``` - -### MAX - -``` -SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：统计表/超级表中某列的值最大值。 - -**返回数据类型**：同应用的字段。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表。 - -**示例**： - -``` -taos> SELECT MAX(current), MAX(voltage) FROM meters; - max(current) | max(voltage) | -====================================== - 13.40000 | 223 | -Query OK, 1 row(s) in set (0.001123s) - -taos> SELECT MAX(current), MAX(voltage) FROM d1001; - max(current) | max(voltage) | -====================================== - 12.60000 | 221 | -Query OK, 1 row(s) in set (0.000987s) -``` - -### FIRST - -``` -SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：统计表/超级表中某列的值最先写入的非 NULL 值。 - -**返回数据类型**：同应用的字段。 - -**应用字段**：所有字段。 - -**适用于**：表、超级表。 - -**使用说明**: - -- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)； -- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL； -- 如果结果集中所有列全部为 NULL 值，则不返回结果。 - -**示例**： - -``` -taos> SELECT FIRST(*) FROM meters; - first(ts) | first(current) | first(voltage) | first(phase) | -========================================================================================= -2018-10-03 14:38:04.000 | 10.20000 | 220 | 0.23000 | -Query OK, 1 row(s) in set (0.004767s) - -taos> SELECT FIRST(current) FROM d1002; - first(current) | -======================= - 10.20000 | -Query OK, 1 row(s) in set (0.001023s) -``` - -### LAST - -``` -SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：统计表/超级表中某列的值最后写入的非 NULL 值。 - -**返回数据类型**：同应用的字段。 - -**应用字段**：所有字段。 - -**适用于**：表、超级表。 - -**使用说明**: - -- 如果要返回各个列的最后（时间戳最大）一个非 NULL 值，可以使用 LAST(\*)； -- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；如果结果集中所有列全部为 NULL 值，则不返回结果。 -- 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 - - -**示例**： - -``` -taos> SELECT LAST(*) FROM meters; - last(ts) | last(current) | last(voltage) | last(phase) | -======================================================================================== -2018-10-03 14:38:16.800 | 12.30000 | 221 | 0.31000 | -Query OK, 1 row(s) in set (0.001452s) - -taos> SELECT LAST(current) FROM d1002; - last(current) | -======================= - 10.30000 | -Query OK, 1 row(s) in set (0.000843s) -``` - -### TOP - -``` -SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**： 统计表/超级表中某列的值最大 _k_ 个非 NULL 值。如果多条数据取值一样，全部取用又会超出 k 条限制时，系统会从相同值中随机选取符合要求的数量返回。 - -**返回数据类型**：同应用的字段。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表。 - -**使用说明**: - -- *k*值取值范围 1≤*k*≤100； -- 系统同时返回该记录关联的时间戳列； -- 限制：TOP 函数不支持 FILL 子句。 - -**示例**： - -``` -taos> SELECT TOP(current, 3) FROM meters; - ts | top(current, 3) | -================================================= -2018-10-03 14:38:15.000 | 12.60000 | -2018-10-03 14:38:16.600 | 13.40000 | -2018-10-03 14:38:16.800 | 12.30000 | -Query OK, 3 row(s) in set (0.001548s) - -taos> SELECT TOP(current, 2) FROM d1001; - ts | top(current, 2) | -================================================= -2018-10-03 14:38:15.000 | 12.60000 | -2018-10-03 14:38:16.800 | 12.30000 | -Query OK, 2 row(s) in set (0.000810s) -``` - -### BOTTOM - -``` -SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：统计表/超级表中某列的值最小 _k_ 个非 NULL 值。如果多条数据取值一样，全部取用又会超出 k 条限制时，系统会从相同值中随机选取符合要求的数量返回。 - -**返回数据类型**：同应用的字段。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表。 - -**使用说明**: - -- *k*值取值范围 1≤*k*≤100； -- 系统同时返回该记录关联的时间戳列； -- 限制：BOTTOM 函数不支持 FILL 子句。 - -**示例**： - -``` -taos> SELECT BOTTOM(voltage, 2) FROM meters; - ts | bottom(voltage, 2) | -=============================================== -2018-10-03 14:38:15.000 | 218 | -2018-10-03 14:38:16.650 | 218 | -Query OK, 2 row(s) in set (0.001332s) - -taos> SELECT BOTTOM(current, 2) FROM d1001; - ts | bottom(current, 2) | -================================================= -2018-10-03 14:38:05.000 | 10.30000 | -2018-10-03 14:38:16.800 | 12.30000 | -Query OK, 2 row(s) in set (0.000793s) -``` - -### PERCENTILE - -``` -SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; -``` - -**功能说明**：统计表中某列的值百分比分位数。 - -**返回数据类型**： 双精度浮点数 Double。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表。 - -**使用说明**：*P*值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX。 - -**示例**： - -``` -taos> SELECT PERCENTILE(current, 20) FROM d1001; -percentile(current, 20) | -============================ - 11.100000191 | -Query OK, 1 row(s) in set (0.000787s) -``` - -### APERCENTILE - -``` -SELECT APERCENTILE(field_name, P[, algo_type]) -FROM { tb_name | stb_name } [WHERE clause] -``` - -**功能说明**：统计表/超级表中指定列的值百分比分位数，与 PERCENTILE 函数相似，但是返回近似结果。 - -**返回数据类型**： 双精度浮点数 Double。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 -**适用于**：表、超级表。 +**功能说明**：获得指定列的绝对值 -**使用说明** +**返回结果类型**：如果输入值为整数，输出值是 UBIGINT 类型。如果输入值是 FLOAT/DOUBLE 数据类型，输出值是 DOUBLE 数据类型。 -- **P**值有效取值范围 0≤P≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX； -- **algo_type**的有效输入：**default** 和 **t-digest** -- 用于指定计算近似分位数的算法。可不提供第三个参数的输入，此时将使用 default 的算法进行计算，即 apercentile(column_name, 50, "default") 与 apercentile(column_name, 50) 等价。 -- 当使用“t-digest”参数的时候，将使用 t-digest 方式采样计算近似分位数。但该参数指定计算算法的功能从 2.2.0.x 版本开始支持，2.2.0.0 之前的版本不支持指定使用算法的功能。 +**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 **嵌套子查询支持**：适用于内层查询和外层查询。 -``` -taos> SELECT APERCENTILE(current, 20) FROM d1001; -apercentile(current, 20) | -============================ - 10.300000191 | -Query OK, 1 row(s) in set (0.000645s) +**适用于**: 表和超级表 -taos> select apercentile (count, 80, 'default') from stb1; - apercentile (c0, 80, 'default') | -================================== - 601920857.210056424 | -Query OK, 1 row(s) in set (0.012363s) +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -taos> select apercentile (count, 80, 't-digest') from stb1; - apercentile (c0, 80, 't-digest') | -=================================== - 605869120.966666579 | -Query OK, 1 row(s) in set (0.011639s) +#### ACOS + +```sql + SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` -### LAST_ROW +**功能说明**：获得指定列的反余弦结果 -``` -SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; +**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL + +**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表 + +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +#### ASIN + +```sql + SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` -**功能说明**：返回表/超级表的最后一条记录。 +**功能说明**：获得指定列的反正弦结果 -**返回数据类型**：同应用的字段。 +**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**应用字段**：所有字段。 +**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 -**适用于**：表、超级表。 +**嵌套子查询支持**：适用于内层查询和外层查询。 -**使用说明**： +**适用于**: 表和超级表 -- 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 -- 不能与 INTERVAL 一起使用。 +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -**示例**： -``` - taos> SELECT LAST_ROW(current) FROM meters; - last_row(current) | - ======================= - 12.30000 | - Query OK, 1 row(s) in set (0.001238s) +#### ATAN - taos> SELECT LAST_ROW(current) FROM d1002; - last_row(current) | - ======================= - 10.30000 | - Query OK, 1 row(s) in set (0.001042s) +```sql + SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` -### INTERP [2.3.1 及之后的版本] +**功能说明**：获得指定列的反正切结果 -``` -SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [ RANGE(timestamp1,timestamp2) ] [EVERY(interval)] [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})]; -``` +**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**功能说明**：返回表/超级表的指定时间截面指定列的记录值（插值）。 +**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 -**返回数据类型**：同字段类型。 +**嵌套子查询支持**：适用于内层查询和外层查询。 -**应用字段**：数值型字段。 +**适用于**: 表和超级表 -**适用于**：表、超级表、嵌套查询。 +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -**使用说明** - -- INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。 -- INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 -- INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1<=timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。如果没有指定 RANGE，那么满足过滤条件的输入数据中第一条记录的 timestamp 即为 timestamp1，最后一条记录的 timestamp 即为 timestamp2，同样也满足 timestamp1 <= timestamp2。 -- INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。如果没有指定 EVERY，则默认窗口大小为无穷大，即从 timestamp1 开始只有一个窗口。 -- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值，如果没有 FILL 字段则默认不插值，即输出为原始记录值或不输出（原始记录不存在）。 -- INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 group by tbname 一起使用，当作用嵌套查询外层时内层子查询不能含 GROUP BY 信息。 -- INTERP 的插值结果不受 ORDER BY timestamp 的影响，ORDER BY timestamp 只影响输出结果的排序。 - -**SQL示例（基于文档中广泛使用的电表 schema )**： - -- 单点线性插值 - -``` - taos> SELECT INTERP(current) FROM t1 RANGE('2017-7-14 18:40:00','2017-7-14 18:40:00') FILL(LINEAR); -``` - -- 在2017-07-14 18:00:00到2017-07-14 19:00:00间每隔5秒钟进行取值(不插值) - -``` - taos> SELECT INTERP(current) FROM t1 RANGE('2017-7-14 18:00:00','2017-7-14 19:00:00') EVERY(5s); -``` - -- 在2017-07-14 18:00:00到2017-07-14 19:00:00间每隔5秒钟进行线性插值 - -``` - taos> SELECT INTERP(current) FROM t1 RANGE('2017-7-14 18:00:00','2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR); -``` - -- 在所有时间范围内每隔 5 秒钟进行向后插值 - -``` - taos> SELECT INTERP(current) FROM t1 EVERY(5s) FILL(NEXT); -``` - -- 根据 2017-07-14 17:00:00 到 2017-07-14 20:00:00 间的数据进行从 2017-07-14 18:00:00 到 2017-07-14 19:00:00 间每隔 5 秒钟进行线性插值 - -``` - taos> SELECT INTERP(current) FROM t1 where ts >= '2017-07-14 17:00:00' and ts <= '2017-07-14 20:00:00' RANGE('2017-7-14 18:00:00'，'2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR); -``` - -### INTERP [2.3.1 之前的版本] - -``` -SELECT INTERP(field_name) FROM { tb_name | stb_name } WHERE ts='timestamp' [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})]; -``` - -**功能说明**：返回表/超级表的指定时间截面、指定字段的记录。 - -**返回数据类型**：同字段类型。 - -**应用字段**：数值型字段。 - -**适用于**：表、超级表。 - -**使用说明**: - -- 从 2.0.15.0 及以后版本可用 -- INTERP 必须指定时间断面，如果该时间断面不存在直接对应的数据，那么会根据 FILL 参数的设定进行插值。此外，条件语句里面可附带筛选条件，例如标签、tbname。 -- INTERP 查询要求查询的时间区间必须位于数据集合（表）的所有记录的时间范围之内。如果给定的时间戳位于时间范围之外，即使有插值指令，仍然不返回结果。 -- 单个 INTERP 函数查询只能够针对一个时间点进行查询，如果需要返回等时间间隔的断面数据，可以通过 INTERP 配合 EVERY 的方式来进行查询处理（而不是使用 INTERVAL），其含义是每隔固定长度的时间进行插值 - -**示例**： - -``` - taos> SELECT INTERP(*) FROM meters WHERE ts='2017-7-14 18:40:00.004'; - interp(ts) | interp(current) | interp(voltage) | interp(phase) | - ========================================================================================== - 2017-07-14 18:40:00.004 | 9.84020 | 216 | 0.32222 | - Query OK, 1 row(s) in set (0.002652s) -``` - -如果给定的时间戳无对应的数据，在不指定插值生成策略的情况下，不会返回结果，如果指定了插值策略，会根据插值策略返回结果。 - -``` - taos> SELECT INTERP(*) FROM meters WHERE tbname IN ('d636') AND ts='2017-7-14 18:40:00.005'; - Query OK, 0 row(s) in set (0.004022s) - - taos> SELECT INTERP(*) FROM meters WHERE tbname IN ('d636') AND ts='2017-7-14 18:40:00.005' FILL(PREV); - interp(ts) | interp(current) | interp(voltage) | interp(phase) | - ========================================================================================== - 2017-07-14 18:40:00.005 | 9.88150 | 217 | 0.32500 | - Query OK, 1 row(s) in set (0.003056s) -``` - -如下所示代码表示在时间区间 `['2017-7-14 18:40:00', '2017-7-14 18:40:00.014']` 中每隔 5 毫秒 进行一次断面计算。 - -``` - taos> SELECT INTERP(current) FROM d636 WHERE ts>='2017-7-14 18:40:00' AND ts<='2017-7-14 18:40:00.014' EVERY(5a); - ts | interp(current) | - ================================================= - 2017-07-14 18:40:00.000 | 10.04179 | - 2017-07-14 18:40:00.010 | 10.16123 | - Query OK, 2 row(s) in set (0.003487s) -``` - -### TAIL - -``` -SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause]; -``` - -**功能说明**：返回跳过最后 offset_val 个，然后取连续 k 个记录，不忽略 NULL 值。offset_val 可以不输入。此时返回最后的 k 个记录。当有 offset_val 输入的情况下，该函数功能等效于 `order by ts desc LIMIT k OFFSET offset_val`。 - -**参数范围**：k: [1,100] offset_val: [0,100]。 - -**返回结果数据类型**：同应用的字段。 - -**应用字段**：适合于除时间主列外的任何类型字段。 - -**支持版本**：2.6.0.0 及之后的版本。 - -**示例**： - -``` -taos> select ts,dbig from tail2; - ts | dbig | -================================================== -2021-10-15 00:31:33.000 | 1 | -2021-10-17 00:31:31.000 | NULL | -2021-12-24 00:31:34.000 | 2 | -2022-01-01 08:00:05.000 | 19 | -2022-01-01 08:00:06.000 | NULL | -2022-01-01 08:00:07.000 | 9 | -Query OK, 6 row(s) in set (0.001952s) - -taos> select tail(dbig,2,2) from tail2; -ts | tail(dbig,2,2) | -================================================== -2021-12-24 00:31:34.000 | 2 | -2022-01-01 08:00:05.000 | 19 | -Query OK, 2 row(s) in set (0.002307s) -``` - -### UNIQUE - -``` -SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause]; -``` - -**功能说明**：返回该列的数值首次出现的值。该函数功能与 distinct 相似，但是可以匹配标签和时间戳信息。可以针对除时间列以外的字段进行查询，可以匹配标签和时间戳，其中的标签和时间戳是第一次出现时刻的标签和时间戳。 - -**返回结果数据类型**：同应用的字段。 - -**应用字段**：适合于除时间类型以外的字段。 - -**支持版本**：2.6.0.0 及之后的版本。 - -**使用说明**: - -- 该函数可以应用在普通表和超级表上。不能和窗口操作一起使用，例如 interval/state_window/session_window 。 -- 由于返回数据量未知，考虑到内存因素，为了函数可以正常返回结果，建议不重复的数据量在 10 万级别，否则会报错。 - -**示例**： - -``` -taos> select ts,voltage from unique1; - ts | voltage | -================================================== -2021-10-17 00:31:31.000 | 1 | -2022-01-24 00:31:31.000 | 1 | -2021-10-17 00:31:31.000 | 1 | -2021-12-24 00:31:31.000 | 2 | -2022-01-01 08:00:01.000 | 19 | -2021-10-17 00:31:31.000 | NULL | -2022-01-01 08:00:02.000 | NULL | -2022-01-01 08:00:03.000 | 9 | -Query OK, 8 row(s) in set (0.003018s) - -taos> select unique(voltage) from unique1; -ts | unique(voltage) | -================================================== -2021-10-17 00:31:31.000 | 1 | -2021-10-17 00:31:31.000 | NULL | -2021-12-24 00:31:31.000 | 2 | -2022-01-01 08:00:01.000 | 19 | -2022-01-01 08:00:03.000 | 9 | -Query OK, 5 row(s) in set (0.108458s) -``` - -## 计算函数 - -### DIFF - - ```sql - SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHERE clause]; - ``` - -**功能说明**：统计表中某列的值与前一行对应值的差。 ignore_negative 取值为 0|1 , 可以不填，默认值为 0. 不忽略负值。ignore_negative 为 1 时表示忽略负数。 - -**返回结果数据类型**：同应用字段。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表。 - -**使用说明**: - -- 输出结果行数是范围内总行数减一，第一行没有结果输出。 -- 从 2.1.3.0 版本开始，DIFF 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 -- 从 2.6.0 开始，DIFF 函数支持 ignore_negative 参数 - -**示例**： - - ```sql - taos> SELECT DIFF(current) FROM d1001; - ts | diff(current) | - ================================================= - 2018-10-03 14:38:15.000 | 2.30000 | - 2018-10-03 14:38:16.800 | -0.30000 | - Query OK, 2 row(s) in set (0.001162s) - ``` - -### DERIVATIVE - -``` -SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHERE clause]; -``` - -**功能说明**：统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定，最小可以是 1 秒（1s）；ignore_negative 参数的值可以是 0 或 1，为 1 时表示忽略负值。 - -**返回数据类型**：双精度浮点数。 - -**应用字段**：不能应用在 timestamp、binary、nchar、bool 类型字段。 - -**适用于**：表、超级表 - -**使用说明**: - -- 从 2.1.3.0 及以后版本可用；输出结果行数是范围内总行数减一，第一行没有结果输出。 -- DERIVATIVE 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 - -**示例**： - -``` -taos> select derivative(current, 10m, 0) from t1; - ts | derivative(current, 10m, 0) | -======================================================== - 2021-08-20 10:11:22.790 | 0.500000000 | - 2021-08-20 11:11:22.791 | 0.166666620 | - 2021-08-20 12:11:22.791 | 0.000000000 | - 2021-08-20 13:11:22.792 | 0.166666620 | - 2021-08-20 14:11:22.792 | -0.666666667 | -Query OK, 5 row(s) in set (0.004883s) -``` - -### SPREAD - -``` -SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：统计表/超级表中某列的最大值和最小值之差。 - -**返回数据类型**：双精度浮点数。 - -**应用字段**：不能应用在 binary、nchar、bool 类型字段。 - -**适用于**：表、超级表。 - -**使用说明**：可用于 TIMESTAMP 字段，此时表示记录的时间覆盖范围。 - -**示例**： - -``` -taos> SELECT SPREAD(voltage) FROM meters; - spread(voltage) | -============================ - 5.000000000 | -Query OK, 1 row(s) in set (0.001792s) - -taos> SELECT SPREAD(voltage) FROM d1001; - spread(voltage) | -============================ - 3.000000000 | -Query OK, 1 row(s) in set (0.000836s) -``` - -### CEIL +#### CEIL ``` SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; @@ -981,170 +105,7 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; - 支持 +、-、\*、/ 运算，如 ceil(col1) + ceil(col2)。 - 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -### FLOOR - -``` -SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：获得指定列的向下取整数的结果。 - 其他使用说明参见 CEIL 函数描述。 - -### ROUND - -``` -SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：获得指定列的四舍五入的结果。 - 其他使用说明参见 CEIL 函数描述。 - -### CSUM - -```sql - SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - - **功能说明**：累加和（Cumulative sum），输出行与输入行数相同。 - - **返回结果类型**： 输入列如果是整数类型返回值为长整型 （int64_t），浮点数返回值为双精度浮点数（Double）。无符号整数类型返回值为无符号长整型（uint64_t）。 返回结果中同时带有每行记录对应的时间戳。 - - **适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在标签之上。 - - **嵌套子查询支持**： 适用于内层查询和外层查询。 - - **使用说明**： - - - 不支持 +、-、*、/ 运算，如 csum(col1) + csum(col2)。 - - 只能与聚合（Aggregation）函数一起使用。 该函数可以应用在普通表和超级表上。 - - 使用在超级表上的时候，需要搭配 Group by tbname使用，将结果强制规约到单个时间线。 - -**支持版本**： 从2.3.0.x开始支持 - -### MAVG - -```sql - SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause] -``` - - **功能说明**： 计算连续 k 个值的移动平均数（moving average）。如果输入行数小于 k，则无结果输出。参数 k 的合法输入范围是 1≤ k ≤ 1000。 - - **返回结果类型**： 返回双精度浮点数类型。 - - **适用数据类型**： 不能应用在 timestamp、binary、nchar、bool 类型上；在超级表查询中使用时，不能应用在标签之上。 - - **嵌套子查询支持**： 适用于内层查询和外层查询。 - - **使用说明**： - - - 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1); - - 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用； - - 该函数可以应用在普通表和超级表上；使用在超级表上的时候，需要搭配 Group by tbname使用，将结果强制规约到单个时间线。 - -**支持版本**： 从2.3.0.x开始支持 - -### SAMPLE - -```sql - SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause] -``` - - **功能说明**： 获取数据的 k 个采样值。参数 k 的合法输入范围是 1≤ k ≤ 1000。 - - **返回结果类型**： 同原始数据类型， 返回结果中带有该行记录的时间戳。 - - **适用数据类型**： 在超级表查询中使用时，不能应用在标签之上。 - - **嵌套子查询支持**： 适用于内层查询和外层查询。 - - **使用说明**： - - - 不能参与表达式计算；该函数可以应用在普通表和超级表上； - - 使用在超级表上的时候，需要搭配 Group by tbname 使用，将结果强制规约到单个时间线。 - -**支持版本**： 从2.3.0.x开始支持 - -### ASIN - -```sql - SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**功能说明**：获得指定列的反正弦结果 - -**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL - -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**使用说明**： - -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 - -### ACOS - -```sql - SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**功能说明**：获得指定列的反余弦结果 - -**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL - -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**使用说明**： - -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 - -### ATAN - -```sql - SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**功能说明**：获得指定列的反正切结果 - -**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL - -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**使用说明**： - -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 - -### SIN - -```sql - SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**功能说明**：获得指定列的正弦结果 - -**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL - -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**使用说明**： - -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 - -### COS +#### COS ```sql SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] @@ -1158,53 +119,20 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **嵌套子查询支持**：适用于内层查询和外层查询。 -**使用说明**： +**适用于**: 表和超级表 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -### TAN +#### FLOOR -```sql - SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` +SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` -**功能说明**：获得指定列的正切结果 +**功能说明**：获得指定列的向下取整数的结果。 + 其他使用说明参见 CEIL 函数描述。 -**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL - -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**使用说明**： - -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 - -### POW - -```sql - SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] -``` - -**功能说明**：获得指定列的指数为 power 的幂 - -**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL - -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**使用说明**： - -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 - -### LOG +#### LOG ```sql SELECT LOG(field_name, base) FROM { tb_name | stb_name } [WHERE clause] @@ -1218,33 +146,59 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **嵌套子查询支持**：适用于内层查询和外层查询。 -**使用说明**： +**适用于**: 表和超级表 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -### ABS + +#### POW ```sql - SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause] + SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] ``` -**功能说明**：获得指定列的绝对值 +**功能说明**：获得指定列的指数为 power 的幂 -**返回结果类型**：如果输入值为整数，输出值是 UBIGINT 类型。如果输入值是 FLOAT/DOUBLE 数据类型，输出值是 DOUBLE 数据类型。 +**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL **适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 **嵌套子查询支持**：适用于内层查询和外层查询。 -**使用说明**： +**适用于**: 表和超级表 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -### SQRT + +#### ROUND + +``` +SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：获得指定列的四舍五入的结果。 + 其他使用说明参见 CEIL 函数描述。 + + +#### SIN + +```sql + SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**功能说明**：获得指定列的正弦结果 + +**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL + +**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表 + +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +#### SQRT ```sql SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause] @@ -1258,39 +212,49 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **嵌套子查询支持**：适用于内层查询和外层查询。 -**使用说明**： +**适用于**: 表和超级表 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -- 该函数可以应用在普通表和超级表上。 -- 版本2.6.0.x后支持 +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -### CAST +#### TAN ```sql - SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] + SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` -**功能说明**：数据类型转换函数，输入参数 expression 支持普通列、常量、标量函数及它们之间的四则运算，不支持 tag 列，只适用于 select 子句中。 +**功能说明**：获得指定列的正切结果 -**返回结果类型**：CAST 中指定的类型（type_name）。 +**返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**： +**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 -- 输入参数 expression 的类型可以是除 JSON 外目前所有类型字段（BOOL/TINYINT/SMALLINT/INT/BIGINT/FLOAT/DOUBLE/BINARY(M)/TIMESTAMP/NCHAR(M)/TINYINT UNSIGNED/SMALLINT UNSIGNED/INT UNSIGNED/BIGINT UNSIGNED）; -- 输出目标类型只支持 BIGINT/BINARY(N)/TIMESTAMP/NCHAR(N)/BIGINT UNSIGNED。 +**嵌套子查询支持**：适用于内层查询和外层查询。 -**使用说明**： +**适用于**: 表和超级表 -- 对于不能支持的类型转换会直接报错。 -- 如果输入值为NULL则输出值也为NULL。 -- 对于类型支持但某些值无法正确转换的情况对应的转换后的值以转换函数输出为准。目前可能遇到的几种情况： - 1）BINARY/NCHAR转BIGINT/BIGINT UNSIGNED时可能出现的无效字符情况，例如"a"可能转为0。 - 2）有符号数或TIMESTAMP转BIGINT UNSIGNED可能遇到的溢出问题。 - 3）BIGINT UNSIGNED转BIGINT可能遇到的溢出问题。 - 4）FLOAT/DOUBLE转BIGINT/BIGINT UNSIGNED可能遇到的溢出问题。 -- 版本2.6.0.x后支持 +**使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -### CONCAT +### 字符串函数 + +字符串函数的输入参数为字符串类型，返回结果为数值类型或字符串类型。 + +#### CHAR_LENGTH + +``` + SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] +``` + +**功能说明**：以字符计数的字符串长度。 + +**返回结果类型**：INT。如果输入值为NULL，输出值为NULL。 + +**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表 + +#### CONCAT ```sql SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] @@ -1298,19 +262,16 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **功能说明**：字符串连接函数。 -**返回结果类型**：同输入参数类型，BINARY 或者 NCHAR。 +**返回结果类型**：如果所有参数均为BINARY类型，则结果类型为BINARY。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。 -**适用数据类型**：输入参数或者全部是 BINARY 格式的字符串或者列，或者全部是 NCHAR 格式的字符串或者列。不能应用在 TAG 列。 +**适用数据类型**：BINARY, NCHAR。不能应用在 TAG 列。 该函数最小参数个数为2个，最大参数个数为8个。 -**使用说明**： +**嵌套子查询支持**：适用于内层查询和外层查询。 -- 如果输入值为NULL，输出值为NULL。 -- 该函数最小参数个数为2个，最大参数个数为8个。 -- 该函数可以应用在普通表和超级表上。 -- 该函数适用于内层查询和外层查询。 -- 版本2.6.0.x后支持 +**适用于**: 表和超级表 -### CONCAT_WS + +#### CONCAT_WS ``` SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] @@ -1318,19 +279,16 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **功能说明**：带分隔符的字符串连接函数。 -**返回结果类型**：同输入参数类型，BINARY 或者 NCHAR。 +**返回结果类型**：如果所有参数均为BINARY类型，则结果类型为BINARY。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。如果separator值不为NULL，其他输入为NULL，输出为空串。 -**适用数据类型**：输入参数或者全部是 BINARY 格式的字符串或者列，或者全部是 NCHAR 格式的字符串或者列。不能应用在 TAG 列。 +**适用数据类型**：BINARY, NCHAR。不能应用在 TAG 列。 该函数最小参数个数为3个，最大参数个数为9个。 -**使用说明**： +**嵌套子查询支持**：适用于内层查询和外层查询。 -- 如果separator值为NULL，输出值为NULL。如果separator值不为NULL，其他输入为NULL，输出为空串 -- 该函数最小参数个数为3个，最大参数个数为9个。 -- 该函数可以应用在普通表和超级表上。 -- 该函数适用于内层查询和外层查询。 -- 版本2.6.0.x后支持 +**适用于**: 表和超级表 -### LENGTH + +#### LENGTH ``` SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] @@ -1342,33 +300,12 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 -**使用说明** +**嵌套子查询支持**：适用于内层查询和外层查询。 -- 如果输入值为NULL，输出值为NULL。 -- 该函数可以应用在普通表和超级表上。 -- 函数适用于内层查询和外层查询。 -- 版本2.6.0.x后支持 +**适用于**: 表和超级表 -### CHAR_LENGTH -``` - SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] -``` - -**功能说明**：以字符计数的字符串长度。 - -**返回结果类型**：INT。 - -**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 - -**使用说明** - -- 如果输入值为NULL，输出值为NULL。 -- 该函数可以应用在普通表和超级表上。 -- 该函数适用于内层查询和外层查询。 -- 版本2.6.0.x后支持 - -### LOWER +#### LOWER ``` SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause] @@ -1376,37 +313,16 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **功能说明**：将字符串参数值转换为全小写字母。 -**返回结果类型**：同输入类型。 +**返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 **适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 -**使用说明**： +**嵌套子查询支持**：适用于内层查询和外层查询。 -- 如果输入值为NULL，输出值为NULL。 -- 该函数可以应用在普通表和超级表上。 -- 该函数适用于内层查询和外层查询。 -- 版本2.6.0.x后支持 +**适用于**: 表和超级表 -### UPPER -``` - SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause] -``` - -**功能说明**：将字符串参数值转换为全大写字母。 - -**返回结果类型**：同输入类型。 - -**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 - -**使用说明**： - -- 如果输入值为NULL，输出值为NULL。 -- 该函数可以应用在普通表和超级表上。 -- 该函数适用于内层查询和外层查询。 -- 版本2.6.0.x后支持 - -### LTRIM +#### LTRIM ``` SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] @@ -1414,37 +330,33 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **功能说明**：返回清除左边空格后的字符串。 -**返回结果类型**：同输入类型。 +**返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 **适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 -**使用说明**： +**嵌套子查询支持**：适用于内层查询和外层查询。 -- 如果输入值为NULL，输出值为NULL。 -- 该函数可以应用在普通表和超级表上。 -- 该函数适用于内层查询和外层查询。 -- 版本2.6.0.x后支持 +**适用于**: 表和超级表 -### RTRIM + +#### RTRIM ``` - SELECT RTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] + SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] ``` **功能说明**：返回清除右边空格后的字符串。 -**返回结果类型**：同输入类型。 +**返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 **适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 -**使用说明**： +**嵌套子查询支持**：适用于内层查询和外层查询。 -- 如果输入值为NULL，输出值为NULL。 -- 该函数可以应用在普通表和超级表上。 -- 该函数适用于内层查询和外层查询。 -- 版本2.6.0.x后支持 +**适用于**: 表和超级表 -### SUBSTR + +#### SUBSTR ``` SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause] @@ -1452,128 +364,121 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **功能说明**：从源字符串 str 中的指定位置 pos 开始取一个长度为 len 的子串并返回。 -**返回结果类型**：同输入类型。 +**返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 + +**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。输入参数pos可以为正数，也可以为负数。如果pos是正数，表示开始位置从字符串开头正数计算。如果pos为负数，表示开始位置从字符串结尾倒数计算。如果输入参数len被忽略，返回的子串包含从pos开始的整个字串。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表 + + +#### UPPER + +``` + SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause] +``` + +**功能说明**：将字符串参数值转换为全大写字母。 + +**返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 **适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 -**使用说明**： +**嵌套子查询支持**：适用于内层查询和外层查询。 -- 如果输入值为NULL，输出值为NULL。 -- 输入参数pos可以为正数，也可以为负数。如果pos是正数，表示开始位置从字符串开头正数计算。如果pos为负数，表示开始位置从字符串结尾倒数计算。如果输入参数len被忽略，返回的子串包含从pos开始的整个字串。 -- 该函数可以应用在普通表和超级表上。 -- 该函数适用于内层查询和外层查询。 -- 版本2.6.0.x后支持 +**适用于**: 表和超级表 -### STATECOUNT -``` -SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clause]; -``` +### 转换函数 -**功能说明**：返回满足某个条件的连续记录的个数，结果作为新的一列追加在每行后面。条件根据参数计算，如果条件为 true 则加 1，条件为 false 则重置为-1，如果数据为 NULL，跳过该条数据。 +转换函数将值从一种数据类型转换为另一种数据类型。 -**参数范围**： - -- oper : LT (小于)、GT（大于）、LE（小于等于）、GE（大于等于）、NE（不等于）、EQ（等于），不区分大小写。 -- val : 数值型 - -**返回结果类型**：整形。 - -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上。 - -**嵌套子查询支持**：不支持应用在子查询上。 - -**支持的版本**：2.6 开始的版本。 - -**使用说明**： - -- 该函数可以应用在普通表上，在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname） -- 不能和窗口操作一起使用，例如 interval/state_window/session_window。 - -**示例**： - -``` -taos> select ts,dbig from statef2; - ts | dbig | -======================================================== -2021-10-15 00:31:33.000000000 | 1 | -2021-10-17 00:31:31.000000000 | NULL | -2021-12-24 00:31:34.000000000 | 2 | -2022-01-01 08:00:05.000000000 | 19 | -2022-01-01 08:00:06.000000000 | NULL | -2022-01-01 08:00:07.000000000 | 9 | -Query OK, 6 row(s) in set (0.002977s) - -taos> select stateCount(dbig,GT,2) from statef2; -ts | dbig | statecount(dbig,gt,2) | -================================================================================ -2021-10-15 00:31:33.000000000 | 1 | -1 | -2021-10-17 00:31:31.000000000 | NULL | NULL | -2021-12-24 00:31:34.000000000 | 2 | -1 | -2022-01-01 08:00:05.000000000 | 19 | 1 | -2022-01-01 08:00:06.000000000 | NULL | NULL | -2022-01-01 08:00:07.000000000 | 9 | 2 | -Query OK, 6 row(s) in set (0.002791s) -``` - -### STATEDURATION +#### CAST ```sql -SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [WHERE clause]; + SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] ``` -**功能说明**：返回满足某个条件的连续记录的时间长度，结果作为新的一列追加在每行后面。条件根据参数计算，如果条件为 true 则加上两个记录之间的时间长度（第一个满足条件的记录时间长度记为 0），条件为 false 则重置为-1，如果数据为 NULL，跳过该条数据。 +**功能说明**：数据类型转换函数，输入参数 expression 支持普通列、常量、标量函数及它们之间的四则运算，不支持 tag 列，只适用于 select 子句中。 -**参数范围**： +**返回结果类型**：CAST 中指定的类型（type_name），可以是 BIGINT、BIGINT UNSIGNED、BINARY、VARCHAR、NCHAR和TIMESTAMP。 -- oper : LT (小于)、GT（大于）、LE（小于等于）、GE（大于等于）、NE（不等于）、EQ（等于），不区分大小写。 -- val : 数值型 -- unit : 时间长度的单位，范围[1s、1m、1h ]，不足一个单位舍去。默认为 1s。 - -**返回结果类型**：整形。 - -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上。 - -**嵌套子查询支持**：不支持应用在子查询上。 - -**支持的版本**：2.6 开始的版本。 +**适用数据类型**：输入参数 expression 的类型可以是BLOB、MEDIUMBLOB和JSON外的所有类型 **使用说明**： -- 该函数可以应用在普通表上，在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname） -- 不能和窗口操作一起使用，例如 interval/state_window/session_window。 +- 对于不能支持的类型转换会直接报错。 +- 如果输入值为NULL则输出值也为NULL。 +- 对于类型支持但某些值无法正确转换的情况对应的转换后的值以转换函数输出为准。目前可能遇到的几种情况： + 1）字符串类型转换数值类型时可能出现的无效字符情况，例如"a"可能转为0，但不会报错。 + 2）转换到数值类型时，数值大于type_name可表示的范围时，则会溢出，但不会报错。 + 3）转换到字符串类型时，如果转换后长度超过type_name的长度，则会截断，但不会报错。 -**示例**： +#### TO_ISO8601 -``` -taos> select ts,dbig from statef2; - ts | dbig | -======================================================== -2021-10-15 00:31:33.000000000 | 1 | -2021-10-17 00:31:31.000000000 | NULL | -2021-12-24 00:31:34.000000000 | 2 | -2022-01-01 08:00:05.000000000 | 19 | -2022-01-01 08:00:06.000000000 | NULL | -2022-01-01 08:00:07.000000000 | 9 | -Query OK, 6 row(s) in set (0.002407s) - -taos> select stateDuration(dbig,GT,2) from statef2; -ts | dbig | stateduration(dbig,gt,2) | -=================================================================================== -2021-10-15 00:31:33.000000000 | 1 | -1 | -2021-10-17 00:31:31.000000000 | NULL | NULL | -2021-12-24 00:31:34.000000000 | 2 | -1 | -2022-01-01 08:00:05.000000000 | 19 | 0 | -2022-01-01 08:00:06.000000000 | NULL | NULL | -2022-01-01 08:00:07.000000000 | 9 | 2 | -Query OK, 6 row(s) in set (0.002613s) +```sql +SELECT TO_ISO8601(ts_val | ts_col) FROM { tb_name | stb_name } [WHERE clause]; ``` -## 时间函数 +**功能说明**：将 UNIX 时间戳转换成为 ISO8601 标准的日期时间格式，并附加客户端时区信息。 -从 2.6.0.0 版本开始，TDengine 查询引擎支持以下时间相关函数： +**返回结果数据类型**：BINARY 类型。 -### NOW +**适用数据类型**：UNIX 时间戳常量或是 TIMESTAMP 类型的列 + +**适用于**：表、超级表。 + +**使用说明**： + +- 如果输入是 UNIX 时间戳常量，返回格式精度由时间戳的位数决定; +- 如果输入是 TIMSTAMP 类型的列，返回格式的时间戳精度与当前 DATABASE 设置的时间精度一致。 + + +#### TO_JSON + +```sql +SELECT TO_JSON(str_literal) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**: 将字符串常量转换为 JSON 类型。 + +**返回结果数据类型**: JSON + +**适用数据类型**: JSON 字符串，形如 '{ "literal" : literal }'。'{}'表示空值。键必须为字符串字面量，值可以为数值字面量、字符串字面量、布尔字面量或空值字面量。str_literal中不支持转义符。 + +**适用于**: 表和超级表 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + + +#### TO_UNIXTIMESTAMP + +```sql +SELECT TO_UNIXTIMESTAMP(datetime_string | ts_col) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：将日期时间格式的字符串转换成为 UNIX 时间戳。 + +**返回结果数据类型**：长整型 INT64。 + +**应用字段**：字符串常量或是 BINARY/NCHAR 类型的列。 + +**适用于**：表、超级表。 + +**使用说明**： + +- 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 0。 +- 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 + + +### 时间和日期函数 + +时间和日期函数对时间戳类型进行操作。 + +所有返回当前时间的函数，如NOW、TODAY和TIMEZONE，在一条SQL语句中不论出现多少次都只会被计算一次。 + +#### NOW ```sql SELECT NOW() FROM { tb_name | stb_name } [WHERE clause]; @@ -1595,32 +500,63 @@ INSERT INTO tb_name VALUES (NOW(), ...); b(纳秒)、u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 -**示例**： + +#### TIMEDIFF ```sql -taos> SELECT NOW() FROM meters; - now() | -========================== - 2022-02-02 02:02:02.456 | -Query OK, 1 row(s) in set (0.002093s) - -taos> SELECT NOW() + 1h FROM meters; - now() + 1h | -========================== - 2022-02-02 03:02:02.456 | -Query OK, 1 row(s) in set (0.002093s) - -taos> SELECT COUNT(voltage) FROM d1001 WHERE ts < NOW(); - count(voltage) | -============================= - 5 | -Query OK, 5 row(s) in set (0.004475s) - -taos> INSERT INTO d1001 VALUES (NOW(), 10.2, 219, 0.32); -Query OK, 1 of 1 row(s) in database (0.002210s) +SELECT TIMEDIFF(ts_val1 | datetime_string1 | ts_col1, ts_val2 | datetime_string2 | ts_col2 [, time_unit]) FROM { tb_name | stb_name } [WHERE clause]; ``` -### TODAY +**功能说明**：计算两个时间戳之间的差值，并近似到时间单位 time_unit 指定的精度。 + +**返回结果数据类型**：长整型 INT64。 + +**应用字段**：UNIX 时间戳，日期时间格式的字符串，或者 TIMESTAMP 类型的列。 + +**适用于**：表、超级表。 + +**使用说明**： +- 支持的时间单位 time_unit 如下： + 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)。 +- 如果时间单位 time_unit 未指定， 返回的时间差值精度与当前 DATABASE 设置的时间精度一致。 + + +#### TIMETRUNCATE + +```sql +SELECT TIMETRUNCATE(ts_val | datetime_string | ts_col, time_unit) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：将时间戳按照指定时间单位 time_unit 进行截断。 + +**返回结果数据类型**：TIMESTAMP 时间戳类型。 + +**应用字段**：UNIX 时间戳，日期时间格式的字符串，或者 TIMESTAMP 类型的列。 + +**适用于**：表、超级表。 + +**使用说明**： +- 支持的时间单位 time_unit 如下： + 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)。 +- 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 + + +#### TIMEZONE + +```sql +SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：返回客户端当前时区信息。 + +**返回结果数据类型**：BINARY 类型。 + +**应用字段**：无 + +**适用于**：表、超级表。 + + +#### TODAY ```sql SELECT TODAY() FROM { tb_name | stb_name } [WHERE clause]; @@ -1642,212 +578,640 @@ INSERT INTO tb_name VALUES (TODAY(), ...); b(纳秒)，u(微秒)，a(毫秒)，s(秒)，m(分)，h(小时)，d(天)，w(周)。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 -**示例**： -```sql -taos> SELECT TODAY() FROM meters; - today() | -========================== - 2022-02-02 00:00:00.000 | -Query OK, 1 row(s) in set (0.002093s) +## 聚合函数 -taos> SELECT TODAY() + 1h FROM meters; - today() + 1h | -========================== - 2022-02-02 01:00:00.000 | -Query OK, 1 row(s) in set (0.002093s) +聚合函数为查询结果集的每一个分组返回单个结果行。可以由 GROUP BY 或窗口切分子句指定分组，如果没有，则整个查询结果集视为一个分组。 -taos> SELECT COUNT(voltage) FROM d1001 WHERE ts < TODAY(); - count(voltage) | -============================= - 5 | -Query OK, 5 row(s) in set (0.004475s) +TDengine 支持针对数据的聚合查询。提供如下聚合函数。 -taos> INSERT INTO d1001 VALUES (TODAY(), 10.2, 219, 0.32); -Query OK, 1 of 1 row(s) in database (0.002210s) +### AVG + +``` +SELECT AVG(field_name) FROM tb_name [WHERE clause]; ``` -### TIMEZONE +**功能说明**：统计表/超级表中某列的平均值。 -```sql -SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause]; -``` +**返回数据类型**：双精度浮点数 Double。 -**功能说明**：返回客户端当前时区信息。 - -**返回结果数据类型**：BINARY 类型。 - -**应用字段**：无 +**适用数据类型**：数值类型。 **适用于**：表、超级表。 -**示例**： -```sql -taos> SELECT TIMEZONE() FROM meters; - timezone() | -================================= - UTC (UTC, +0000) | -Query OK, 1 row(s) in set (0.002093s) +### COUNT + +``` +SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]; ``` -### TO_ISO8601 +**功能说明**：统计表/超级表中记录行数或某列的非空值个数。 -```sql -SELECT TO_ISO8601(ts_val | ts_col) FROM { tb_name | stb_name } [WHERE clause]; -``` +**返回数据类型**：长整型 INT64。 -**功能说明**：将 UNIX 时间戳转换成为 ISO8601 标准的日期时间格式，并附加客户端时区信息。 - -**返回结果数据类型**：BINARY 类型。 - -**应用字段**：UNIX 时间戳常量或是 TIMESTAMP 类型的列 +**适用数据类型**：应用全部字段。 **适用于**：表、超级表。 -**使用说明**： +**使用说明**: -- 如果输入是 UNIX 时间戳常量，返回格式精度由时间戳的位数决定; -- 如果输入是 TIMSTAMP 类型的列，返回格式的时间戳精度与当前 DATABASE 设置的时间精度一致。 +- 可以使用星号(\*)来替代具体的字段，使用星号(\*)返回全部记录数量。 +- 针对同一表的（不包含 NULL 值）字段查询结果均相同。 +- 如果统计对象是具体的列，则返回该列中非 NULL 值的记录数量。 -**示例**： -```sql -taos> SELECT TO_ISO8601(1643738400) FROM meters; - to_iso8601(1643738400) | -============================== - 2022-02-02T02:00:00+0800 | +### ELAPSED -taos> SELECT TO_ISO8601(ts) FROM meters; - to_iso8601(ts) | -============================== - 2022-02-02T02:00:00+0800 | - 2022-02-02T02:00:00+0800 | - 2022-02-02T02:00:00+0800 | +```mysql +SELECT ELAPSED(ts_primary_key [, time_unit]) FROM { tb_name | stb_name } [WHERE clause] [INTERVAL(interval [, offset]) [SLIDING sliding]]; ``` -### TO_UNIXTIMESTAMP +**功能说明**：elapsed函数表达了统计周期内连续的时间长度，和twa函数配合使用可以计算统计曲线下的面积。在通过INTERVAL子句指定窗口的情况下，统计在给定时间范围内的每个窗口内有数据覆盖的时间范围；如果没有INTERVAL子句，则返回整个给定时间范围内的有数据覆盖的时间范围。注意，ELAPSED返回的并不是时间范围的绝对值，而是绝对值除以time_unit所得到的单位个数。 -```sql -SELECT TO_UNIXTIMESTAMP(datetime_string | ts_col) FROM { tb_name | stb_name } [WHERE clause]; -``` +**返回结果类型**：Double -**功能说明**：将日期时间格式的字符串转换成为 UNIX 时间戳。 - -**返回结果数据类型**：长整型 INT64。 - -**应用字段**：字符串常量或是 BINARY/NCHAR 类型的列。 - -**适用于**：表、超级表。 - -**使用说明**： - -- 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 0。 -- 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 - -**示例**： - -```sql -taos> SELECT TO_UNIXTIMESTAMP("2022-02-02T02:00:00.000Z") FROM meters; -to_unixtimestamp("2022-02-02T02:00:00.000Z") | -============================================== - 1643767200000 | - -taos> SELECT TO_UNIXTIMESTAMP(col_binary) FROM meters; - to_unixtimestamp(col_binary) | -======================================== - 1643767200000 | - 1643767200000 | - 1643767200000 | -``` - -### TIMETRUNCATE - -```sql -SELECT TIMETRUNCATE(ts_val | datetime_string | ts_col, time_unit) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：将时间戳按照指定时间单位 time_unit 进行截断。 - -**返回结果数据类型**：TIMESTAMP 时间戳类型。 - -**应用字段**：UNIX 时间戳，日期时间格式的字符串，或者 TIMESTAMP 类型的列。 - -**适用于**：表、超级表。 - -**使用说明**： -- 支持的时间单位 time_unit 如下： - 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)。 -- 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 - -**示例**： - -```sql -taos> SELECT TIMETRUNCATE(1643738522000, 1h) FROM meters; - timetruncate(1643738522000, 1h) | -=================================== - 2022-02-02 02:00:00.000 | -Query OK, 1 row(s) in set (0.001499s) - -taos> SELECT TIMETRUNCATE("2022-02-02 02:02:02", 1h) FROM meters; - timetruncate("2022-02-02 02:02:02", 1h) | -=========================================== - 2022-02-02 02:00:00.000 | -Query OK, 1 row(s) in set (0.003903s) - -taos> SELECT TIMETRUNCATE(ts, 1h) FROM meters; - timetruncate(ts, 1h) | -========================== - 2022-02-02 02:00:00.000 | - 2022-02-02 02:00:00.000 | - 2022-02-02 02:00:00.000 | -Query OK, 3 row(s) in set (0.003903s) -``` - -### TIMEDIFF - -```sql -SELECT TIMEDIFF(ts_val1 | datetime_string1 | ts_col1, ts_val2 | datetime_string2 | ts_col2 [, time_unit]) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：计算两个时间戳之间的差值，并近似到时间单位 time_unit 指定的精度。 - -**返回结果数据类型**：长整型 INT64。 - -**应用字段**：UNIX 时间戳，日期时间格式的字符串，或者 TIMESTAMP 类型的列。 - -**适用于**：表、超级表。 - -**使用说明**： -- 支持的时间单位 time_unit 如下： - 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)。 -- 如果时间单位 time_unit 未指定， 返回的时间差值精度与当前 DATABASE 设置的时间精度一致。 +**适用数据类型**：Timestamp类型 **支持的版本**：2.6.0.0 及以后的版本。 -**示例**： +**适用于**: 表，超级表，嵌套查询的外层查询 + +**说明**： +- field_name参数只能是表的第一列，即timestamp主键列。 +- 按time_unit参数指定的时间单位返回，最小是数据库的时间分辨率。time_unit参数未指定时，以数据库的时间分辨率为时间单位。 +- 可以和interval组合使用，返回每个时间窗口的时间戳差值。需要特别注意的是，除第一个时间窗口和最后一个时间窗口外，中间窗口的时间戳差值均为窗口长度。 +- order by asc/desc不影响差值的计算结果。 +- 对于超级表，需要和group by tbname子句组合使用，不可以直接使用。 +- 对于普通表，不支持和group by子句组合使用。 +- 对于嵌套查询，仅当内层查询会输出隐式时间戳列时有效。例如select elapsed(ts) from (select diff(value) from sub1)语句，diff函数会让内层查询输出隐式时间戳列，此为主键列，可以用于elapsed函数的第一个参数。相反，例如select elapsed(ts) from (select * from sub1) 语句，ts列输出到外层时已经没有了主键列的含义，无法使用elapsed函数。此外，elapsed函数作为一个与时间线强依赖的函数，形如select elapsed(ts) from (select diff(value) from st group by tbname)尽管会返回一条计算结果，但并无实际意义，这种用法后续也将被限制。 +- 不支持与leastsquares、diff、derivative、top、bottom、last_row、interp等函数混合使用。 + +### LEASTSQUARES + +``` +SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]; +``` + +**功能说明**：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val 是自变量初始值，step_val 是自变量的步长值。 + +**返回数据类型**：字符串表达式（斜率, 截距）。 + +**适用数据类型**：field_name 必须是数值类型。 + +**适用于**：表。 + + +### MODE + +``` +SELECT MODE(field_name) FROM tb_name [WHERE clause]; +``` + +**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，输出空。不能匹配标签、时间戳输出。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**： 数值类型。 + +**适用于**：表和超级表。 + + +### SPREAD + +``` +SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：统计表/超级表中某列的最大值和最小值之差。 + +**返回数据类型**：双精度浮点数。 + +**适用数据类型**：数值类型或TIMESTAMP类型。 + +**适用于**：表和超级表。 + + +### STDDEV + +``` +SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; +``` + +**功能说明**：统计表中某列的均方差。 + +**返回数据类型**：双精度浮点数 Double。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + + +### SUM + +``` +SELECT SUM(field_name) FROM tb_name [WHERE clause]; +``` + +**功能说明**：统计表/超级表中某列的和。 + +**返回数据类型**：双精度浮点数 Double 和长整型 INT64。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + + +### HYPERLOGLOG + +``` +SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**： + - 采用 hyperloglog 算法，返回某列的基数。该算法在数据量很大的情况下，可以明显降低内存的占用，但是求出来的基数是个估算值，标准误差（标准误差是多次实验，每次的平均数的标准差，不是与真实结果的误差）为 0.81%。 + - 在数据量较少的时候该算法不是很准确，可以使用 select count(data) from (select unique(col) as data from table) 的方法。 + +**返回结果类型**：整形。 + +**适用数据类型**：任何类型。 + +**适用于**：表和超级表。 + + +### HISTOGRAM + +``` +SELECT HISTOGRAM(field_name，bin_type, bin_description, normalized) FROM tb_name [WHERE clause]; +``` + +**功能说明**：统计数据按照用户指定区间的分布。 + +**返回结果类型**：如归一化参数 normalized 设置为 1，返回结果为双精度浮点类型 DOUBLE，否则为长整形 INT64。 + +**适用数据类型**：数值型字段。 + +**适用于**: 表和超级表。 + +**详细说明**： +1. bin_type 用户指定的分桶类型, 有效输入类型为"user_input“, ”linear_bin", "log_bin"。 +2. bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)： + - "user_input": "[1, 3, 5, 7]" + 用户指定 bin 的具体数值。 + + - "linear_bin": "{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" + "start" 表示数据起始点，"width" 表示每次 bin 偏移量, "count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点跟终点， + 生成区间为[-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]。 + + - "log_bin": "{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" + "start" 表示数据起始点，"factor" 表示按指数递增的因子，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点跟终点， + 生成区间为[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]。 +3. normalized 是否将返回结果归一化到 0~1 之间 。有效输入为 0 和 1。 + + +## 选择函数 + +选择函数根据语义在查询结果集中选择一行或多行结果返回。用户可以同时指定输出 ts 列或其他列（包括 tbname 和标签列），这样就可以方便地知道被选出的值是源于哪个数据行的。 + +### APERCENTILE + +``` +SELECT APERCENTILE(field_name, P[, algo_type]) +FROM { tb_name | stb_name } [WHERE clause] +``` + +**功能说明**：统计表/超级表中指定列的值的近似百分比分位数，与 PERCENTILE 函数相似，但是返回近似结果。 + +**返回数据类型**： 双精度浮点数 Double。 + +**适用数据类型**：数值类型。P值范围是[0,100]，当为0时等同于MIN，为100时等同于MAX。如果不指定 algo_type 则使用默认算法 。 + +**适用于**：表、超级表。 + +### BOTTOM + +``` +SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：统计表/超级表中某列的值最小 _k_ 个非 NULL 值。如果多条数据取值一样，全部取用又会超出 k 条限制时，系统会从相同值中随机选取符合要求的数量返回。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + +**使用说明**: + +- *k*值取值范围 1≤*k*≤100； +- 系统同时返回该记录关联的时间戳列； +- 限制：BOTTOM 函数不支持 FILL 子句。 + + +### INTERP + +``` +SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [ RANGE(timestamp1,timestamp2) ] [EVERY(interval)] [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})]; +``` + +**功能说明**：返回指定时间截面指定列的记录值或插值。 + +**返回数据类型**：同字段类型。 + +**适用数据类型**：数值类型。 + +**适用于**：表、超级表。 + +**使用说明** + +- INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。 +- INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 +- INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1<=timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。如果没有指定 RANGE，那么满足过滤条件的输入数据中第一条记录的 timestamp 即为 timestamp1，最后一条记录的 timestamp 即为 timestamp2，同样也满足 timestamp1 <= timestamp2。 +- INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。如果没有指定 EVERY，则默认窗口大小为无穷大，即从 timestamp1 开始只有一个窗口。 +- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值，如果没有 FILL 字段则默认不插值，即输出为原始记录值或不输出（原始记录不存在）。 +- INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 group by tbname 一起使用，当作用嵌套查询外层时内层子查询不能含 GROUP BY 信息。 +- INTERP 的插值结果不受 ORDER BY timestamp 的影响，ORDER BY timestamp 只影响输出结果的排序。 + +### LAST + +``` +SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：统计表/超级表中某列的值最后写入的非 NULL 值。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：所有字段。 + +**适用于**：表和超级表。 + +**使用说明**: + +- 如果要返回各个列的最后（时间戳最大）一个非 NULL 值，可以使用 LAST(\*)； +- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；如果结果集中所有列全部为 NULL 值，则不返回结果。 +- 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 + + +### LAST_ROW + +``` +SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; +``` + +**功能说明**：返回表/超级表的最后一条记录。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：所有字段。 + +**适用于**：表和超级表。 + +**使用说明**： + +- 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 +- 不能与 INTERVAL 一起使用。 + +### MAX + +``` +SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：统计表/超级表中某列的值最大值。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + + +### MIN + +``` +SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; +``` + +**功能说明**：统计表/超级表中某列的值最小值。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + + +### PERCENTILE + +``` +SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; +``` + +**功能说明**：统计表中某列的值百分比分位数。 + +**返回数据类型**： 双精度浮点数 Double。 + +**应用字段**：数值类型。 + +**适用于**：表。 + +**使用说明**：*P*值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX。 + +### FIRST + +``` +SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：统计表/超级表中某列的值最先写入的非 NULL 值。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：所有字段。 + +**适用于**：表和超级表。 + +**使用说明**: + +- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)； +- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL； +- 如果结果集中所有列全部为 NULL 值，则不返回结果。 + +### TAIL + +``` +SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause]; +``` + +**功能说明**：返回跳过最后 offset_val 个，然后取连续 k 个记录，不忽略 NULL 值。offset_val 可以不输入。此时返回最后的 k 个记录。当有 offset_val 输入的情况下，该函数功能等效于 `order by ts desc LIMIT k OFFSET offset_val`。 + +**参数范围**：k: [1,100] offset_val: [0,100]。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：适合于除时间主列外的任何类型。 + +**适用于**：表、超级表。 + + +### TOP + +``` +SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**： 统计表/超级表中某列的值最大 _k_ 个非 NULL 值。如果多条数据取值一样，全部取用又会超出 k 条限制时，系统会从相同值中随机选取符合要求的数量返回。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：数值类型。 + +**适用于**：表、超级表。 + +**使用说明**: + +- *k*值取值范围 1≤*k*≤100； +- 系统同时返回该记录关联的时间戳列； +- 限制：TOP 函数不支持 FILL 子句。 + +### UNIQUE + +``` +SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause]; +``` + +**功能说明**：返回该列的数值首次出现的值。该函数功能与 distinct 相似，但是可以匹配标签和时间戳信息。可以针对除时间列以外的字段进行查询，可以匹配标签和时间戳，其中的标签和时间戳是第一次出现时刻的标签和时间戳。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：适合于除时间类型以外的字段。 + +**适用于**: 表和超级表。 + + +## 时序数据特有函数 + +时序数据特有函数是 TDengine 为了满足时序数据的查询场景而量身定做出来的。在通用数据库中，实现类似功能通常需要复杂的查询语法，且效率很低。TDengine 以函数的方式内置了这些功能，最大程度的减轻了用户的使用成本。 + +### CSUM ```sql -taos> SELECT TIMEDIFF(1643738400000, 1643742000000) FROM meters; - timediff(1643738400000, 1643742000000) | -========================================= - 3600000 | -Query OK, 1 row(s) in set (0.002553s) -taos> SELECT TIMEDIFF(1643738400000, 1643742000000, 1h) FROM meters; - timediff(1643738400000, 1643742000000, 1h) | -============================================= - 1 | -Query OK, 1 row(s) in set (0.003726s) - -taos> SELECT TIMEDIFF("2022-02-02 03:00:00", "2022-02-02 02:00:00", 1h) FROM meters; - timediff("2022-02-02 03:00:00", "2022-02-02 02:00:00", 1h) | -============================================================= - 1 | -Query OK, 1 row(s) in set (0.001937s) - -taos> SELECT TIMEDIFF(ts_col1, ts_col2, 1h) FROM meters; - timediff(ts_col1, ts_col2, 1h) | -=================================== - 1 | -Query OK, 1 row(s) in set (0.001937s) + SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` + +**功能说明**：累加和（Cumulative sum），输出行与输入行数相同。 + +**返回结果类型**： 输入列如果是整数类型返回值为长整型 （int64_t），浮点数返回值为双精度浮点数（Double）。无符号整数类型返回值为无符号长整型（uint64_t）。 返回结果中同时带有每行记录对应的时间戳。 + +**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在标签之上。 + +**嵌套子查询支持**： 适用于内层查询和外层查询。 + +**适用于**：表和超级表 + +**使用说明**： + + - 不支持 +、-、*、/ 运算，如 csum(col1) + csum(col2)。 + - 只能与聚合（Aggregation）函数一起使用。 该函数可以应用在普通表和超级表上。 + - 使用在超级表上的时候，需要搭配 Group by tbname使用，将结果强制规约到单个时间线。 + + +### DERIVATIVE + +``` +SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHERE clause]; +``` + +**功能说明**：统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定，最小可以是 1 秒（1s）；ignore_negative 参数的值可以是 0 或 1，为 1 时表示忽略负值。 + +**返回数据类型**：双精度浮点数。 + +**适用数据类型**：数值类型。 + +**适用于**：表、超级表 + +**使用说明**: DERIVATIVE 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 + + +### DIFF + + ```sql + SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHERE clause]; + ``` + +**功能说明**：统计表中某列的值与前一行对应值的差。 ignore_negative 取值为 0|1 , 可以不填，默认值为 0. 不忽略负值。ignore_negative 为 1 时表示忽略负数。 + +**返回数据类型**：同应用字段。 + +**适用数据类型**：数值类型。 + +**适用于**：表、超级表。 + +**使用说明**: 输出结果行数是范围内总行数减一，第一行没有结果输出。 + + +### IRATE + +``` +SELECT IRATE(field_name) FROM tb_name WHERE clause; +``` + +**功能说明**：计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率；如果这两个值呈递减关系，那么只取最后一个数用于计算，而不是使用二者差值。 + +**返回数据类型**：双精度浮点数 Double。 + +**适用数据类型**：数值类型。 + +**适用于**：表、超级表。 + +### MAVG + +```sql + SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause] +``` + + **功能说明**： 计算连续 k 个值的移动平均数（moving average）。如果输入行数小于 k，则无结果输出。参数 k 的合法输入范围是 1≤ k ≤ 1000。 + + **返回结果类型**： 返回双精度浮点数类型。 + + **适用数据类型**： 不能应用在 timestamp、binary、nchar、bool 类型上；在超级表查询中使用时，不能应用在标签之上。 + + **嵌套子查询支持**： 适用于内层查询和外层查询。 + + **适用于**：表和超级表 + + **使用说明**： + + - 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1); + - 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用； + - 使用在超级表上的时候，需要搭配 Group by tbname使用，将结果强制规约到单个时间线。 + +### SAMPLE + +```sql + SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause] +``` + + **功能说明**： 获取数据的 k 个采样值。参数 k 的合法输入范围是 1≤ k ≤ 1000。 + + **返回结果类型**： 同原始数据类型， 返回结果中带有该行记录的时间戳。 + + **适用数据类型**： 在超级表查询中使用时，不能应用在标签之上。 + + **嵌套子查询支持**： 适用于内层查询和外层查询。 + + **适用于**：表和超级表 + + **使用说明**： + + - 不能参与表达式计算；该函数可以应用在普通表和超级表上； + - 使用在超级表上的时候，需要搭配 Group by tbname 使用，将结果强制规约到单个时间线。 + +### STATECOUNT + +``` +SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：返回满足某个条件的连续记录的个数，结果作为新的一列追加在每行后面。条件根据参数计算，如果条件为 true 则加 1，条件为 false 则重置为-1，如果数据为 NULL，跳过该条数据。 + +**参数范围**： + +- oper : LT (小于)、GT（大于）、LE（小于等于）、GE（大于等于）、NE（不等于）、EQ（等于），不区分大小写。 +- val : 数值型 + +**返回结果类型**：整形。 + +**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上。 + +**嵌套子查询支持**：不支持应用在子查询上。 + +**适用于**：表和超级表。 + +**使用说明**： + +- 该函数可以应用在普通表上，在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname） +- 不能和窗口操作一起使用，例如 interval/state_window/session_window。 + + +### STATEDURATION + +```sql +SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：返回满足某个条件的连续记录的时间长度，结果作为新的一列追加在每行后面。条件根据参数计算，如果条件为 true 则加上两个记录之间的时间长度（第一个满足条件的记录时间长度记为 0），条件为 false 则重置为-1，如果数据为 NULL，跳过该条数据。 + +**参数范围**： + +- oper : LT (小于)、GT（大于）、LE（小于等于）、GE（大于等于）、NE（不等于）、EQ（等于），不区分大小写。 +- val : 数值型 +- unit : 时间长度的单位，范围[1s、1m、1h ]，不足一个单位舍去。默认为 1s。 + +**返回结果类型**：整形。 + +**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上。 + +**嵌套子查询支持**：不支持应用在子查询上。 + +**适用于**：表和超级表。 + +**使用说明**： + +- 该函数可以应用在普通表上，在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname） +- 不能和窗口操作一起使用，例如 interval/state_window/session_window。 + + +### TWA + +``` +SELECT TWA(field_name) FROM tb_name WHERE clause; +``` + +**功能说明**：时间加权平均函数。统计表中某列在一段时间内的时间加权平均。 + +**返回数据类型**：双精度浮点数 Double。 + +**适用数据类型**：数值类型。 + +**适用于**：表、超级表。 + +**使用说明**： TWA 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 + + +## 系统信息函数 + +### DATABASE + +``` +SELECT DATABASE(); +``` + +**说明**：返回当前登录的数据库。如果登录的时候没有指定默认数据库，且没有使用USE命令切换数据库，则返回NULL。 + + +### CLIENT_VERSION + +``` +SELECT CLIENT_VERSION(); +``` + +**说明**：返回客户端版本。 + +### SERVER_VERSION + +``` +SELECT SERVER_VERSION(); +``` + +**说明**：返回服务端版本。 + +### SERVER_STATUS + +``` +SELECT SERVER_VERSION(); +``` + +**说明**：返回服务端当前的状态。 From 9283c5c3e9f551f384801202e1c8854db5853229 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 11:18:10 +0800 Subject: [PATCH 42/57] fix:fix error in new tag format --- source/libs/parser/src/parInsert.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 412afee1f2..149302e125 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -781,7 +781,7 @@ static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTa } static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, - int16_t timePrec, char* tmpTokenBuf, STagVal *val, SMsgBuf* pMsgBuf) { + int16_t timePrec, STagVal *val, SMsgBuf* pMsgBuf) { int64_t iv; uint64_t uv; char* endptr = NULL; @@ -795,7 +795,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, } val->cid = pSchema->colId; - val->type = pSchema->bytes; + val->type = pSchema->type; switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { @@ -955,12 +955,6 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, val->nData = output; break; } - case TSDB_DATA_TYPE_JSON: { - if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); - } - //return func(pMsgBuf, pToken->z, pToken->n, param); - } case TSDB_DATA_TYPE_TIMESTAMP: { if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); @@ -971,7 +965,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, } } - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } // pSql -> tag1_value, ...) @@ -1021,7 +1015,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint isJson = true; }else{ STagVal val = {0}; - code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, &val, &pCxt->msg); + code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(tmpTokenBuf); goto end; From 0c44be16af50ca5bbc48f34df01fee9d2c7647d5 Mon Sep 17 00:00:00 2001 From: Bo Ding Date: Wed, 1 Jun 2022 11:27:52 +0800 Subject: [PATCH 43/57] docs: fix minor md grammar error --- docs-en/12-taos-sql/12-keywords.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs-en/12-taos-sql/12-keywords.md b/docs-en/12-taos-sql/12-keywords.md index 8f045f4801..ed0c96b4e4 100644 --- a/docs-en/12-taos-sql/12-keywords.md +++ b/docs-en/12-taos-sql/12-keywords.md @@ -56,6 +56,7 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam Get the table name and tag values of all subtables in a STable. ```mysql SELECT TBNAME, location FROM meters; +``` Count the number of subtables in a STable. ```mysql From 8908407918e22a935c28c5af0a30499de6fa4b27 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 1 Jun 2022 11:33:15 +0800 Subject: [PATCH 44/57] fix(os): add log data dir configure --- include/os/osDir.h | 11 +++++++++++ source/dnode/vnode/test/tsdbSmaTest.cpp | 2 +- source/libs/catalog/test/catalogTests.cpp | 2 +- source/libs/qworker/test/qworkerTests.cpp | 2 +- source/libs/scalar/test/filter/filterTests.cpp | 2 +- source/libs/scalar/test/scalar/scalarTests.cpp | 2 +- source/libs/scheduler/test/schedulerTests.cpp | 2 +- source/os/src/osEnv.c | 4 ++-- 8 files changed, 19 insertions(+), 8 deletions(-) diff --git a/include/os/osDir.h b/include/os/osDir.h index a4c686e280..9019d4f802 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -33,8 +33,19 @@ extern "C" { #ifdef WINDOWS #define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\" +#define TD_CFG_DIR_PATH "C:\\TDengine\\cfg\\" +#define TD_DATA_DIR_PATH "C:\\TDengine\\data\\" +#define TD_LOG_DIR_PATH "C:\\TDengine\\log\\" +#elif defined(_TD_DARWIN_64) +#define TD_TMP_DIR_PATH "/tmp/taosd/" +#define TD_CFG_DIR_PATH "/usr/local/etc/taos/" +#define TD_DATA_DIR_PATH "/usr/local/var/lib/taos/" +#define TD_LOG_DIR_PATH "/usr/local/var/log/taos/" #else #define TD_TMP_DIR_PATH "/tmp/" +#define TD_CFG_DIR_PATH "/etc/taos/" +#define TD_DATA_DIR_PATH "/var/lib/taos/" +#define TD_LOG_DIR_PATH "/var/log/taos/" #endif typedef struct TdDir *TdDirPtr; diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index ab617cb186..4d2741f751 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -368,7 +368,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { SDiskCfg pDisks = {0}; pDisks.level = 0; pDisks.primary = 1; - strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN); + strncpy(pDisks.dir, TD_DATA_DIR_PATH, TSDB_FILENAME_LEN); int32_t numOfDisks = 1; pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); EXPECT_NE(pTsdb->pTfs, nullptr); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 81d206a0f3..19c5bb6dcd 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -137,7 +137,7 @@ void ctgTestInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsLogDir, TD_LOG_DIR_PATH); ctgdEnableDebug("api"); ctgdEnableDebug("meta"); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 1b959fbe63..16dcd7b6e0 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -108,7 +108,7 @@ void qwtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsLogDir, TD_LOG_DIR_PATH); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 59c3104e96..7fb1ffbd64 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -60,7 +60,7 @@ void flttInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsLogDir, TD_LOG_DIR_PATH); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 6a32c65775..169f01c053 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -74,7 +74,7 @@ void scltInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsLogDir, TD_LOG_DIR_PATH); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index d5c834e5cf..4bf114ad8f 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -79,7 +79,7 @@ void schtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsLogDir, TD_LOG_DIR_PATH); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 6746025f78..6ae3d8a0c0 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -70,11 +70,11 @@ void osDefaultInit() { #elif defined(_TD_DARWIN_64) if (configDir[0] == 0) { - strcpy(configDir, "/tmp/taosd"); + strcpy(configDir, "/usr/local/etc/taos"); } strcpy(tsDataDir, "/usr/local/var/lib/taos"); strcpy(tsLogDir, "/usr/local/var/log/taos"); - strcpy(tsTempDir, "/usr/local/etc/taos"); + strcpy(tsTempDir, "/tmp/taosd"); strcpy(tsOsName, "Darwin"); #else From 2f87b0511b87554072b0b3f3ac5c98a4deef2b46 Mon Sep 17 00:00:00 2001 From: Bo Ding Date: Wed, 1 Jun 2022 11:40:06 +0800 Subject: [PATCH 45/57] docs: fix minor md grammar error (#13345) --- docs-cn/12-taos-sql/12-keywords.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs-cn/12-taos-sql/12-keywords.md b/docs-cn/12-taos-sql/12-keywords.md index 0e8e1edfee..5c68e5da7e 100644 --- a/docs-cn/12-taos-sql/12-keywords.md +++ b/docs-cn/12-taos-sql/12-keywords.md @@ -93,10 +93,13 @@ title: TDengine 参数限制与保留关键字 `TBNAME` 可以视为超级表中一个特殊的标签，代表子表的表名。 获取一个超级表所有的子表名及相关的标签信息： + ```mysql SELECT TBNAME, location FROM meters; +``` 统计超级表下辖子表数量： + ```mysql SELECT COUNT(TBNAME) FROM meters; ``` From 2cbb30a0db38ae1fd2e733d06ac544676aa76212 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 12:11:31 +0800 Subject: [PATCH 46/57] refactor: adjust error msg --- include/util/taoserror.h | 3 +-- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/util/src/terror.c | 7 +++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 3669348a18..be82e14708 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -183,8 +183,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_BNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0357) #define TSDB_CODE_MND_TOO_FEW_MNODES TAOS_DEF_ERROR_CODE(0, 0x0358) #define TSDB_CODE_MND_TOO_MANY_MNODES TAOS_DEF_ERROR_CODE(0, 0x0359) -#define TSDB_CODE_MND_MNODE_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x035A) -#define TSDB_CODE_MND_CANT_DROP_MASTER TAOS_DEF_ERROR_CODE(0, 0x035B) +#define TSDB_CODE_MND_CANT_DROP_MASTER TAOS_DEF_ERROR_CODE(0, 0x035A) // mnode-acct #define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 73d7eff04a..aeff018aa8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -613,7 +613,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId); if (pMObj != NULL) { - terrno = TSDB_CODE_MND_MNODE_DEPLOYED; + terrno = TSDB_CODE_MND_MNODE_NOT_EXIST; goto DROP_DNODE_OVER; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a54d8c2c03..c1f5c92be7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -186,10 +186,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_ALREADY_EXIST, "Snode already exists" TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_NOT_EXIST, "Snode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_ALREADY_EXIST, "Bnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_NOT_EXIST, "Bnode not there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_FEW_MNODES, "Too few mnodes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "Too many mnodes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_DEPLOYED, "Mnode deployed in this dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CANT_DROP_MASTER, "Can't drop mnode which is master") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_FEW_MNODES, "The replicas of mnode cannot less than 1") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "The replicas of mnode cannot exceed 3") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CANT_DROP_MASTER, "Can't drop mnode which is LEADER") // mnode-acct TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") From 094aaa207778b675145bc778f7950fa52cf15424 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 12:44:28 +0800 Subject: [PATCH 47/57] fix:fix error in new tag format --- source/libs/parser/src/parInsert.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 149302e125..529e10be65 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1733,7 +1733,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN goto end; } }else{ - STagVal val = {0}; + STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ val.pData = (uint8_t*)bind[c].buffer; val.nData = colLen; @@ -2120,7 +2120,7 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; SSmlKv* kv = taosArrayGetP(cols, i); - STagVal val = {0}; + STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ val.pData = (uint8_t *)kv->value; val.nData = kv->length; @@ -2131,10 +2131,10 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), pTagSchema->bytes - VARSTR_HEADER_SIZE, &output)) { if (errno == E2BIG) { taosMemoryFree(p); - code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); goto end; } char buf[512] = {0}; From 133069e72abb60b661a07ed0094dbe1b5c035579 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 1 Jun 2022 12:57:31 +0800 Subject: [PATCH 48/57] chore: update for taosdump help refine (#13357) for 3.0 [TD-16151] --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index 4d83d8c629..717f5aaa5f 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 4d83d8c62973506f760bcaa3a33f4665ed9046d0 +Subproject commit 717f5aaa5f0a1b4d92bb2ae68858fec554fb5eda From eb82ac04a0ca86daebc108d6b3cf0e707debad9d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 13:06:25 +0800 Subject: [PATCH 49/57] fix:fix error in new tag format --- source/libs/executor/src/scanoperator.c | 3 ++- source/libs/parser/src/parInsert.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 89340f2b85..c5d19981cf 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -327,7 +327,8 @@ void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_ for (int32_t i = 0; i < pBlock->info.rows; ++i) { colDataAppend(pColInfoData, i, data, (data == NULL)); } - if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ + if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL && + IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ taosMemoryFree(data); } } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 529e10be65..4cc3da5beb 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -2126,7 +2126,7 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p val.nData = kv->length; }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ int32_t output = 0; - void *p = taosMemoryCalloc(1, kv->length * TSDB_NCHAR_SIZE); + void *p = taosMemoryCalloc(1, pTagSchema->bytes - VARSTR_HEADER_SIZE); if(p == NULL){ code = TSDB_CODE_OUT_OF_MEMORY; goto end; From d86d8803be02a081a9a0b0b1b0bfd0afb61232e9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 1 Jun 2022 13:14:28 +0800 Subject: [PATCH 50/57] stmt new api --- include/client/taos.h | 1 + source/libs/parser/src/parInsert.c | 18 +- source/libs/parser/src/parTranslater.c | 3 +- tests/script/api/batchprepare.c | 318 +++++++++++++++++++------ 4 files changed, 263 insertions(+), 77 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 94e6221b31..97b94e0a53 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -142,6 +142,7 @@ DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags); DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_set_tags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags); DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int* fieldNum, TAOS_FIELD_E** fields); DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int* fieldNum, TAOS_FIELD_E** fields); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index a6e2473c4f..8a843c2c1a 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1525,14 +1525,14 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN SKvParam param = {.builder = &tagBuilder}; for (int c = 0; c < tags->numOfBound; ++c) { + SSchema* pTagSchema = &pSchema[tags->boundColumns[c]]; + param.schema = pTagSchema; + if (bind[c].is_null && bind[c].is_null[0]) { KvRowAppend(&pBuf, NULL, 0, ¶m); continue; } - SSchema* pTagSchema = &pSchema[tags->boundColumns[c]]; - param.schema = pTagSchema; - int32_t colLen = pTagSchema->bytes; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; @@ -1731,16 +1731,16 @@ int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_ return TSDB_CODE_OUT_OF_MEMORY; } - SSchema* pSchema = &pSchema[boundInfo->boundColumns[0]]; - if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type) { + SSchema* schema = &pSchema[boundInfo->boundColumns[0]]; + if (TSDB_DATA_TYPE_TIMESTAMP == schema->type) { (*fields)[0].precision = timePrec; } for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { - SSchema* pSchema = &pSchema[boundInfo->boundColumns[i]]; - strcpy((*fields)[i].name, pSchema->name); - (*fields)[i].type = pSchema->type; - (*fields)[i].bytes = pSchema->bytes; + schema = &pSchema[boundInfo->boundColumns[i]]; + strcpy((*fields)[i].name, schema->name); + (*fields)[i].type = schema->type; + (*fields)[i].bytes = schema->bytes; } } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0dda71023f..a64d6de71f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -830,7 +830,8 @@ static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNo if (!IS_VAR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); } - if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || !IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) { + if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || + ((!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) && (((SExprNode*)(pOp->pRight))->resType.type != TSDB_DATA_TYPE_NULL))) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 2ded58a979..7dd7621d0b 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -10,6 +10,9 @@ #include "../../../include/client/taos.h" #define FUNCTION_TEST_IDX 1 +#define TIME_PRECISION_MILLI 0 +#define TIME_PRECISION_MICRO 1 +#define TIME_PRECISION_NANO 2 int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; @@ -32,6 +35,8 @@ typedef enum { BP_BIND_COL, } BP_BIND_TYPE; +#define BP_BIND_TYPE_STR(t) (((t) == BP_BIND_COL) ? "column" : "tag") + OperInfo operInfo[] = { {">", 2, false}, {">=", 2, false}, @@ -57,11 +62,12 @@ FuncInfo funcInfo[] = { {"min", 1}, }; +#define BP_STARTUP_TS 1591060628000 + char *bpStbPrefix = "st"; char *bpTbPrefix = "t"; int32_t bpDefaultStbId = 1; - - +int64_t bpTs; //char *operatorList[] = {">", ">=", "<", "<=", "=", "<>", "in", "not in"}; //char *varoperatorList[] = {">", ">=", "<", "<=", "=", "<>", "in", "not in", "like", "not like", "match", "nmatch"}; @@ -188,8 +194,10 @@ typedef struct { bool printCreateTblSql; bool printQuerySql; bool printStmtSql; + bool printVerbose; bool autoCreateTbl; bool numericParam; + uint8_t precision; int32_t rowNum; //row num for one table int32_t bindColNum; int32_t bindTagNum; @@ -209,12 +217,15 @@ typedef struct { int32_t caseRunNum; // total run case num } CaseCtrl; -#if 1 +#if 0 CaseCtrl gCaseCtrl = { // default + .precision = TIME_PRECISION_MICRO, .bindNullNum = 0, .printCreateTblSql = false, .printQuerySql = true, .printStmtSql = true, + .printVerbose = false, + .printRes = false, .autoCreateTbl = false, .numericParam = false, .rowNum = 0, @@ -230,7 +241,6 @@ CaseCtrl gCaseCtrl = { // default .funcIdxListNum = 0, .funcIdxList = NULL, .checkParamNum = false, - .printRes = false, .runTimes = 0, .caseIdx = -1, .caseNum = -1, @@ -240,26 +250,35 @@ CaseCtrl gCaseCtrl = { // default #endif -#if 0 +#if 1 CaseCtrl gCaseCtrl = { + .precision = TIME_PRECISION_MILLI, .bindNullNum = 0, - .printCreateTblSql = true, + .printCreateTblSql = false, .printQuerySql = true, .printStmtSql = true, + .printVerbose = false, + .printRes = true, .autoCreateTbl = false, + .numericParam = false, .rowNum = 0, .bindColNum = 0, .bindTagNum = 0, .bindRowNum = 0, + .bindColTypeNum = 0, + .bindColTypeList = NULL, .bindTagTypeNum = 0, .bindTagTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .funcIdxListNum = 0, + .funcIdxList = NULL, .checkParamNum = false, - .printRes = false, .runTimes = 0, - .caseIdx = 1, - .caseNum = 1, + .caseIdx = -1, + .caseNum = -1, .caseRunIdx = -1, - .caseRunNum = 1, + .caseRunNum = -1, }; #endif @@ -891,7 +910,6 @@ int32_t prepareColData(BP_BIND_TYPE bType, BindData *data, int32_t bindIdx, int3 int32_t prepareInsertData(BindData *data) { - static int64_t tsData = 1591060628000; uint64_t allRowNum = gCurCase->rowNum * gCurCase->tblNum; data->colNum = 0; @@ -918,7 +936,7 @@ int32_t prepareInsertData(BindData *data) { } for (int32_t i = 0; i < allRowNum; ++i) { - data->tsData[i] = tsData++; + data->tsData[i] = bpTs++; data->boolData[i] = (bool)(i % 2); data->tinyData[i] = (int8_t)i; data->utinyData[i] = (uint8_t)(i+1); @@ -956,7 +974,6 @@ int32_t prepareInsertData(BindData *data) { } int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { - static int64_t tsData = 1591060628000; uint64_t bindNum = gCurCase->rowNum / gCurCase->bindRowNum; data->colNum = 0; @@ -982,7 +999,7 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { } for (int32_t i = 0; i < bindNum; ++i) { - data->tsData[i] = tsData + tblIdx*gCurCase->rowNum + rand()%gCurCase->rowNum; + data->tsData[i] = bpTs + tblIdx*gCurCase->rowNum + rand()%gCurCase->rowNum; data->boolData[i] = (bool)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); data->tinyData[i] = (int8_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); data->utinyData[i] = (uint8_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); @@ -1014,7 +1031,6 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { - static int64_t tsData = 1591060628000; uint64_t bindNum = gCurCase->rowNum / gCurCase->bindRowNum; data->colNum = 0; @@ -1040,7 +1056,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { } for (int32_t i = 0; i < bindNum; ++i) { - data->tsData[i] = tsData + tblIdx*gCurCase->rowNum + rand()%gCurCase->rowNum; + data->tsData[i] = bpTs + tblIdx*gCurCase->rowNum + rand()%gCurCase->rowNum; data->boolData[i] = (bool)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); data->tinyData[i] = (int8_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); data->utinyData[i] = (uint8_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); @@ -1202,39 +1218,7 @@ int32_t bpAppendValueString(char *buf, int type, void *value, int32_t valueLen, } -int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { - static int32_t n = 0; - - if (gCurCase->bindRowNum > 1) { - if (0 == (n++%2)) { - if (taos_stmt_bind_param_batch(stmt, bind)) { - printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - } else { - for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { - if (taos_stmt_bind_single_param_batch(stmt, bind++, i)) { - printf("!!!taos_stmt_bind_single_param_batch error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - } - } - } else { - if (0 == (n++%2)) { - if (taos_stmt_bind_param_batch(stmt, bind)) { - printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - } else { - if (taos_stmt_bind_param(stmt, bind)) { - printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - } - } - return 0; -} void bpCheckIsInsert(TAOS_STMT *stmt, int32_t insert) { int32_t isInsert = 0; @@ -1280,15 +1264,12 @@ void bpCheckAffectedRowsOnce(TAOS_STMT *stmt, int32_t expectedNum) { } void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_BIND* bind) { - TAOS_RES* res = taos_stmt_use_result(stmt); - int32_t sqlResNum = 0; - int32_t stmtResNum = 0; - bpFetchRows(res, gCaseCtrl.printRes, &stmtResNum); - + // query using sql char sql[1024]; int32_t len = 0; char* p = stmtSql; char* s = NULL; + int32_t sqlResNum = 0; for (int32_t i = 0; true; ++i, p=s+1) { s = strchr(p, '?'); @@ -1313,6 +1294,12 @@ void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_B } bpExecQuery(taos, sql, gCaseCtrl.printRes, &sqlResNum); + + // query using stmt + TAOS_RES* res = taos_stmt_use_result(stmt); + int32_t stmtResNum = 0; + bpFetchRows(res, gCaseCtrl.printRes, &stmtResNum); + if (sqlResNum != stmtResNum) { printf("!!!sql res num %d mis-match stmt res num %d\n", sqlResNum, stmtResNum); exit(1); @@ -1321,9 +1308,165 @@ void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_B printf("***sql res num match stmt res num %d\n", stmtResNum); } +void bpCheckColTagFields(TAOS_STMT *stmt, int32_t fieldNum, TAOS_FIELD_E* pFields, int32_t expecteNum, TAOS_MULTI_BIND* pBind, BP_BIND_TYPE type) { + int32_t code = 0; + + if (fieldNum != expecteNum) { + printf("!!!%s field num %d mis-match expect num %d\n", BP_BIND_TYPE_STR(type), fieldNum, expecteNum); + exit(1); + } + + if (type == BP_BIND_COL) { + if (pFields[0].precision != gCaseCtrl.precision) { + printf("!!!db precision %d mis-match expect %d\n", pFields[0].precision, gCaseCtrl.precision); + exit(1); + } + } + + for (int32_t i = 0; i < fieldNum; ++i) { + if (pFields[i].type != pBind[i].buffer_type) { + printf("!!!%s %dth field type %d mis-match expect type %d\n", BP_BIND_TYPE_STR(type), i, pFields[i].type, pBind[i].buffer_type); + exit(1); + } + + if (pFields[i].type == TSDB_DATA_TYPE_BINARY) { + if (pFields[i].bytes != (pBind[i].buffer_length + 2)) { + printf("!!!%s %dth field len %d mis-match expect len %d\n", BP_BIND_TYPE_STR(type), i, pFields[i].bytes, (pBind[i].buffer_length + 2)); + exit(1); + } + } else if (pFields[i].type == TSDB_DATA_TYPE_NCHAR) { + if (pFields[i].bytes != (pBind[i].buffer_length * 4 + 2)) { + printf("!!!%s %dth field len %d mis-match expect len %d\n", BP_BIND_TYPE_STR(type), i, pFields[i].bytes, (pBind[i].buffer_length + 2)); + exit(1); + } + } else if (pFields[i].bytes != pBind[i].buffer_length) { + printf("!!!%s %dth field len %d mis-match expect len %d\n", BP_BIND_TYPE_STR(type), i, pFields[i].bytes, pBind[i].buffer_length); + exit(1); + } + } + + if (type == BP_BIND_COL) { + int fieldType = 0; + int fieldBytes = 0; + for (int32_t i = 0; i < fieldNum; ++i) { + code = taos_stmt_get_param(stmt, i, &fieldType, &fieldBytes); + if (code) { + printf("!!!taos_stmt_get_param error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (pFields[i].type != fieldType) { + printf("!!!%s %dth field type %d mis-match expect type %d\n", BP_BIND_TYPE_STR(type), i, fieldType, pFields[i].type); + exit(1); + } + + if (pFields[i].bytes != fieldBytes) { + printf("!!!%s %dth field len %d mis-match expect len %d\n", BP_BIND_TYPE_STR(type), i, fieldBytes, pFields[i].bytes); + exit(1); + } + } + } + + if (gCaseCtrl.printVerbose) { + printf("%s fields check passed\n", BP_BIND_TYPE_STR(type)); + } +} + + +void bpCheckTagFields(TAOS_STMT *stmt, TAOS_MULTI_BIND* pBind) { + int32_t code = 0; + int fieldNum = 0; + TAOS_FIELD_E* pFields = NULL; + code = taos_stmt_get_tag_fields(stmt, &fieldNum, &pFields); + if (code != 0){ + printf("!!!taos_stmt_get_tag_fields error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckColTagFields(stmt, fieldNum, pFields, gCurCase->bindTagNum, pBind, BP_BIND_TAG); +} + +void bpCheckColFields(TAOS_STMT *stmt, TAOS_MULTI_BIND* pBind) { + if (gCurCase->testType == TTYPE_QUERY) { + return; + } + + int32_t code = 0; + int fieldNum = 0; + TAOS_FIELD_E* pFields = NULL; + code = taos_stmt_get_col_fields(stmt, &fieldNum, &pFields); + if (code != 0){ + printf("!!!taos_stmt_get_col_fields error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckColTagFields(stmt, fieldNum, pFields, gCurCase->bindColNum, pBind, BP_BIND_COL); +} + +void bpShowBindParam(TAOS_MULTI_BIND *bind, int32_t num) { + for (int32_t i = 0; i < num; ++i) { + TAOS_MULTI_BIND* b = &bind[i]; + printf("Bind %d: type[%d],buf[%p],buflen[%d],len[%],null[%d],num[%d]\n", + i, b->buffer_type, b->buffer, b->buffer_length, b->length ? *b->length : 0, b->is_null ? *b->is_null : 0, b->num); + } +} + +int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { + static int32_t n = 0; + + bpCheckColFields(stmt, bind); + + if (gCurCase->bindRowNum > 1) { + if (0 == (n++%2)) { + if (taos_stmt_bind_param_batch(stmt, bind)) { + printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); + bpShowBindParam(bind, gCurCase->bindColNum); + exit(1); + } + } else { + for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { + if (taos_stmt_bind_single_param_batch(stmt, bind+i, i)) { + printf("!!!taos_stmt_bind_single_param_batch %d error:%s\n", taos_stmt_errstr(stmt), i); + bpShowBindParam(bind, gCurCase->bindColNum); + exit(1); + } + } + } + } else { + if (0 == (n++%2)) { + if (taos_stmt_bind_param_batch(stmt, bind)) { + printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); + bpShowBindParam(bind, gCurCase->bindColNum); + exit(1); + } + } else { + if (taos_stmt_bind_param(stmt, bind)) { + printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + bpShowBindParam(bind, gCurCase->bindColNum); + exit(1); + } + } + } + + return 0; +} + int32_t bpSetTableNameTags(BindData *data, int32_t tblIdx, char *tblName, TAOS_STMT *stmt) { + int32_t code = 0; if (gCurCase->bindTagNum > 0) { - return taos_stmt_set_tbname_tags(stmt, tblName, data->pTags + tblIdx * gCurCase->bindTagNum); + if ((rand() % 2) == 0) { + code = taos_stmt_set_tbname(stmt, tblName); + if (code != 0){ + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckTagFields(stmt, data->pTags + tblIdx * gCurCase->bindTagNum); + + return taos_stmt_set_tags(stmt, data->pTags + tblIdx * gCurCase->bindTagNum); + } else { + return taos_stmt_set_tbname_tags(stmt, tblName, data->pTags + tblIdx * gCurCase->bindTagNum); + } } else { return taos_stmt_set_tbname(stmt, tblName); } @@ -1755,7 +1898,7 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname_tags(stmt, buf, data.pTags + t * gCurCase->bindTagNum); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname_tags error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -2223,14 +2366,48 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * } } +char *bpPrecisionStr(uint8_t precision) { + switch (precision) { + case TIME_PRECISION_MILLI: + return "ms"; + case TIME_PRECISION_MICRO: + return "us"; + case TIME_PRECISION_NANO: + return "ns"; + default: + return "unknwon"; + } +} + +void bpSetStartupTs() { + switch (gCaseCtrl.precision) { + case TIME_PRECISION_MILLI: + bpTs = BP_STARTUP_TS; + break; + case TIME_PRECISION_MICRO: + bpTs = BP_STARTUP_TS * 1000; + break; + case TIME_PRECISION_NANO: + bpTs = BP_STARTUP_TS * 1000000; + break; + default: + bpTs = BP_STARTUP_TS; + break; + } +} + void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { TAOS_RES *result; int code; + char createDbSql[128] = {0}; result = taos_query(taos, "drop database demo"); taos_free_result(result); - result = taos_query(taos, "create database demo keep 36500"); + sprintf(createDbSql, "create database demo keep 36500 precision \"%s\"", bpPrecisionStr(gCaseCtrl.precision)); + printf("\tCreate Database SQL:%s\n", createDbSql); + + result = taos_query(taos, createDbSql); code = taos_errno(result); if (code != 0) { printf("!!!failed to create database, reason:%s\n", taos_errstr(result)); @@ -2278,6 +2455,8 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { CaseCfg cfg = gCase[caseIdx]; CaseCfg cfgBk; gCurCase = &cfg; + + bpSetStartupTs(); if ((gCaseCtrl.bindColTypeNum || gCaseCtrl.bindColNum) && (gCurCase->colNum != gFullColNum)) { return 1; @@ -2413,22 +2592,28 @@ void* runCaseList(TAOS *taos) { } void runAll(TAOS *taos) { -#if 1 - - strcpy(gCaseCtrl.caseCatalog, "Normal Test"); + strcpy(gCaseCtrl.caseCatalog, "Default Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); + strcpy(gCaseCtrl.caseCatalog, "Micro DB precision Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); + gCaseCtrl.precision = TIME_PRECISION_MICRO; + runCaseList(taos); + gCaseCtrl.precision = TIME_PRECISION_MILLI; + strcpy(gCaseCtrl.caseCatalog, "Nano DB precision Test"); + printf("%s Begin\n", gCaseCtrl.caseCatalog); + gCaseCtrl.precision = TIME_PRECISION_NANO; + runCaseList(taos); + gCaseCtrl.precision = TIME_PRECISION_MILLI; + strcpy(gCaseCtrl.caseCatalog, "Auto Create Table Test"); gCaseCtrl.autoCreateTbl = true; printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); gCaseCtrl.autoCreateTbl = false; - -#endif -/* strcpy(gCaseCtrl.caseCatalog, "Null Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindNullNum = 1; @@ -2441,6 +2626,7 @@ void runAll(TAOS *taos) { runCaseList(taos); gCaseCtrl.bindRowNum = 0; +#if 0 strcpy(gCaseCtrl.caseCatalog, "Row Num Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.rowNum = 1000; @@ -2448,23 +2634,21 @@ void runAll(TAOS *taos) { runCaseList(taos); gCaseCtrl.rowNum = 0; gCaseCtrl.printRes = true; -*/ strcpy(gCaseCtrl.caseCatalog, "Runtimes Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.runTimes = 2; runCaseList(taos); gCaseCtrl.runTimes = 0; +#endif -#if 1 strcpy(gCaseCtrl.caseCatalog, "Check Param Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.checkParamNum = true; runCaseList(taos); gCaseCtrl.checkParamNum = false; -#endif -/* +#if 0 strcpy(gCaseCtrl.caseCatalog, "Bind Col Num Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindColNum = 6; @@ -2476,7 +2660,7 @@ void runAll(TAOS *taos) { gCaseCtrl.bindColTypeNum = tListLen(bindColTypeList); gCaseCtrl.bindColTypeList = bindColTypeList; runCaseList(taos); -*/ +#endif printf("All Test End\n"); } From 98f62165ae7bbc3e969f96df291f72c3b610ca12 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 1 Jun 2022 13:37:33 +0800 Subject: [PATCH 51/57] refactor(tmq): tq meta --- example/src/tmq.c | 4 - source/dnode/vnode/CMakeLists.txt | 7 +- source/dnode/vnode/src/inc/tq.h | 7 ++ source/dnode/vnode/src/tq/tq.c | 126 +-------------------------- source/dnode/vnode/src/tq/tqMeta.c | 134 +++++++++++++++++++++++++++++ source/dnode/vnode/src/tq/tqPush.c | 2 + 6 files changed, 151 insertions(+), 129 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 5eecb5e4cc..e61ad69e6b 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -165,7 +165,6 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "group.id", "tg2"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); - /*tmq_conf_set(conf, "td.connect.db", "abc1");*/ tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "false"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -191,7 +190,6 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { return; } int32_t cnt = 0; - /*clock_t startTime = clock();*/ while (running) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0); if (tmqmessage) { @@ -204,8 +202,6 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { /*break;*/ } } - /*clock_t endTime = clock();*/ - /*printf("log cnt: %d %f s\n", cnt, (double)(endTime - startTime) / CLOCKS_PER_SEC);*/ err = tmq_consumer_close(tmq); if (err) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 17445b7abe..ea2a256663 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -52,10 +52,11 @@ target_sources( # tq "src/tq/tq.c" "src/tq/tqExec.c" - "src/tq/tqCommit.c" - "src/tq/tqOffset.c" - "src/tq/tqPush.c" + "src/tq/tqMeta.c" "src/tq/tqRead.c" + "src/tq/tqOffset.c" + #"src/tq/tqPush.c" + #"src/tq/tqCommit.c" ) target_include_directories( vnode diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 34a7ff823a..89ea969d92 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -168,6 +168,13 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead* int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId); +// tqMeta + +int32_t tqMetaOpen(STQ* pTq); +int32_t tqMetaClose(STQ* pTq); +int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle); +int32_t tqMetaDeleteHandle(STQ* pTq, const char* key); + // tqOffset STqOffsetStore* STqOffsetOpen(STqOffsetCfg*); void STqOffsetClose(STqOffsetStore*); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4bce829b10..93f305ba77 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -14,7 +14,6 @@ */ #include "tq.h" -#include "tdbInt.h" int32_t tqInit() { int8_t old; @@ -47,51 +46,6 @@ void tqCleanUp() { } } -int tqExecKeyCompare(const void* pKey1, int32_t kLen1, const void* pKey2, int32_t kLen2) { - return strcmp(pKey1, pKey2); -} - -int32_t tqStoreHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { - int32_t code; - int32_t vlen; - tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); - ASSERT(code == 0); - - void* buf = taosMemoryCalloc(1, vlen); - if (buf == NULL) { - ASSERT(0); - } - - SEncoder encoder; - tEncoderInit(&encoder, buf, vlen); - - if (tEncodeSTqHandle(&encoder, pHandle) < 0) { - ASSERT(0); - } - - TXN txn; - - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - ASSERT(0); - } - - if (tdbBegin(pTq->pMetaStore, &txn) < 0) { - ASSERT(0); - } - - if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, &txn) < 0) { - ASSERT(0); - } - - if (tdbCommit(pTq->pMetaStore, &txn) < 0) { - ASSERT(0); - } - - tEncoderClear(&encoder); - taosMemoryFree(buf); - return 0; -} - STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { STQ* pTq = taosMemoryMalloc(sizeof(STQ)); if (pTq == NULL) { @@ -108,60 +62,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); - if (tdbOpen(path, 16 * 1024, 1, &pTq->pMetaStore) < 0) { - ASSERT(0); - } - - if (tdbTbOpen("handles", -1, -1, tqExecKeyCompare, pTq->pMetaStore, &pTq->pExecStore) < 0) { - ASSERT(0); - } - - TXN txn; - - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - ASSERT(0); - } - - TBC* pCur; - if (tdbTbcOpen(pTq->pExecStore, &pCur, &txn) < 0) { - ASSERT(0); - } - - void* pKey; - int kLen; - void* pVal; - int vLen; - - tdbTbcMoveToFirst(pCur); - SDecoder decoder; - - while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { - STqHandle handle; - tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - tDecodeSTqHandle(&decoder, &handle); - handle.pWalReader = walOpenReadHandle(pTq->pVnode->pWal); - for (int32_t i = 0; i < 5; i++) { - handle.execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); - } - if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - for (int32_t i = 0; i < 5; i++) { - SReadHandle reader = { - .reader = handle.execHandle.pExecReader[i], - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, - }; - handle.execHandle.exec.execCol.task[i] = - qCreateStreamExecTaskInfo(handle.execHandle.exec.execCol.qmsg, &reader); - ASSERT(handle.execHandle.exec.execCol.task[i]); - } - } else { - handle.execHandle.exec.execDb.pFilterOutTbUid = - taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - } - taosHashPut(pTq->handles, pKey, kLen, &handle, sizeof(STqHandle)); - } - - if (tdbTxnClose(&txn) < 0) { + if (tqMetaOpen(pTq) < 0) { ASSERT(0); } @@ -174,7 +75,7 @@ void tqClose(STQ* pTq) { taosHashCleanup(pTq->handles); taosHashCleanup(pTq->pStreamTasks); taosHashCleanup(pTq->pushMgr); - tdbClose(pTq->pMetaStore); + tqMetaClose(pTq); taosMemoryFree(pTq); } // TODO @@ -256,9 +157,6 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ taosWLockLatch(&pHandle->pushHandle.lock); - /*SRpcHandleInfo* pInfo = atomic_load_ptr(&pHandle->pushHandle.pInfo);*/ - /*ASSERT(pInfo);*/ - SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pHandle->pushHandle.reqOffset; rsp.blockData = taosArrayInit(0, sizeof(void*)); @@ -303,7 +201,6 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ }; tmsgSendRsp(&resp); - /*atomic_store_ptr(&pHandle->pushHandle.pInfo, NULL);*/ memset(&pHandle->pushHandle.info, 0, sizeof(SRpcHandleInfo)); taosWUnLockLatch(&pHandle->pushHandle.lock); @@ -508,24 +405,9 @@ int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) { int32_t code = taosHashRemove(pTq->handles, pReq->subKey, strlen(pReq->subKey)); ASSERT(code == 0); - TXN txn; - - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) { ASSERT(0); } - - if (tdbBegin(pTq->pMetaStore, &txn) < 0) { - ASSERT(0); - } - - if (tdbTbDelete(pTq->pExecStore, pReq->subKey, (int)strlen(pReq->subKey), &txn) < 0) { - /*ASSERT(0);*/ - } - - if (tdbCommit(pTq->pMetaStore, &txn) < 0) { - ASSERT(0); - } - return 0; } @@ -583,7 +465,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { atomic_add_fetch_32(&pHandle->epoch, 1); } - if (tqStoreHandle(pTq, req.subKey, pHandle) < 0) { + if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { // TODO } return 0; diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index f2f48bbc8a..74162a9f49 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -12,3 +12,137 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "tdbInt.h" +#include "tq.h" + +int tqExecKeyCompare(const void* pKey1, int32_t kLen1, const void* pKey2, int32_t kLen2) { + return strcmp(pKey1, pKey2); +} + +int32_t tqMetaOpen(STQ* pTq) { + if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaStore) < 0) { + ASSERT(0); + } + + if (tdbTbOpen("handles", -1, -1, tqExecKeyCompare, pTq->pMetaStore, &pTq->pExecStore) < 0) { + ASSERT(0); + } + + TXN txn; + + if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + ASSERT(0); + } + + TBC* pCur; + if (tdbTbcOpen(pTq->pExecStore, &pCur, &txn) < 0) { + ASSERT(0); + } + + void* pKey; + int kLen; + void* pVal; + int vLen; + + tdbTbcMoveToFirst(pCur); + SDecoder decoder; + + while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { + STqHandle handle; + tDecoderInit(&decoder, (uint8_t*)pVal, vLen); + tDecodeSTqHandle(&decoder, &handle); + handle.pWalReader = walOpenReadHandle(pTq->pVnode->pWal); + for (int32_t i = 0; i < 5; i++) { + handle.execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); + } + if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + for (int32_t i = 0; i < 5; i++) { + SReadHandle reader = { + .reader = handle.execHandle.pExecReader[i], + .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, + }; + handle.execHandle.exec.execCol.task[i] = + qCreateStreamExecTaskInfo(handle.execHandle.exec.execCol.qmsg, &reader); + ASSERT(handle.execHandle.exec.execCol.task[i]); + } + } else { + handle.execHandle.exec.execDb.pFilterOutTbUid = + taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + } + taosHashPut(pTq->handles, pKey, kLen, &handle, sizeof(STqHandle)); + } + + if (tdbTxnClose(&txn) < 0) { + ASSERT(0); + } + return 0; +} + +int32_t tqMetaClose(STQ* pTq) { + tdbClose(pTq->pMetaStore); + return 0; +} + +int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { + int32_t code; + int32_t vlen; + tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); + ASSERT(code == 0); + + void* buf = taosMemoryCalloc(1, vlen); + if (buf == NULL) { + ASSERT(0); + } + + SEncoder encoder; + tEncoderInit(&encoder, buf, vlen); + + if (tEncodeSTqHandle(&encoder, pHandle) < 0) { + ASSERT(0); + } + + TXN txn; + + if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + ASSERT(0); + } + + if (tdbBegin(pTq->pMetaStore, &txn) < 0) { + ASSERT(0); + } + + if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, &txn) < 0) { + ASSERT(0); + } + + if (tdbCommit(pTq->pMetaStore, &txn) < 0) { + ASSERT(0); + } + + tEncoderClear(&encoder); + taosMemoryFree(buf); + return 0; +} + +int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { + TXN txn; + + if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + ASSERT(0); + } + + if (tdbBegin(pTq->pMetaStore, &txn) < 0) { + ASSERT(0); + } + + if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), &txn) < 0) { + /*ASSERT(0);*/ + } + + if (tdbCommit(pTq->pMetaStore, &txn) < 0) { + ASSERT(0); + } + + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index f2f48bbc8a..e31566f3fa 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -12,3 +12,5 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +#include "tq.h" From 6a15fd7e5584b20b3a095fd6f2a6046bd39a9a9d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 13:45:29 +0800 Subject: [PATCH 52/57] fix:fix compile error in windows --- include/util/tencode.h | 187 +++++++---------------------------------- 1 file changed, 30 insertions(+), 157 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index 3e1d675870..bde2185b02 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -461,151 +461,64 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { } // =========================================== -#define tPutV(p, v) \ - do { \ - int32_t n = 0; \ - for (;;) { \ - if (v <= 0x7f) { \ - if (p) p[n] = v; \ - n++; \ - break; \ - } \ - if (p) p[n] = (v & 0x7f) | 0x80; \ - n++; \ - v >>= 7; \ - } \ - return n; \ - } while (0) +#define tPutV(p, v) \ + int32_t n = 0; \ + for (;;) { \ + if (v <= 0x7f) { \ + if (p) p[n] = v; \ + n++; \ + break; \ + } \ + if (p) p[n] = (v & 0x7f) | 0x80; \ + n++; \ + v >>= 7; \ + } \ + return n; -#define tGetV(p, v) \ - do { \ - int32_t n = 0; \ - if (v) *v = 0; \ - for (;;) { \ - if (p[n] <= 0x7f) { \ - if (v) (*v) |= (p[n] << (7 * n)); \ - n++; \ - break; \ - } \ - if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ - n++; \ - } \ - return n; \ - } while (0) +#define tGetV(p, v) \ + int32_t n = 0; \ + if (v) *v = 0; \ + for (;;) { \ + if (p[n] <= 0x7f) { \ + if (v) (*v) |= (p[n] << (7 * n)); \ + n++; \ + break; \ + } \ + if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ + n++; \ + } \ + return n; // PUT -static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) { - if (p) ((uint8_t*)p)[0] = v; - return sizeof(uint8_t); -} - static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) { if (p) ((int8_t*)p)[0] = v; return sizeof(int8_t); } -static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) { - if (p) ((uint16_t*)p)[0] = v; - return sizeof(uint16_t); -} - -static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) { - if (p) ((int16_t*)p)[0] = v; - return sizeof(int16_t); -} - -static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) { - if (p) ((uint32_t*)p)[0] = v; - return sizeof(uint32_t); -} - -static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) { - if (p) ((int32_t*)p)[0] = v; - return sizeof(int32_t); -} - -static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) { - if (p) ((uint64_t*)p)[0] = v; - return sizeof(uint64_t); -} - static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) { if (p) ((int64_t*)p)[0] = v; return sizeof(int64_t); } -static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) { - union { - uint32_t ui; - float f; - } v = {.f = f}; - - return tPutU32(p, v.ui); -} - -static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) { - union { - uint64_t ui; - double d; - } v = {.d = d}; - - return tPutU64(p, v.ui); -} - -static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); } +static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v) } static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); } -static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); } +static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v) } static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); } -static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); } - -static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); } - -// GET -static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) { - if (v) *v = ((uint8_t*)p)[0]; - return sizeof(uint8_t); -} - static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) { if (v) *v = ((int8_t*)p)[0]; return sizeof(int8_t); } -static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) { - if (v) *v = ((uint16_t*)p)[0]; - return sizeof(uint16_t); -} - -static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) { - if (v) *v = ((int16_t*)p)[0]; - return sizeof(int16_t); -} - -static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) { - if (v) *v = ((uint32_t*)p)[0]; - return sizeof(uint32_t); -} - -static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) { - if (v) *v = ((int32_t*)p)[0]; - return sizeof(int32_t); -} - -static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) { - if (v) *v = ((uint64_t*)p)[0]; - return sizeof(uint64_t); -} - static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) { if (v) *v = ((int64_t*)p)[0]; return sizeof(int64_t); } -static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); } +static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v) } static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { int32_t n; @@ -617,7 +530,7 @@ static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { return n; } -static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); } +static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v) } static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { int32_t n; @@ -629,46 +542,6 @@ static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { return n; } -static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); } - -static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { - int32_t n; - uint64_t tv; - - n = tGetU64v(p, &tv); - if (v) *v = ZIGZAGD(int64_t, tv); - - return n; -} - -static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f) { - int32_t n = 0; - - union { - uint32_t ui; - float f; - } v; - - n = tGetU32(p, &v.ui); - - *f = v.f; - return n; -} - -static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d) { - int32_t n = 0; - - union { - uint64_t ui; - double d; - } v; - - n = tGetU64(p, &v.ui); - - *d = v.d; - return n; -} - // ===================== static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) { int n = 0; From cd8b5b7e90aa98d0884333ad6b39311b4b5b8774 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 1 Jun 2022 13:55:34 +0800 Subject: [PATCH 53/57] docs: refine functions page in EN of 3.0 --- docs-cn/12-taos-sql/07-function.md | 38 +- docs-en/12-taos-sql/07-function.md | 2447 ++++++++++------------------ 2 files changed, 916 insertions(+), 1569 deletions(-) diff --git a/docs-cn/12-taos-sql/07-function.md b/docs-cn/12-taos-sql/07-function.md index d2cd89f152..993a1dac6e 100644 --- a/docs-cn/12-taos-sql/07-function.md +++ b/docs-cn/12-taos-sql/07-function.md @@ -808,6 +808,25 @@ SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; - 系统同时返回该记录关联的时间戳列； - 限制：BOTTOM 函数不支持 FILL 子句。 +### FIRST + +``` +SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**功能说明**：统计表/超级表中某列的值最先写入的非 NULL 值。 + +**返回数据类型**：同应用的字段。 + +**适用数据类型**：所有字段。 + +**适用于**：表和超级表。 + +**使用说明**: + +- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)； +- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL； +- 如果结果集中所有列全部为 NULL 值，则不返回结果。 ### INTERP @@ -919,25 +938,6 @@ SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; **使用说明**：*P*值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX。 -### FIRST - -``` -SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**功能说明**：统计表/超级表中某列的值最先写入的非 NULL 值。 - -**返回数据类型**：同应用的字段。 - -**适用数据类型**：所有字段。 - -**适用于**：表和超级表。 - -**使用说明**: - -- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)； -- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL； -- 如果结果集中所有列全部为 NULL 值，则不返回结果。 ### TAIL diff --git a/docs-en/12-taos-sql/07-function.md b/docs-en/12-taos-sql/07-function.md index 1a0dc28fa0..6c45474b88 100644 --- a/docs-en/12-taos-sql/07-function.md +++ b/docs-en/12-taos-sql/07-function.md @@ -1,10 +1,658 @@ --- -title: Functions +title: SQL Functions +toc_max_heading_level: 4 --- +## Single-Row Functions + +Single-Row functions return a result row for each row in the query result. + +### Numeric Functions + +#### ABS + +```sql +SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The absolute of a specific column. + +**Return value type**: UBIGINT if the input value is integer; DOUBLE if the input value is FLOAT/DOUBLE. + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable. + +**Applicable nested query**: Inner query and Outer query. + +**More explanations**: + +- Can't be used with tags. +- Can't be used with aggregate functions. + +#### ACOS + +```sql +SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The anti-cosine of a specific column + +**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- Can't be used with tags +- Can't be used with aggregate functions + +#### ASIN + +```sql +SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The anti-sine of a specific column + +**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- Can't be used with tags +- Can't be used with aggregate functions + +#### ATAN + +```sql +SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: anti-tangent of a specific column + +**Description**: The anti-cosine of a specific column + +**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- Can't be used with tags +- Can't be used with aggregate functions + +#### CEIL + +``` +SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**Description**: The rounded up value of a specific column + +**Return value type**: Same as the column being used + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and outer query + +**More explanations**: + +- Can't be used on any tags of any type +- Arithmetic operation can be performed on the result of `ceil` function +- Can't be used with aggregate functions + +#### COS + +```sql +SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The cosine of a specific column + +**Description**: The anti-cosine of a specific column + +**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- Can't be used with tags +- Can't be used with aggregate functions + +#### FLOOR + +``` +SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**Description**: The rounded down value of a specific column + +**More explanations**: The restrictions are same as those of the `CEIL` function. + +#### LOG + +```sql +SELECT LOG(field_name, base) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The log of a specific with `base` as the radix + +**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- Can't be used with tags +- Can't be used with aggregate functions + +#### POW + +```sql +SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The power of a specific column with `power` as the index + +**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- Can't be used with tags +- Can't be used with aggregate functions + +#### ROUND + +``` +SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**Description**: The rounded value of a specific column. + +**More explanations**: The restrictions are same as `CEIL` function. + +#### SIN + +```sql +SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The sine of a specific column + +**Description**: The anti-cosine of a specific column + +**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- Can't be used with tags +- Can't be used with aggregate functions + +#### SQRT + +```sql +SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The square root of a specific column + +**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- Can't be used with tags +- Can't be used with aggregate functions + +#### TAN + +```sql +SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The tangent of a specific column + +**Description**: The anti-cosine of a specific column + +**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL + +**Applicable data types**: Numeric types. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- Can't be used with tags +- Can't be used with aggregate functions + +### String Functions + +String functiosn take strings as input and output numbers or strings. + +#### CHAR_LENGTH + +``` +SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The length in number of characters of a string + +**Return value type**: Integer + +**Applicable data types**: BINARY or NCHAR, can't be used on tags + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations** + +- If the input value is NULL, the output is NULL too + +#### CONCAT + +```sql +SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The concatenation result of two or more strings, the number of strings to be concatenated is at least 2 and at most 8 + +**Return value type**: If all input strings are BINARY type, the result is BINARY type too. If any one of input strings is NCHAR type, then the result is NCHAR. + +**Applicable data types**: BINARY, NCHAR. Can't be used on tag columns. At least 2 input strings are requird, and at most 8 input strings are allowed. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +#### CONCAT_WS + +``` +SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The concatenation result of two or more strings with separator, the number of strings to be concatenated is at least 3 and at most 9 + +**Return value type**: If all input strings are BINARY type, the result is BINARY type too. If any one of input strings is NCHAR type, then the result is NCHAR. + +**Applicable data types**: BINARY, NCHAR. Can't be used on tag columns. At least 3 input strings are requird, and at most 9 input strings are allowed. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- If the value of `separator` is NULL, the output is NULL. If the value of `separator` is not NULL but other input are all NULL, the output is empty string. + +#### LENGTH + +``` +SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The length in bytes of a string + +**Return value type**: Integer + +**Applicable data types**: BINARY or NCHAR, can't be used on tags + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations** + +- If the input value is NULL, the output is NULL too + +#### LOWER + +``` +SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: Convert the input string to lower case + +**Return value type**: Same as input + +**Applicable data types**: BINARY or NCHAR, can't be used on tags + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations** + +- If the input value is NULL, the output is NULL too + +#### LTRIM + +``` +SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: Remove the left leading blanks of a string + +**Return value type**: Same as input + +**Applicable data types**: BINARY or NCHAR, can't be used on tags + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations** + +- If the input value is NULL, the output is NULL too + +#### RTRIM + +``` +SELECT RTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: Remove the right tailing blanks of a string + +**Return value type**: Same as input + +**Applicable data types**: BINARY or NCHAR, can't be used on tags + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations** + +- If the input value is NULL, the output is NULL too + +#### SUBSTR + +``` +SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: The sub-string starting from `pos` with length of `len` from the original string `str` + +**Return value type**: Same as input + +**Applicable data types**: BINARY or NCHAR, can't be used on tags + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations**: + +- If the input is NULL, the output is NULL +- Parameter `pos` can be an positive or negative integer; If it's positive, the starting position will be counted from the beginning of the string; if it's negative, the starting position will be counted from the end of the string. +- If `len` is not specified, it means from `pos` to the end. + +#### UPPER + +``` +SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: Convert the input string to upper case + +**Return value type**: Same as input + +**Applicable data types**: BINARY or NCHAR, can't be used on tags + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query + +**More explanations** + +- If the input value is NULL, the output is NULL too + +### Conversion Functions + +This kind of functions convert from one data type to another one. + +#### CAST + +```sql +SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] +``` + +**Description**: It's used for type casting. The input parameter `expression` can be data columns, constants, scalar functions or arithmetic between them. Can't be used with tags, and can only be used in `select` clause. + +**Return value type**: The type specified by parameter `type_name` + +**Applicable data types**: + +- Parameter `expression` can be any data type except for JSON, more specifically it can be any of BOOL/TINYINT/SMALLINT/INT/BIGINT/FLOAT/DOUBLE/BINARY(M)/TIMESTAMP/NCHAR(M)/TINYINT UNSIGNED/SMALLINT UNSIGNED/INT UNSIGNED/BIGINT UNSIGNED +- The output data type specified by `type_name` can only be one of BIGINT/BINARY(N)/TIMESTAMP/NCHAR(N)/BIGINT UNSIGNED + +**More explanations**: + +- Error will be reported for unsupported type casting +- NULL will be returned if the input value is NULL +- Some values of some supported data types may not be casted, below are known issues: + 1）When casting BINARY/NCHAR to BIGINT/BIGINT UNSIGNED, some characters may be treated as illegal, for example "a" may be converted to 0. + 2）There may be overflow when casting singed integer or TIMESTAMP to unsigned BIGINT + 3）There may be overflow when casting unsigned BIGINT to BIGINT + 4）There may be overflow when casting FLOAT/DOUBLE to BIGINT or UNSIGNED BIGINT + +#### TO_ISO8601 + +```sql +SELECT TO_ISO8601(ts_val | ts_col) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**Description**: The ISO8601 date/time format converted from a UNIX timestamp, plus the timezone of the client side system + +**Return value type**: BINARY + +**Applicable column types**: TIMESTAMP, constant or a column + +**Applicable table types**: table, STable + +**More explanations**: + +- If the input is UNIX timestamp constant, the precision of the returned value is determined by the digits of the input timestamp +- If the input is a column of TIMESTAMP type, The precision of the returned value is same as the precision set for the current data base in use + +#### TO_JSON + +```sql +SELECT TO_JSON(str_literal) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**Description**: Convert a JSON string to a JSON body。 + +**Return value type**: JSON + +**Applicable column types**: JSON string, in the format like '{ "literal" : literal }'. '{}' is NULL value. keys in the string must be string constants, values can be constants of numeric types, bool, string or NULL. Escaping characters are not allowed in the JSON string. + +**Applicable table types**: table, STable + +**Applicable nested query**: Inner query and Outer query. + +#### TO_UNIXTIMESTAMP + +```sql +SELECT TO_UNIXTIMESTAMP(datetime_string | ts_col) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**Description**: UNIX timestamp converted from a string of date/time format + +**Return value type**: Long integer + +**Applicable column types**: Constant or column of BINARY/NCHAR + +**Applicable table types**: table, STable + +**More explanations**: + +- The input string must be compatible with ISO8601/RFC3339 standard, 0 will be returned if the string can't be converted +- The precision of the returned timestamp is same as the precision set for the current data base in use + +### DateTime Functions + +This kind of functiosn oeprate on timestamp data. NOW(), TODAY() and TIMEZONE() are executed only once even though they may occurr multiple times in a single SQL statement. + +#### NOW + +```sql +SELECT NOW() FROM { tb_name | stb_name } [WHERE clause]; +SELECT select_expr FROM { tb_name | stb_name } WHERE ts_col cond_operatior NOW(); +INSERT INTO tb_name VALUES (NOW(), ...); +``` + +**Description**: The current time of the client side system + +**Return value type**: TIMESTAMP + +**Applicable column types**: TIMESTAMP only + +**Applicable table types**: table, STable + +**More explanations**: + +- Add and Subtract operation can be performed, for example NOW() + 1s, the time unit can be: + b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), w(week) +- The precision of the returned timestamp is same as the precision set for the current data base in use + +#### TIMEDIFF + +```sql +SELECT TIMEDIFF(ts_val1 | datetime_string1 | ts_col1, ts_val2 | datetime_string2 | ts_col2 [, time_unit]) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**Description**: The difference between two timestamps, and rounded to the time unit specified by `time_unit` + +**Return value type**: Long Integer + +**Applicable column types**: UNIX timestamp constant, string constant of date/time format, or a column of TIMESTAMP type + +**Applicable table types**: table, STable + +**More explanations**: + +- Time unit specified by `time_unit` can be: + 1u(microsecond),1a(millisecond),1s(second),1m(minute),1h(hour),1d(day). +- The precision of the returned timestamp is same as the precision set for the current data base in use + +#### TIMETRUNCATE + +```sql +SELECT TIMETRUNCATE(ts_val | datetime_string | ts_col, time_unit) FROM { tb_name | stb_name } [WHERE clause]; +``` + +**Description**: Truncate the input timestamp with unit specified by `time_unit` + +**Return value type**: TIMESTAMP + +**Applicable column types**: UNIX timestamp constant, string constant of date/time format, or a column of timestamp + +**Applicable table types**: table, STable + +**More explanations**: + +- Time unit specified by `time_unit` can be: + 1u(microsecond),1a(millisecond),1s(second),1m(minute),1h(hour),1d(day). +- The precision of the returned timestamp is same as the precision set for the current data base in use + +#### TIMEZONE + +```sql +SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause]; +``` + +**Description**: The timezone of the client side system + +**Return value type**: BINARY + +**Applicable column types**: None + +**Applicable table types**: table, STable + +#### TODAY + +```sql +SELECT TODAY() FROM { tb_name | stb_name } [WHERE clause]; +SELECT select_expr FROM { tb_name | stb_name } WHERE ts_col cond_operatior TODAY()]; +INSERT INTO tb_name VALUES (TODAY(), ...); +``` + +**Description**: The timestamp of 00:00:00 of the client side system + +**Return value type**: TIMESTAMP + +**Applicable column types**: TIMESTAMP only + +**Applicable table types**: table, STable + +**More explanations**: + +- Add and Subtract operation can be performed, for example NOW() + 1s, the time unit can be: + b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), w(week) +- The precision of the returned timestamp is same as the precision set for the current data base in use + ## Aggregate Functions -Aggregate queries are supported in TDengine by the following aggregate functions and selection functions. +Aggregate functions return single result row for each group in the query result set. Groups are determined by `GROUP BY` clause or time window clause if they are used; or the whole result is considered a group if neither of them is used. + +### AVG + +``` +SELECT AVG(field_name) FROM tb_name [WHERE clause]; +``` + +**Description**: Get the average value of a column in a table or STable + +**Return value type**: Double precision floating number + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table, STable ### COUNT @@ -25,141 +673,30 @@ SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]; - Wildcard (\*) is used to represent all columns. The `COUNT` function is used to get the total number of all rows. - The number of non-NULL values will be returned if this function is used on a specific column. -**Examples**: +### ELAPSED -``` -taos> SELECT COUNT(*), COUNT(voltage) FROM meters; - count(*) | count(voltage) | -================================================ - 9 | 9 | -Query OK, 1 row(s) in set (0.004475s) - -taos> SELECT COUNT(*), COUNT(voltage) FROM d1001; - count(*) | count(voltage) | -================================================ - 3 | 3 | -Query OK, 1 row(s) in set (0.001075s) +```mysql +SELECT ELAPSED(field_name[, time_unit]) FROM { tb_name | stb_name } [WHERE clause] [INTERVAL(interval [, offset]) [SLIDING sliding]]; ``` -### AVG +**Description**：`elapsed` function can be used to calculate the continuous time length in which there is valid data. If it's used with `INTERVAL` clause, the returned result is the calcualted time length within each time window. If it's used without `INTERVAL` caluse, the returned result is the calculated time length within the specified time range. Please be noted that the return value of `elapsed` is the number of `time_unit` in the calculated time length. -``` -SELECT AVG(field_name) FROM tb_name [WHERE clause]; -``` +**Return value type**：Double -**Description**: Get the average value of a column in a table or STable +**Applicable Column type**：Timestamp -**Return value type**: Double precision floating number +**Applicable tables**: table, STable, outter in nested query -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Explanations**： -**Applicable table types**: table, STable - -**Examples**: - -``` -taos> SELECT AVG(current), AVG(voltage), AVG(phase) FROM meters; - avg(current) | avg(voltage) | avg(phase) | -==================================================================================== - 11.466666751 | 220.444444444 | 0.293333333 | -Query OK, 1 row(s) in set (0.004135s) - -taos> SELECT AVG(current), AVG(voltage), AVG(phase) FROM d1001; - avg(current) | avg(voltage) | avg(phase) | -==================================================================================== - 11.733333588 | 219.333333333 | 0.316666673 | -Query OK, 1 row(s) in set (0.000943s) -``` - -### TWA - -``` -SELECT TWA(field_name) FROM tb_name WHERE clause; -``` - -**Description**: Time weighted average on a specific column within a time range - -**Return value type**: Double precision floating number - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table, STable - -**More explanations**: - -- Since version 2.1.3.0, function TWA can be used on stable with `GROUP BY`, i.e. timelines generated by `GROUP BY tbname` on a STable. - -### IRATE - -``` -SELECT IRATE(field_name) FROM tb_name WHERE clause; -``` - -**Description**: instantaneous rate on a specific column. The last two samples in the specified time range are used to calculate instantaneous rate. If the last sample value is smaller, then only the last sample value is used instead of the difference between the last two sample values. - -**Return value type**: Double precision floating number - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table, STable - -**More explanations**: - -- Since version 2.1.3.0, function IRATE can be used on stble with `GROUP BY`, i.e. timelines generated by `GROUP BY tbname` on a STable. - -### SUM - -``` -SELECT SUM(field_name) FROM tb_name [WHERE clause]; -``` - -**Description**: The sum of a specific column in a table or STable - -**Return value type**: Double precision floating number or long integer - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table, STable - -**Examples**: - -``` -taos> SELECT SUM(current), SUM(voltage), SUM(phase) FROM meters; - sum(current) | sum(voltage) | sum(phase) | -================================================================================ - 103.200000763 | 1984 | 2.640000001 | -Query OK, 1 row(s) in set (0.001702s) - -taos> SELECT SUM(current), SUM(voltage), SUM(phase) FROM d1001; - sum(current) | sum(voltage) | sum(phase) | -================================================================================ - 35.200000763 | 658 | 0.950000018 | -Query OK, 1 row(s) in set (0.000980s) -``` - -### STDDEV - -``` -SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; -``` - -**Description**: Standard deviation of a specific column in a table or STable - -**Return value type**: Double precision floating number - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table, STable (since version 2.0.15.1) - -**Examples**: - -``` -taos> SELECT STDDEV(current) FROM d1001; - stddev(current) | -============================ - 1.020892909 | -Query OK, 1 row(s) in set (0.000915s) -``` +- `field_name` parameter can only be the first column of a table, i.e. timestamp primary key. +- The minimum value of `time_unit` is the time precision of the database. If `time_unit` is not specified, the time precision of the database is used as the default ime unit. +- It can be used with `INTERVAL` to get the time valid time length of each time window. Please be noted that the return value is same as the time window for all time windows except for the first and the last time window. +- `order by asc/desc` has no effect on the result. +- `group by tbname` must be used together when `elapsed` is used against a STable. +- `group by` must NOT be used together when `elapsed` is used against a table or sub table. +- When used in nested query, it's only applicable when the inner query outputs an implicit timestamp column as the primary key. For example, `select elapsed(ts) from (select diff(value) from sub1)` is legal usage while `select elapsed(ts) from (select * from sub1)` is not. +- It can't be used with `leastsquares`, `diff`, `derivative`, `top`, `bottom`, `last_row`, `interp`. ### LEASTSQUARES @@ -175,16 +712,6 @@ SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause] **Applicable table types**: table only -**Examples**: - -``` -taos> SELECT LEASTSQUARES(current, 1, 1) FROM d1001; - leastsquares(current, 1, 1) | -===================================================== -{slop:1.000000, intercept:9.733334} | -Query OK, 1 row(s) in set (0.000921s) -``` - ### MODE ``` @@ -199,27 +726,50 @@ SELECT MODE(field_name) FROM tb_name [WHERE clause]; **More explanations**:Considering the number of returned result set is unpredictable, it's suggested to limit the number of unique values to 100,000, otherwise error will be returned. -**Applicable version**:Since version 2.6.0.0 - -**Examples**: +### SPREAD ``` -taos> select voltage from d002; - voltage | -======================== - 1 | - 1 | - 2 | - 19 | -Query OK, 4 row(s) in set (0.003545s) - -taos> select mode(voltage) from d002; - mode(voltage) | -======================== - 1 | -Query OK, 1 row(s) in set (0.019393s) +SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` +**Description**: The difference between the max and the min of a specific column + +**Return value type**: Double precision floating point + +**Applicable column types**: Data types except for binary, nchar, and bool + +**Applicable table types**: table, STable + +**More explanations**: Can be used on a column of TIMESTAMP type, the result is the time range size. + +### STDDEV + +``` +SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; +``` + +**Description**: Standard deviation of a specific column in a table or STable + +**Return value type**: Double precision floating number + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table, STable + +### SUM + +``` +SELECT SUM(field_name) FROM tb_name [WHERE clause]; +``` + +**Description**: The sum of a specific column in a table or STable + +**Return value type**: Double precision floating number or long integer + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table, STable + ### HYPERLOGLOG ``` @@ -234,31 +784,6 @@ SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause]; **More explanations**: The benefit of using hyperloglog algorithm is that the memory usage is under control when the data volume is huge. However, when the data volume is very small, the result may be not accurate, it's recommented to use `select count(data) from (select unique(col) as data from table)` in this case. -**Applicable versions**:Since version 2.6.0.0 - -**Examples**: - -``` -taos> select dbig from shll; - dbig | -======================== - 1 | - 1 | - 1 | - NULL | - 2 | - 19 | - NULL | - 9 | -Query OK, 8 row(s) in set (0.003755s) - -taos> select hyperloglog(dbig) from shll; - hyperloglog(dbig)| -======================== - 4 | -Query OK, 1 row(s) in set (0.008388s) -``` - ### HISTOGRAM ``` @@ -271,262 +796,57 @@ SELECT HISTOGRAM(field_name，bin_type, bin_description, normalized) FROM tb_nam **Applicable column type**：Numerical types. -**Applicable versions**：Since version 2.6.0.0. - **Applicable table types**: table, STable **Explanations**： 1. bin_type: parameter to indicate the bucket type, valid inputs are: "user_input", "linear_bin", "log_bin"。 -2. bin_description: parameter to describe how to generate buckets，can be in the following JSON formats for each bin_type respectively: +2. bin_description: parameter to describe how to generate buckets，can be in the following JSON formats for each bin_type respectively: - - "user_input": "[1, 3, 5, 7]": User specified bin values. + - "user_input": "[1, 3, 5, 7]": User specified bin values. - - "linear_bin": "{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" - "start" - bin starting point. - "width" - bin offset. - "count" - number of bins generated. - "infinity" - whether to add（-inf, inf）as start/end point in generated set of bins. - The above "linear_bin" descriptor generates a set of bins: [-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]. + - "linear_bin": "{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" + "start" - bin starting point. + "width" - bin offset. + "count" - number of bins generated. + "infinity" - whether to add（-inf, inf）as start/end point in generated set of bins. + The above "linear_bin" descriptor generates a set of bins: [-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]. - - "log_bin": "{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" - "start" - bin starting point. - "factor" - exponential factor of bin offset. - "count" - number of bins generated. - "infinity" - whether to add（-inf, inf）as start/end point in generated range of bins. - The above "log_bin" descriptor generates a set of bins:[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]. + - "log_bin": "{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" + "start" - bin starting point. + "factor" - exponential factor of bin offset. + "count" - number of bins generated. + "infinity" - whether to add（-inf, inf）as start/end point in generated range of bins. + The above "log_bin" descriptor generates a set of bins:[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]. 3. normalized: setting to 1/0 to turn on/off result normalization. -**Example**： +## Selector Functions -```mysql -taos> SELECT HISTOGRAM(voltage, "user_input", "[1,3,5,7]", 1) FROM meters; - histogram(voltage, "user_input", "[1,3,5,7]", 1) | - ======================================================= - {"lower_bin":1, "upper_bin":3, "count":0.333333} | - {"lower_bin":3, "upper_bin":5, "count":0.333333} | - {"lower_bin":5, "upper_bin":7, "count":0.333333} | - Query OK, 3 row(s) in set (0.004273s) +Selector functiosn choose one or more rows in the query result set to retrun according toe the semantics. You can specify to output ts column and other columns including tbname and tags so that you can easily know which rows the selected values belong to. -taos> SELECT HISTOGRAM(voltage, 'linear_bin', '{"start": 1, "width": 3, "count": 3, "infinity": false}', 0) FROM meters; - histogram(voltage, 'linear_bin', '{"start": 1, "width": 3, " | - =================================================================== - {"lower_bin":1, "upper_bin":4, "count":3} | - {"lower_bin":4, "upper_bin":7, "count":3} | - {"lower_bin":7, "upper_bin":10, "count":3} | - Query OK, 3 row(s) in set (0.004887s) - -taos> SELECT HISTOGRAM(voltage, 'log_bin', '{"start": 1, "factor": 3, "count": 3, "infinity": true}', 0) FROM meters; - histogram(voltage, 'log_bin', '{"start": 1, "factor": 3, "count" | - =================================================================== - {"lower_bin":-inf, "upper_bin":1, "count":3} | - {"lower_bin":1, "upper_bin":3, "count":2} | - {"lower_bin":3, "upper_bin":9, "count":6} | - {"lower_bin":9, "upper_bin":27, "count":3} | - {"lower_bin":27, "upper_bin":inf, "count":1} | -``` - -### ELAPSED - -```mysql -SELECT ELAPSED(field_name[, time_unit]) FROM { tb_name | stb_name } [WHERE clause] [INTERVAL(interval [, offset]) [SLIDING sliding]]; -``` - -**Description**：`elapsed` function can be used to calculate the continuous time length in which there is valid data. If it's used with `INTERVAL` clause, the returned result is the calcualted time length within each time window. If it's used without `INTERVAL` caluse, the returned result is the calculated time length within the specified time range. Please be noted that the return value of `elapsed` is the number of `time_unit` in the calculated time length. - -**Return value type**：Double - -**Applicable Column type**：Timestamp - -**Applicable versions**：Sicne version 2.6.0.0 - -**Applicable tables**: table, STable, outter in nested query - -**Explanations**： -- `field_name` parameter can only be the first column of a table, i.e. timestamp primary key. -- The minimum value of `time_unit` is the time precision of the database. If `time_unit` is not specified, the time precision of the database is used as the default ime unit. -- It can be used with `INTERVAL` to get the time valid time length of each time window. Please be noted that the return value is same as the time window for all time windows except for the first and the last time window. -- `order by asc/desc` has no effect on the result. -- `group by tbname` must be used together when `elapsed` is used against a STable. -- `group by` must NOT be used together when `elapsed` is used against a table or sub table. -- When used in nested query, it's only applicable when the inner query outputs an implicit timestamp column as the primary key. For example, `select elapsed(ts) from (select diff(value) from sub1)` is legal usage while `select elapsed(ts) from (select * from sub1)` is not. -- It can't be used with `leastsquares`, `diff`, `derivative`, `top`, `bottom`, `last_row`, `interp`. - -## Selection Functions - -When any select function is used, timestamp column or tag columns including `tbname` can be specified to show that the selected value are from which rows. - -### MIN +### APERCENTILE ``` -SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; +SELECT APERCENTILE(field_name, P[, algo_type]) +FROM { tb_name | stb_name } [WHERE clause] ``` -**Description**: The minimum value of a specific column in a table or STable +**Description**: Similar to `PERCENTILE`, but a simulated result is returned -**Return value type**: Same as the data type of the column being operated upon +**Return value type**: Double precision floating point **Applicable column types**: Data types except for timestamp, binary, nchar and bool **Applicable table types**: table, STable -**Examples**: +**More explanations** -``` -taos> SELECT MIN(current), MIN(voltage) FROM meters; - min(current) | min(voltage) | -====================================== - 10.20000 | 218 | -Query OK, 1 row(s) in set (0.001765s) +- _P_ is in range [0,100], when _P_ is 0, the result is same as using function MIN; when _P_ is 100, the result is same as function MAX. +- **algo_type** can only be input as `default` or `t-digest`, if it's not specified `default` will be used, i.e. `apercentile(column_name, 50)` is same as `apercentile(column_name, 50, "default")`. +- When `t-digest` is used, `t-digest` sampling is used to calculate. -taos> SELECT MIN(current), MIN(voltage) FROM d1001; - min(current) | min(voltage) | -====================================== - 10.30000 | 218 | -Query OK, 1 row(s) in set (0.000950s) -``` - -### MAX - -``` -SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The maximum value of a specific column of a table or STable - -**Return value type**: Same as the data type of the column being operated upon - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table, STable - -**Examples**: - -``` -taos> SELECT MAX(current), MAX(voltage) FROM meters; - max(current) | max(voltage) | -====================================== - 13.40000 | 223 | -Query OK, 1 row(s) in set (0.001123s) - -taos> SELECT MAX(current), MAX(voltage) FROM d1001; - max(current) | max(voltage) | -====================================== - 12.60000 | 221 | -Query OK, 1 row(s) in set (0.000987s) -``` - -### FIRST - -``` -SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The first non-null value of a specific column in a table or STable - -**Return value type**: Same as the column being operated upon - -**Applicable column types**: Any data type - -**Applicable table types**: table, STable - -**More explanations**: - -- FIRST(\*) can be used to get the first non-null value of all columns -- NULL will be returned if all the values of the specified column are all NULL -- A result will NOT be returned if all the columns in the result set are all NULL - -**Examples**: - -``` -taos> SELECT FIRST(*) FROM meters; - first(ts) | first(current) | first(voltage) | first(phase) | -========================================================================================= -2018-10-03 14:38:04.000 | 10.20000 | 220 | 0.23000 | -Query OK, 1 row(s) in set (0.004767s) - -taos> SELECT FIRST(current) FROM d1002; - first(current) | -======================= - 10.20000 | -Query OK, 1 row(s) in set (0.001023s) -``` - -### LAST - -``` -SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The last non-NULL value of a specific column in a table or STable - -**Return value type**: Same as the column being operated upon - -**Applicable column types**: Any data type - -**Applicable table types**: table, STable - -**More explanations**: - -- LAST(\*) can be used to get the last non-NULL value of all columns -- If the values of a column in the result set are all NULL, NULL is returned for that column; if all columns in the result are all NULL, no result will be returned. -- When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times. - -**Examples**: - -``` -taos> SELECT LAST(*) FROM meters; - last(ts) | last(current) | last(voltage) | last(phase) | -======================================================================================== -2018-10-03 14:38:16.800 | 12.30000 | 221 | 0.31000 | -Query OK, 1 row(s) in set (0.001452s) - -taos> SELECT LAST(current) FROM d1002; - last(current) | -======================= - 10.30000 | -Query OK, 1 row(s) in set (0.000843s) -``` - -### TOP - -``` -SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The greatest _k_ values of a specific column in a table or STable. If a value has multiple occurrences in the column but counting all of them in will exceed the upper limit _k_, then a part of them will be returned randomly. - -**Return value type**: Same as the column being operated upon - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table, STable - -**More explanations**: - -- _k_ must be in range [1,100] -- The timestamp associated with the selected values are returned too -- Can't be used with `FILL` - -**Examples**: - -``` -taos> SELECT TOP(current, 3) FROM meters; - ts | top(current, 3) | -================================================= -2018-10-03 14:38:15.000 | 12.60000 | -2018-10-03 14:38:16.600 | 13.40000 | -2018-10-03 14:38:16.800 | 12.30000 | -Query OK, 3 row(s) in set (0.001548s) - -taos> SELECT TOP(current, 2) FROM d1001; - ts | top(current, 2) | -================================================= -2018-10-03 14:38:15.000 | 12.60000 | -2018-10-03 14:38:16.800 | 12.30000 | -Query OK, 2 row(s) in set (0.000810s) -``` +**Nested query**: It can be used in both the outer query and inner query in a nested query. ### BOTTOM @@ -548,100 +868,13 @@ SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; - The timestamp associated with the selected values are returned too - Can't be used with `FILL` -**Examples**: +### FIRST ``` -taos> SELECT BOTTOM(voltage, 2) FROM meters; - ts | bottom(voltage, 2) | -=============================================== -2018-10-03 14:38:15.000 | 218 | -2018-10-03 14:38:16.650 | 218 | -Query OK, 2 row(s) in set (0.001332s) - -taos> SELECT BOTTOM(current, 2) FROM d1001; - ts | bottom(current, 2) | -================================================= -2018-10-03 14:38:05.000 | 10.30000 | -2018-10-03 14:38:16.800 | 12.30000 | -Query OK, 2 row(s) in set (0.000793s) +SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` -### PERCENTILE - -``` -SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; -``` - -**Description**: The value whose rank in a specific column matches the specified percentage. If such a value matching the specified percentage doesn't exist in the column, an interpolation value will be returned. - -**Return value type**: Double precision floating point - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table - -**More explanations**: _P_ is in range [0,100], when _P_ is 0, the result is same as using function MIN; when _P_ is 100, the result is same as function MAX. - -**Examples**: - -``` -taos> SELECT PERCENTILE(current, 20) FROM d1001; -percentile(current, 20) | -============================ - 11.100000191 | -Query OK, 1 row(s) in set (0.000787s) -``` - -### APERCENTILE - -``` -SELECT APERCENTILE(field_name, P[, algo_type]) -FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: Similar to `PERCENTILE`, but a simulated result is returned - -**Return value type**: Double precision floating point - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table, STable - -**More explanations** - -- _P_ is in range [0,100], when _P_ is 0, the result is same as using function MIN; when _P_ is 100, the result is same as function MAX. -- **algo_type** can only be input as `default` or `t-digest`, if it's not specified `default` will be used, i.e. `apercentile(column_name, 50)` is same as `apercentile(column_name, 50, "default")`. -- When `t-digest` is used, `t-digest` sampling is used to calculate. It can be used from version 2.2.0.0. - -**Nested query**: It can be used in both the outer query and inner query in a nested query. - -``` -taos> SELECT APERCENTILE(current, 20) FROM d1001; -apercentile(current, 20) | -============================ - 10.300000191 | -Query OK, 1 row(s) in set (0.000645s) - -taos> select apercentile (count, 80, 'default') from stb1; - apercentile (c0, 80, 'default') | -================================== - 601920857.210056424 | -Query OK, 1 row(s) in set (0.012363s) - -taos> select apercentile (count, 80, 't-digest') from stb1; - apercentile (c0, 80, 't-digest') | -=================================== - 605869120.966666579 | -Query OK, 1 row(s) in set (0.011639s) -``` - -### LAST_ROW - -``` -SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; -``` - -**Description**: The last row of a table or STable +**Description**: The first non-null value of a specific column in a table or STable **Return value type**: Same as the column being operated upon @@ -651,26 +884,11 @@ SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; **More explanations**: -- When it's used against a STable, multiple rows with the same and largest timestamp may exist, in this case one of them is returned randomly and it's not guaranteed that the result is same if the query is run multiple times. -- Can't be used with `INTERVAL`. +- FIRST(\*) can be used to get the first non-null value of all columns +- NULL will be returned if all the values of the specified column are all NULL +- A result will NOT be returned if all the columns in the result set are all NULL -**Examples**: - -``` - taos> SELECT LAST_ROW(current) FROM meters; - last_row(current) | - ======================= - 12.30000 | - Query OK, 1 row(s) in set (0.001238s) - - taos> SELECT LAST_ROW(current) FROM d1002; - last_row(current) | - ======================= - 10.30000 | - Query OK, 1 row(s) in set (0.001042s) -``` - -### INTERP [Since version 2.3.1] +### INTERP ``` SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [ RANGE(timestamp1,timestamp2) ] [EVERY(interval)] [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})]; @@ -694,91 +912,89 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [ - `INTERP` can only be used to interpolate in single timeline. So it must be used with `group by tbname` when it's used on a STable. It can't be used with `GROUP BY` when it's used in the inner query of a nested query. - The result of `INTERP` is not influenced by `ORDER BY TIMESTAMP`, which impacts the output order only.. -**Examples**: Based on the `meters` schema used throughout the documents - -- Single point linear interpolation between "2017-07-14 18:40:00" and "2017-07-14 18:40:00: +### LAST ``` - taos> SELECT INTERP(current) FROM t1 RANGE('2017-7-14 18:40:00','2017-7-14 18:40:00') FILL(LINEAR); +SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` -- Get original data every 5 seconds, no interpolation, between "2017-07-14 18:00:00" and "2017-07-14 19:00:00: - -``` - taos> SELECT INTERP(current) FROM t1 RANGE('2017-7-14 18:00:00','2017-7-14 19:00:00') EVERY(5s); -``` - -- Linear interpolation every 5 seconds between "2017-07-14 18:00:00" and "2017-07-14 19:00:00: - -``` - taos> SELECT INTERP(current) FROM t1 RANGE('2017-7-14 18:00:00','2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR); -``` - -- Backward interpolation every 5 seconds - -``` - taos> SELECT INTERP(current) FROM t1 EVERY(5s) FILL(NEXT); -``` - -- Linear interpolation every 5 seconds between "2017-07-14 17:00:00" and "2017-07-14 20:00:00" - -``` - taos> SELECT INTERP(current) FROM t1 where ts >= '2017-07-14 17:00:00' and ts <= '2017-07-14 20:00:00' RANGE('2017-7-14 18:00:00','2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR); -``` - -### INTERP [Since version 2.0.15.0] - -``` -SELECT INTERP(field_name) FROM { tb_name | stb_name } WHERE ts='timestamp' [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})]; -``` - -**Description**: The value of a specific column that matches the specified time slice +**Description**: The last non-NULL value of a specific column in a table or STable **Return value type**: Same as the column being operated upon -**Applicable column types**: Numeric data type +**Applicable column types**: Any data type **Applicable table types**: table, STable **More explanations**: -- Time slice must be specified. If there is no data matching the specified time slice, interpolation is performed based on `FILL` parameter. Conditions such as tags or `tbname` can be used `Where` clause can be used to filter data. -- The timestamp specified must be within the time range of the data rows of the table or STable. If it is beyond the valid time range, nothing is returned even with `FILL` parameter. -- `INTERP` can be used to query only single time point once. `INTERP` can be used with `EVERY` to get the interpolation value every time interval. -- **Examples**: +- LAST(\*) can be used to get the last non-NULL value of all columns +- If the values of a column in the result set are all NULL, NULL is returned for that column; if all columns in the result are all NULL, no result will be returned. +- When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times. + +### LAST_ROW ``` - taos> SELECT INTERP(*) FROM meters WHERE ts='2017-7-14 18:40:00.004'; - interp(ts) | interp(current) | interp(voltage) | interp(phase) | - ========================================================================================== - 2017-07-14 18:40:00.004 | 9.84020 | 216 | 0.32222 | - Query OK, 1 row(s) in set (0.002652s) +SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; ``` -If there is no data corresponding to the specified timestamp, an interpolation value is returned if interpolation policy is specified by `FILL` parameter; or nothing is returned. +**Description**: The last row of a table or STable + +**Return value type**: Same as the column being operated upon + +**Applicable column types**: Any data type + +**Applicable table types**: table, STable + +**More explanations**: + +- When it's used against a STable, multiple rows with the same and largest timestamp may exist, in this case one of them is returned randomly and it's not guaranteed that the result is same if the query is run multiple times. +- Can't be used with `INTERVAL`. + +### MAX ``` - taos> SELECT INTERP(*) FROM meters WHERE tbname IN ('d636') AND ts='2017-7-14 18:40:00.005'; - Query OK, 0 row(s) in set (0.004022s) - - taos> SELECT INTERP(*) FROM meters WHERE tbname IN ('d636') AND ts='2017-7-14 18:40:00.005' FILL(PREV); - interp(ts) | interp(current) | interp(voltage) | interp(phase) | - ========================================================================================== - 2017-07-14 18:40:00.005 | 9.88150 | 217 | 0.32500 | - Query OK, 1 row(s) in set (0.003056s) +SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` -Interpolation is performed every 5 milliseconds between `['2017-7-14 18:40:00', '2017-7-14 18:40:00.014']` +**Description**: The maximum value of a specific column of a table or STable + +**Return value type**: Same as the data type of the column being operated upon + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table, STable + +### MIN ``` - taos> SELECT INTERP(current) FROM d636 WHERE ts>='2017-7-14 18:40:00' AND ts<='2017-7-14 18:40:00.014' EVERY(5a); - ts | interp(current) | - ================================================= - 2017-07-14 18:40:00.000 | 10.04179 | - 2017-07-14 18:40:00.010 | 10.16123 | - Query OK, 2 row(s) in set (0.003487s) +SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; ``` +**Description**: The minimum value of a specific column in a table or STable + +**Return value type**: Same as the data type of the column being operated upon + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table, STable + +### PERCENTILE + +``` +SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; +``` + +**Description**: The value whose rank in a specific column matches the specified percentage. If such a value matching the specified percentage doesn't exist in the column, an interpolation value will be returned. + +**Return value type**: Double precision floating point + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table + +**More explanations**: _P_ is in range [0,100], when _P_ is 0, the result is same as using function MIN; when _P_ is 100, the result is same as function MAX. + ### TAIL ``` @@ -793,30 +1009,26 @@ SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause]; **Applicable column types**: Any data type except form timestamp, i.e. the primary key -**Applicable versions**: Since version 2.6.0.0 - -**Examples**: +### TOP ``` -taos> select ts,dbig from tail2; - ts | dbig | -================================================== -2021-10-15 00:31:33.000 | 1 | -2021-10-17 00:31:31.000 | NULL | -2021-12-24 00:31:34.000 | 2 | -2022-01-01 08:00:05.000 | 19 | -2022-01-01 08:00:06.000 | NULL | -2022-01-01 08:00:07.000 | 9 | -Query OK, 6 row(s) in set (0.001952s) - -taos> select tail(dbig,2,2) from tail2; -ts | tail(dbig,2,2) | -================================================== -2021-12-24 00:31:34.000 | 2 | -2022-01-01 08:00:05.000 | 19 | -Query OK, 2 row(s) in set (0.002307s) +SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ``` +**Description**: The greatest _k_ values of a specific column in a table or STable. If a value has multiple occurrences in the column but counting all of them in will exceed the upper limit _k_, then a part of them will be returned randomly. + +**Return value type**: Same as the column being operated upon + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table, STable + +**More explanations**: + +- _k_ must be in range [1,100] +- The timestamp associated with the selected values are returned too +- Can't be used with `FILL` + ### UNIQUE ``` @@ -829,179 +1041,14 @@ SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause]; **Applicable column types**: Any data types except for timestamp -**Applicable versions**: Since version 2.6.0.0 - **More explanations**: - It can be used against table or STable, but can't be used together with time window, like `interval`, `state_window` or `session_window` . - Considering the number of result sets is unpredictable, it's suggested to limit the distinct values under 100,000 to control the memory usage, otherwise error will be returned. -**Examples**: +## Time-Series Specific Functions -``` -taos> select ts,voltage from unique1; - ts | voltage | -================================================== -2021-10-17 00:31:31.000 | 1 | -2022-01-24 00:31:31.000 | 1 | -2021-10-17 00:31:31.000 | 1 | -2021-12-24 00:31:31.000 | 2 | -2022-01-01 08:00:01.000 | 19 | -2021-10-17 00:31:31.000 | NULL | -2022-01-01 08:00:02.000 | NULL | -2022-01-01 08:00:03.000 | 9 | -Query OK, 8 row(s) in set (0.003018s) - -taos> select unique(voltage) from unique1; -ts | unique(voltage) | -================================================== -2021-10-17 00:31:31.000 | 1 | -2021-10-17 00:31:31.000 | NULL | -2021-12-24 00:31:31.000 | 2 | -2022-01-01 08:00:01.000 | 19 | -2022-01-01 08:00:03.000 | 9 | -Query OK, 5 row(s) in set (0.108458s) -``` - -## Scalar functions - -### DIFF - -```sql -SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHERE clause]; -``` - -**Description**: The different of each row with its previous row for a specific column. `ignore_negative` can be specified as 0 or 1, the default value is 1 if it's not specified. `1` means negative values are ignored. - -**Return value type**: Same as the column being operated upon - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table, STable - -**More explanations**: - -- The number of result rows is the number of rows subtracted by one, no output for the first row -- Since version 2.1.30, `DIFF` can be used on STable with `GROUP by tbname` -- Since version 2.6.0, `ignore_negative` parameter is supported - -**Examples**: - -```sql -taos> SELECT DIFF(current) FROM d1001; - ts | diff(current) | -================================================= -2018-10-03 14:38:15.000 | 2.30000 | -2018-10-03 14:38:16.800 | -0.30000 | -Query OK, 2 row(s) in set (0.001162s) -``` - -### DERIVATIVE - -``` -SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHERE clause]; -``` - -**Description**: The derivative of a specific column. The time rage can be specified by parameter `time_interval`, the minimum allowed time range is 1 second (1s); the value of `ignore_negative` can be 0 or 1, 1 means negative values are ignored. - -**Return value type**: Double precision floating point - -**Applicable column types**: Data types except for timestamp, binary, nchar and bool - -**Applicable table types**: table, STable - -**More explanations**: - -- It is available from version 2.1.3.0, the number of result rows is the number of total rows in the time range subtracted by one, no output for the first row. -- It can be used together with `GROUP BY tbname` against a STable. - -**Examples**: - -``` -taos> select derivative(current, 10m, 0) from t1; - ts | derivative(current, 10m, 0) | -======================================================== - 2021-08-20 10:11:22.790 | 0.500000000 | - 2021-08-20 11:11:22.791 | 0.166666620 | - 2021-08-20 12:11:22.791 | 0.000000000 | - 2021-08-20 13:11:22.792 | 0.166666620 | - 2021-08-20 14:11:22.792 | -0.666666667 | -Query OK, 5 row(s) in set (0.004883s) -``` - -### SPREAD - -``` -SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The difference between the max and the min of a specific column - -**Return value type**: Double precision floating point - -**Applicable column types**: Data types except for binary, nchar, and bool - -**Applicable table types**: table, STable - -**More explanations**: Can be used on a column of TIMESTAMP type, the result is the time range size. - -**Examples**: - -``` -taos> SELECT SPREAD(voltage) FROM meters; - spread(voltage) | -============================ - 5.000000000 | -Query OK, 1 row(s) in set (0.001792s) - -taos> SELECT SPREAD(voltage) FROM d1001; - spread(voltage) | -============================ - 3.000000000 | -Query OK, 1 row(s) in set (0.000836s) -``` - -### CEIL - -``` -SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The rounded up value of a specific column - -**Return value type**: Same as the column being used - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and outer query - -**More explanations**: - -- Can't be used on any tags of any type -- Arithmetic operation can be performed on the result of `ceil` function -- Can't be used with aggregate functions - -### FLOOR - -``` -SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The rounded down value of a specific column - -**More explanations**: The restrictions are same as those of the `CEIL` function. - -### ROUND - -``` -SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The rounded value of a specific column. - -**More explanations**: The restrictions are same as `CEIL` function. +TDengine provides a set of time-series specific functions to better meet the requirements in querying time-series data. In general databases, similar functionalities can only be achieved with much more complex syntax and much worse performance. TDengine provides these functionalities in builtin functions so that the burden on user side is minimized. ### CSUM @@ -1026,7 +1073,61 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; - Can only be used with aggregate functions - `Group by tbname` must be used together on a STable to force the result on a single timeline -**Applicable versions**: Since 2.3.0.x +### DERIVATIVE + +``` +SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHERE clause]; +``` + +**Description**: The derivative of a specific column. The time rage can be specified by parameter `time_interval`, the minimum allowed time range is 1 second (1s); the value of `ignore_negative` can be 0 or 1, 1 means negative values are ignored. + +**Return value type**: Double precision floating point + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table, STable + +**More explanations**: + +- The number of result rows is the number of total rows in the time range subtracted by one, no output for the first row. +- It can be used together with `GROUP BY tbname` against a STable. + +### DIFF + +```sql +SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHERE clause]; +``` + +**Description**: The different of each row with its previous row for a specific column. `ignore_negative` can be specified as 0 or 1, the default value is 1 if it's not specified. `1` means negative values are ignored. + +**Return value type**: Same as the column being operated upon + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table, STable + +**More explanations**: + +- The number of result rows is the number of rows subtracted by one, no output for the first row +- It can be used on STable with `GROUP by tbname` + +### IRATE + +``` +SELECT IRATE(field_name) FROM tb_name WHERE clause; +``` + +**Description**: instantaneous rate on a specific column. The last two samples in the specified time range are used to calculate instantaneous rate. If the last sample value is smaller, then only the last sample value is used instead of the difference between the last two sample values. + +**Return value type**: Double precision floating number + +**Applicable column types**: Data types except for timestamp, binary, nchar and bool + +**Applicable table types**: table, STable + +**More explanations**: + +- It can be used on stble with `GROUP BY`, i.e. timelines generated by `GROUP BY tbname` on a STable. ### MAVG @@ -1034,7 +1135,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause] ``` -**Description**: The moving average of continuous _k_ values of a specific column. If the number of input rows is less than _k_, nothing is returned. The applicable range is _k_ is [1,1000]. +**Description**: The moving average of continuous _k_ values of a specific column. If the number of input rows is less than _k_, nothing is returned. The applicable range of _k_ is [1,1000]. **Return value type**: Double precision floating point @@ -1051,8 +1152,6 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; - Can't be used with aggregate functions. - Must be used with `GROUP BY tbname` when it's used on a STable to force the result on each single timeline. -**Applicable versions**: Since 2.3.0.x - ### SAMPLE ```sql @@ -1074,469 +1173,6 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; - Arithmetic operation can't be operated on the result of `SAMPLE` function - Must be used with `Group by tbname` when it's used on a STable to force the result on each single timeline -**Applicable versions**: Since 2.3.0.x - -### ASIN - -```sql -SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The anti-sine of a specific column - -**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### ACOS - -```sql -SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The anti-cosine of a specific column - -**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### ATAN - -```sql -SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: anti-tangent of a specific column - -**Description**: The anti-cosine of a specific column - -**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### SIN - -```sql -SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The sine of a specific column - -**Description**: The anti-cosine of a specific column - -**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### COS - -```sql -SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The cosine of a specific column - -**Description**: The anti-cosine of a specific column - -**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### TAN - -```sql -SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The tangent of a specific column - -**Description**: The anti-cosine of a specific column - -**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### POW - -```sql -SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The power of a specific column with `power` as the index - -**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### LOG - -```sql -SELECT LOG(field_name, base) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The log of a specific with `base` as the radix - -**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### ABS - -```sql -SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The absolute of a specific column - -**Return value type**: UBIGINT if the input value is integer; DOUBLE if the input value is FLOAT/DOUBLE - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### SQRT - -```sql -SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The square root of a specific column - -**Return value type**: Double if the input value is not NULL; or NULL if the input value is NULL - -**Applicable data types**: Data types except for timestamp, binary, nchar, bool - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Can't be used with tags -- Can't be used with aggregate functions - -### CAST - -```sql -SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: It's used for type casting. The input parameter `expression` can be data columns, constants, scalar functions or arithmetic between them. Can't be used with tags, and can only be used in `select` clause. - -**Return value type**: The type specified by parameter `type_name` - -**Applicable data types**: - -- Parameter `expression` can be any data type except for JSON, more specifically it can be any of BOOL/TINYINT/SMALLINT/INT/BIGINT/FLOAT/DOUBLE/BINARY(M)/TIMESTAMP/NCHAR(M)/TINYINT UNSIGNED/SMALLINT UNSIGNED/INT UNSIGNED/BIGINT UNSIGNED -- The output data type specified by `type_name` can only be one of BIGINT/BINARY(N)/TIMESTAMP/NCHAR(N)/BIGINT UNSIGNED - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- Error will be reported for unsupported type casting -- NULL will be returned if the input value is NULL -- Some values of some supported data types may not be casted, below are known issues: - 1）When casting BINARY/NCHAR to BIGINT/BIGINT UNSIGNED, some characters may be treated as illegal, for example "a" may be converted to 0. - 2）There may be overflow when casting singed integer or TIMESTAMP to unsigned BIGINT - 3）There may be overflow when casting unsigned BIGINT to BIGINT - 4）There may be overflow when casting FLOAT/DOUBLE to BIGINT or UNSIGNED BIGINT - -### CONCAT - -```sql -SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The concatenation result of two or more strings, the number of strings to be concatenated is at least 2 and at most 8 - -**Return value type**: Same as the columns being operated, BINARY or NCHAR; or NULL if all the input are NULL - -**Applicable data types**: The input data must be in either all BINARY or in all NCHAR; can't be used on tag columns - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -### CONCAT_WS - -``` -SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The concatenation result of two or more strings with separator, the number of strings to be concatenated is at least 3 and at most 9 - -**Return value type**: Same as the columns being operated, BINARY or NCHAR; or NULL if all the input are NULL - -**Applicable data types**: The input data must be in either all BINARY or in all NCHAR; can't be used on tag columns - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- If the value of `separator` is NULL, the output is NULL. If the value of `separator` is not NULL but other input are all NULL, the output is empty string. - -### LENGTH - -``` -SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The length in bytes of a string - -**Return value type**: Integer - -**Applicable data types**: BINARY or NCHAR, can't be used on tags - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations** - -- If the input value is NULL, the output is NULL too - -### CHAR_LENGTH - -``` -SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The length in number of characters of a string - -**Return value type**: Integer - -**Applicable data types**: BINARY or NCHAR, can't be used on tags - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations** - -- If the input value is NULL, the output is NULL too - -### LOWER - -``` -SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: Convert the input string to lower case - -**Return value type**: Same as input - -**Applicable data types**: BINARY or NCHAR, can't be used on tags - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations** - -- If the input value is NULL, the output is NULL too - -### UPPER - -``` -SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: Convert the input string to upper case - -**Return value type**: Same as input - -**Applicable data types**: BINARY or NCHAR, can't be used on tags - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations** - -- If the input value is NULL, the output is NULL too - -### LTRIM - -``` -SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: Remove the left leading blanks of a string - -**Return value type**: Same as input - -**Applicable data types**: BINARY or NCHAR, can't be used on tags - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations** - -- If the input value is NULL, the output is NULL too - -### RTRIM - -``` -SELECT RTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: Remove the right tailing blanks of a string - -**Return value type**: Same as input - -**Applicable data types**: BINARY or NCHAR, can't be used on tags - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations** - -- If the input value is NULL, the output is NULL too - -### SUBSTR - -``` -SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause] -``` - -**Description**: The sub-string starting from `pos` with length of `len` from the original string `str` - -**Return value type**: Same as input - -**Applicable data types**: BINARY or NCHAR, can't be used on tags - -**Applicable table types**: table, STable - -**Applicable nested query**: Inner query and Outer query - -**Applicable versions**: From 2.6.0.0 - -**More explanations**: - -- If the input is NULL, the output is NULL -- Parameter `pos` can be an positive or negative integer; If it's positive, the starting position will be counted from the beginning of the string; if it's negative, the starting position will be counted from the end of the string. -- If `len` is not specified, it means from `pos` to the end. - ### STATECOUNT ``` @@ -1558,39 +1194,11 @@ SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clau **Applicable nested query**: Outer query only -**Applicable versions**: From 2.6.0.0 - **More explanations**: - Must be used together with `GROUP BY tbname` when it's used on a STable to force the result into each single timeline] - Can't be used with window operation, like interval/state_window/session_window -**Examples**: - -``` -taos> select ts,dbig from statef2; - ts | dbig | -======================================================== -2021-10-15 00:31:33.000000000 | 1 | -2021-10-17 00:31:31.000000000 | NULL | -2021-12-24 00:31:34.000000000 | 2 | -2022-01-01 08:00:05.000000000 | 19 | -2022-01-01 08:00:06.000000000 | NULL | -2022-01-01 08:00:07.000000000 | 9 | -Query OK, 6 row(s) in set (0.002977s) - -taos> select stateCount(dbig,GT,2) from statef2; -ts | dbig | statecount(dbig,gt,2) | -================================================================================ -2021-10-15 00:31:33.000000000 | 1 | -1 | -2021-10-17 00:31:31.000000000 | NULL | NULL | -2021-12-24 00:31:34.000000000 | 2 | -1 | -2022-01-01 08:00:05.000000000 | 19 | 1 | -2022-01-01 08:00:06.000000000 | NULL | NULL | -2022-01-01 08:00:07.000000000 | 9 | 2 | -Query OK, 6 row(s) in set (0.002791s) -``` - ### STATEDURATION ``` @@ -1613,320 +1221,59 @@ SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [W **Applicable nested query**: Outer query only -**Applicable versions**: From 2.6.0.0 - **More explanations**: - Must be used together with `GROUP BY tbname` when it's used on a STable to force the result into each single timeline] - Can't be used with window operation, like interval/state_window/session_window -**Examples**: +### TWA ``` -taos> select ts,dbig from statef2; - ts | dbig | -======================================================== -2021-10-15 00:31:33.000000000 | 1 | -2021-10-17 00:31:31.000000000 | NULL | -2021-12-24 00:31:34.000000000 | 2 | -2022-01-01 08:00:05.000000000 | 19 | -2022-01-01 08:00:06.000000000 | NULL | -2022-01-01 08:00:07.000000000 | 9 | -Query OK, 6 row(s) in set (0.002407s) - -taos> select stateDuration(dbig,GT,2) from statef2; -ts | dbig | stateduration(dbig,gt,2) | -=================================================================================== -2021-10-15 00:31:33.000000000 | 1 | -1 | -2021-10-17 00:31:31.000000000 | NULL | NULL | -2021-12-24 00:31:34.000000000 | 2 | -1 | -2022-01-01 08:00:05.000000000 | 19 | 0 | -2022-01-01 08:00:06.000000000 | NULL | NULL | -2022-01-01 08:00:07.000000000 | 9 | 2 | -Query OK, 6 row(s) in set (0.002613s) +SELECT TWA(field_name) FROM tb_name WHERE clause; ``` -## Time Functions +**Description**: Time weighted average on a specific column within a time range -Since version 2.6.0.0, below time related functions can be used in TDengine. +**Return value type**: Double precision floating number -### NOW - -```sql -SELECT NOW() FROM { tb_name | stb_name } [WHERE clause]; -SELECT select_expr FROM { tb_name | stb_name } WHERE ts_col cond_operatior NOW(); -INSERT INTO tb_name VALUES (NOW(), ...); -``` - -**Description**: The current time of the client side system - -**Return value type**: TIMESTAMP - -**Applicable column types**: TIMESTAMP only +**Applicable column types**: Data types except for timestamp, binary, nchar and bool **Applicable table types**: table, STable **More explanations**: -- Add and Subtract operation can be performed, for example NOW() + 1s, the time unit can be: - b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), w(week) -- The precision of the returned timestamp is same as the precision set for the current data base in use +- It can be used on stable with `GROUP BY`, i.e. timelines generated by `GROUP BY tbname` on a STable. -**Examples**: +## System Information Functions -```sql -taos> SELECT NOW() FROM meters; - now() | -========================== - 2022-02-02 02:02:02.456 | -Query OK, 1 row(s) in set (0.002093s) +### DATABASE -taos> SELECT NOW() + 1h FROM meters; - now() + 1h | -========================== - 2022-02-02 03:02:02.456 | -Query OK, 1 row(s) in set (0.002093s) - -taos> SELECT COUNT(voltage) FROM d1001 WHERE ts < NOW(); - count(voltage) | -============================= - 5 | -Query OK, 5 row(s) in set (0.004475s) - -taos> INSERT INTO d1001 VALUES (NOW(), 10.2, 219, 0.32); -Query OK, 1 of 1 row(s) in database (0.002210s) +``` +SELECT DATABASE(); ``` -### TODAY +**Description**：Return the current database being used. If the user doesn't specify database when logon and doesn't use `USE` SQL command to switch the datbase, this function returns NULL. -```sql -SELECT TODAY() FROM { tb_name | stb_name } [WHERE clause]; -SELECT select_expr FROM { tb_name | stb_name } WHERE ts_col cond_operatior TODAY()]; -INSERT INTO tb_name VALUES (TODAY(), ...); +### CLIENT_VERSION + +``` +SELECT CLIENT_VERSION(); ``` -**Description**: The timestamp of 00:00:00 of the client side system +**Description**：Return the client version. -**Return value type**: TIMESTAMP +### SERVER_VERSION -**Applicable column types**: TIMESTAMP only - -**Applicable table types**: table, STable - -**More explanations**: - -- Add and Subtract operation can be performed, for example NOW() + 1s, the time unit can be: - b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), w(week) -- The precision of the returned timestamp is same as the precision set for the current data base in use - -**Examples**: - -```sql -taos> SELECT TODAY() FROM meters; - today() | -========================== - 2022-02-02 00:00:00.000 | -Query OK, 1 row(s) in set (0.002093s) - -taos> SELECT TODAY() + 1h FROM meters; - today() + 1h | -========================== - 2022-02-02 01:00:00.000 | -Query OK, 1 row(s) in set (0.002093s) - -taos> SELECT COUNT(voltage) FROM d1001 WHERE ts < TODAY(); - count(voltage) | -============================= - 5 | -Query OK, 5 row(s) in set (0.004475s) - -taos> INSERT INTO d1001 VALUES (TODAY(), 10.2, 219, 0.32); -Query OK, 1 of 1 row(s) in database (0.002210s) +``` +SELECT SERVER_VERSION(); ``` -### TIMEZONE +**Description**：Returns the server version. -```sql -SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause]; +### SERVER_STATUS + +``` +SELECT SERVER_VERSION(); ``` -**Description**: The timezone of the client side system - -**Return value type**: BINARY - -**Applicable column types**: None - -**Applicable table types**: table, STable - -**Examples**: - -```sql -taos> SELECT TIMEZONE() FROM meters; - timezone() | -================================= - UTC (UTC, +0000) | -Query OK, 1 row(s) in set (0.002093s) -``` - -### TO_ISO8601 - -```sql -SELECT TO_ISO8601(ts_val | ts_col) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The ISO8601 date/time format converted from a UNIX timestamp, plus the timezone of the client side system - -**Return value type**: BINARY - -**Applicable column types**: TIMESTAMP, constant or a column - -**Applicable table types**: table, STable - -**More explanations**: - -- If the input is UNIX timestamp constant, the precision of the returned value is determined by the digits of the input timestamp -- If the input is a column of TIMESTAMP type, The precision of the returned value is same as the precision set for the current data base in use - -**Examples**: - -```sql -taos> SELECT TO_ISO8601(1643738400) FROM meters; - to_iso8601(1643738400) | -============================== - 2022-02-02T02:00:00+0800 | - -taos> SELECT TO_ISO8601(ts) FROM meters; - to_iso8601(ts) | -============================== - 2022-02-02T02:00:00+0800 | - 2022-02-02T02:00:00+0800 | - 2022-02-02T02:00:00+0800 | -``` - -### TO_UNIXTIMESTAMP - -```sql -SELECT TO_UNIXTIMESTAMP(datetime_string | ts_col) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: UNIX timestamp converted from a string of date/time format - -**Return value type**: Long integer - -**Applicable column types**: Constant or column of BINARY/NCHAR - -**Applicable table types**: table, STable - -**More explanations**: - -- The input string must be compatible with ISO8601/RFC3339 standard, 0 will be returned if the string can't be converted -- The precision of the returned timestamp is same as the precision set for the current data base in use - -**Examples**: - -```sql -taos> SELECT TO_UNIXTIMESTAMP("2022-02-02T02:00:00.000Z") FROM meters; -to_unixtimestamp("2022-02-02T02:00:00.000Z") | -============================================== - 1643767200000 | - -taos> SELECT TO_UNIXTIMESTAMP(col_binary) FROM meters; - to_unixtimestamp(col_binary) | -======================================== - 1643767200000 | - 1643767200000 | - 1643767200000 | -``` - -### TIMETRUNCATE - -```sql -SELECT TIMETRUNCATE(ts_val | datetime_string | ts_col, time_unit) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: Truncate the input timestamp with unit specified by `time_unit`\ - -**Return value type**: TIMESTAMP\ - -**Applicable column types**: UNIX timestamp constant, string constant of date/time format, or a column of timestamp - -**Applicable table types**: table, STable - -**More explanations**: - -- Time unit specified by `time_unit` can be: - 1u(microsecond),1a(millisecond),1s(second),1m(minute),1h(hour),1d(day). -- The precision of the returned timestamp is same as the precision set for the current data base in use - -**Examples**: - -```sql -taos> SELECT TIMETRUNCATE(1643738522000, 1h) FROM meters; - timetruncate(1643738522000, 1h) | -=================================== - 2022-02-02 02:00:00.000 | -Query OK, 1 row(s) in set (0.001499s) - -taos> SELECT TIMETRUNCATE("2022-02-02 02:02:02", 1h) FROM meters; - timetruncate("2022-02-02 02:02:02", 1h) | -=========================================== - 2022-02-02 02:00:00.000 | -Query OK, 1 row(s) in set (0.003903s) - -taos> SELECT TIMETRUNCATE(ts, 1h) FROM meters; - timetruncate(ts, 1h) | -========================== - 2022-02-02 02:00:00.000 | - 2022-02-02 02:00:00.000 | - 2022-02-02 02:00:00.000 | -Query OK, 3 row(s) in set (0.003903s) -``` - -### TIMEDIFF - -```sql -SELECT TIMEDIFF(ts_val1 | datetime_string1 | ts_col1, ts_val2 | datetime_string2 | ts_col2 [, time_unit]) FROM { tb_name | stb_name } [WHERE clause]; -``` - -**Description**: The difference between two timestamps, and rounded to the time unit specified by `time_unit` - -**Return value type**: Long Integer - -**Applicable column types**: UNIX timestamp constant, string constant of date/time format, or a column of TIMESTAMP type - -**Applicable table types**: table, STable - -**More explanations**: - -- Time unit specified by `time_unit` can be: - 1u(microsecond),1a(millisecond),1s(second),1m(minute),1h(hour),1d(day). -- The precision of the returned timestamp is same as the precision set for the current data base in use - -**Applicable versions**:Since version 2.6.0.0 - -**Examples**: - -```sql -taos> SELECT TIMEDIFF(1643738400000, 1643742000000) FROM meters; - timediff(1643738400000, 1643742000000) | -========================================= - 3600000 | -Query OK, 1 row(s) in set (0.002553s) -taos> SELECT TIMEDIFF(1643738400000, 1643742000000, 1h) FROM meters; - timediff(1643738400000, 1643742000000, 1h) | -============================================= - 1 | -Query OK, 1 row(s) in set (0.003726s) - -taos> SELECT TIMEDIFF("2022-02-02 03:00:00", "2022-02-02 02:00:00", 1h) FROM meters; - timediff("2022-02-02 03:00:00", "2022-02-02 02:00:00", 1h) | -============================================================= - 1 | -Query OK, 1 row(s) in set (0.001937s) - -taos> SELECT TIMEDIFF(ts_col1, ts_col2, 1h) FROM meters; - timediff(ts_col1, ts_col2, 1h) | -=================================== - 1 | -Query OK, 1 row(s) in set (0.001937s) -``` +**Description**：Returns the server's status. From 29e1f5b003d279d95ac88b68d8450ef0874d47a6 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 1 Jun 2022 14:02:31 +0800 Subject: [PATCH 54/57] docs: rephrase titles for function page --- docs-cn/12-taos-sql/07-function.md | 4 ++-- docs-en/12-taos-sql/07-function.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-cn/12-taos-sql/07-function.md b/docs-cn/12-taos-sql/07-function.md index 993a1dac6e..331c2fc035 100644 --- a/docs-cn/12-taos-sql/07-function.md +++ b/docs-cn/12-taos-sql/07-function.md @@ -1,6 +1,6 @@ --- -sidebar_label: SQL 函数 -title: SQL 函数 +sidebar_label: 函数 +title: 函数 toc_max_heading_level: 4 --- diff --git a/docs-en/12-taos-sql/07-function.md b/docs-en/12-taos-sql/07-function.md index 6c45474b88..475575313d 100644 --- a/docs-en/12-taos-sql/07-function.md +++ b/docs-en/12-taos-sql/07-function.md @@ -1,5 +1,5 @@ --- -title: SQL Functions +title: Functions toc_max_heading_level: 4 --- From 752ca8ae2db4dece3b3053ff4e13619686f4f10c Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 1 Jun 2022 14:51:31 +0800 Subject: [PATCH 55/57] docs: remove constraints on tags --- docs-cn/12-taos-sql/07-function.md | 60 ++++++++--------- docs-en/12-taos-sql/07-function.md | 102 +++++++++++------------------ 2 files changed, 68 insertions(+), 94 deletions(-) diff --git a/docs-cn/12-taos-sql/07-function.md b/docs-cn/12-taos-sql/07-function.md index 331c2fc035..04d4adb7d4 100644 --- a/docs-cn/12-taos-sql/07-function.md +++ b/docs-cn/12-taos-sql/07-function.md @@ -20,7 +20,7 @@ toc_max_heading_level: 4 **返回结果类型**：如果输入值为整数，输出值是 UBIGINT 类型。如果输入值是 FLOAT/DOUBLE 数据类型，输出值是 DOUBLE 数据类型。 -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -38,7 +38,7 @@ toc_max_heading_level: 4 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -56,7 +56,7 @@ toc_max_heading_level: 4 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -75,7 +75,7 @@ toc_max_heading_level: 4 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -94,7 +94,7 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：与指定列的原始数据类型一致。例如，如果指定列的原始数据类型为 Float，那么返回的数据类型也为 Float；如果指定列的原始数据类型为 Double，那么返回的数据类型也为 Double。 -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列，无论 tag 列的类型是什么类型。 +**适用数据类型**：数值类型。 **适用于**: 普通表、超级表。 @@ -115,7 +115,7 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -142,7 +142,7 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -161,7 +161,7 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -190,7 +190,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -208,7 +208,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -226,7 +226,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列 +**适用数据类型**：数值类型。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -248,7 +248,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：INT。如果输入值为NULL，输出值为NULL。 -**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 +**适用数据类型**：VARCHAR, NCHAR **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -262,9 +262,9 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **功能说明**：字符串连接函数。 -**返回结果类型**：如果所有参数均为BINARY类型，则结果类型为BINARY。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。 +**返回结果类型**：如果所有参数均为 VARCHAR 类型，则结果类型为 VARCHAR。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。 -**适用数据类型**：BINARY, NCHAR。不能应用在 TAG 列。 该函数最小参数个数为2个，最大参数个数为8个。 +**适用数据类型**：VARCHAR, NCHAR。 该函数最小参数个数为2个，最大参数个数为8个。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -279,9 +279,9 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **功能说明**：带分隔符的字符串连接函数。 -**返回结果类型**：如果所有参数均为BINARY类型，则结果类型为BINARY。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。如果separator值不为NULL，其他输入为NULL，输出为空串。 +**返回结果类型**：如果所有参数均为VARCHAR类型，则结果类型为VARCHAR。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。如果separator值不为NULL，其他输入为NULL，输出为空串。 -**适用数据类型**：BINARY, NCHAR。不能应用在 TAG 列。 该函数最小参数个数为3个，最大参数个数为9个。 +**适用数据类型**：VARCHAR, NCHAR。 该函数最小参数个数为3个，最大参数个数为9个。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -298,7 +298,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：INT。 -**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 +**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -315,7 +315,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 -**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 +**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -332,7 +332,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 -**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 +**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -349,7 +349,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 -**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 +**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -366,7 +366,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 -**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。输入参数pos可以为正数，也可以为负数。如果pos是正数，表示开始位置从字符串开头正数计算。如果pos为负数，表示开始位置从字符串结尾倒数计算。如果输入参数len被忽略，返回的子串包含从pos开始的整个字串。 +**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。输入参数pos可以为正数，也可以为负数。如果pos是正数，表示开始位置从字符串开头正数计算。如果pos为负数，表示开始位置从字符串结尾倒数计算。如果输入参数len被忽略，返回的子串包含从pos开始的整个字串。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -383,7 +383,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。 -**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。 +**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -400,7 +400,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] ``` -**功能说明**：数据类型转换函数，输入参数 expression 支持普通列、常量、标量函数及它们之间的四则运算，不支持 tag 列，只适用于 select 子句中。 +**功能说明**：数据类型转换函数，输入参数 expression 支持普通列、常量、标量函数及它们之间的四则运算，只适用于 select 子句中。 **返回结果类型**：CAST 中指定的类型（type_name），可以是 BIGINT、BIGINT UNSIGNED、BINARY、VARCHAR、NCHAR和TIMESTAMP。 @@ -423,7 +423,7 @@ SELECT TO_ISO8601(ts_val | ts_col) FROM { tb_name | stb_name } [WHERE clause]; **功能说明**：将 UNIX 时间戳转换成为 ISO8601 标准的日期时间格式，并附加客户端时区信息。 -**返回结果数据类型**：BINARY 类型。 +**返回结果数据类型**：VARCHAR 类型。 **适用数据类型**：UNIX 时间戳常量或是 TIMESTAMP 类型的列 @@ -462,7 +462,7 @@ SELECT TO_UNIXTIMESTAMP(datetime_string | ts_col) FROM { tb_name | stb_name } [W **返回结果数据类型**：长整型 INT64。 -**应用字段**：字符串常量或是 BINARY/NCHAR 类型的列。 +**应用字段**：字符串常量或是 VARCHAR/NCHAR 类型的列。 **适用于**：表、超级表。 @@ -549,7 +549,7 @@ SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause]; **功能说明**：返回客户端当前时区信息。 -**返回结果数据类型**：BINARY 类型。 +**返回结果数据类型**：VARCHAR 类型。 **应用字段**：无 @@ -1005,7 +1005,7 @@ SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause]; **返回结果类型**： 输入列如果是整数类型返回值为长整型 （int64_t），浮点数返回值为双精度浮点数（Double）。无符号整数类型返回值为无符号长整型（uint64_t）。 返回结果中同时带有每行记录对应的时间戳。 -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在标签之上。 +**适用数据类型**：数值类型。 **嵌套子查询支持**： 适用于内层查询和外层查询。 @@ -1076,7 +1076,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause; **返回结果类型**： 返回双精度浮点数类型。 - **适用数据类型**： 不能应用在 timestamp、binary、nchar、bool 类型上；在超级表查询中使用时，不能应用在标签之上。 + **适用数据类型**： 数值类型。 **嵌套子查询支持**： 适用于内层查询和外层查询。 @@ -1124,7 +1124,7 @@ SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clau **返回结果类型**：整形。 -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上。 +**适用数据类型**：数值类型。 **嵌套子查询支持**：不支持应用在子查询上。 @@ -1152,7 +1152,7 @@ SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [W **返回结果类型**：整形。 -**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上。 +**适用数据类型**：数值类型。 **嵌套子查询支持**：不支持应用在子查询上。 diff --git a/docs-en/12-taos-sql/07-function.md b/docs-en/12-taos-sql/07-function.md index 475575313d..4eaf7c8a68 100644 --- a/docs-en/12-taos-sql/07-function.md +++ b/docs-en/12-taos-sql/07-function.md @@ -26,8 +26,6 @@ SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query. **More explanations**: - -- Can't be used with tags. - Can't be used with aggregate functions. #### ACOS @@ -47,8 +45,6 @@ SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used with tags - Can't be used with aggregate functions #### ASIN @@ -68,8 +64,6 @@ SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used with tags - Can't be used with aggregate functions #### ATAN @@ -91,8 +85,6 @@ SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used with tags - Can't be used with aggregate functions #### CEIL @@ -112,8 +104,6 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; **Applicable nested query**: Inner query and outer query **More explanations**: - -- Can't be used on any tags of any type - Arithmetic operation can be performed on the result of `ceil` function - Can't be used with aggregate functions @@ -136,8 +126,6 @@ SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used with tags - Can't be used with aggregate functions #### FLOOR @@ -167,8 +155,6 @@ SELECT LOG(field_name, base) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used with tags - Can't be used with aggregate functions #### POW @@ -188,8 +174,6 @@ SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used with tags - Can't be used with aggregate functions #### ROUND @@ -221,8 +205,6 @@ SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used with tags - Can't be used with aggregate functions #### SQRT @@ -242,8 +224,6 @@ SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used with tags - Can't be used with aggregate functions #### TAN @@ -265,8 +245,6 @@ SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause] **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used with tags - Can't be used with aggregate functions ### String Functions @@ -283,7 +261,7 @@ SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] **Return value type**: Integer -**Applicable data types**: BINARY or NCHAR, can't be used on tags +**Applicable data types**: VARCHAR or NCHAR **Applicable table types**: table, STable @@ -301,9 +279,9 @@ SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHER **Description**: The concatenation result of two or more strings, the number of strings to be concatenated is at least 2 and at most 8 -**Return value type**: If all input strings are BINARY type, the result is BINARY type too. If any one of input strings is NCHAR type, then the result is NCHAR. +**Return value type**: If all input strings are VARCHAR type, the result is VARCHAR type too. If any one of input strings is NCHAR type, then the result is NCHAR. -**Applicable data types**: BINARY, NCHAR. Can't be used on tag columns. At least 2 input strings are requird, and at most 8 input strings are allowed. +**Applicable data types**: VARCHAR, NCHAR. Can't be used on tag columns. At least 2 input strings are requird, and at most 8 input strings are allowed. **Applicable table types**: table, STable @@ -317,9 +295,9 @@ SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | st **Description**: The concatenation result of two or more strings with separator, the number of strings to be concatenated is at least 3 and at most 9 -**Return value type**: If all input strings are BINARY type, the result is BINARY type too. If any one of input strings is NCHAR type, then the result is NCHAR. +**Return value type**: If all input strings are VARCHAR type, the result is VARCHAR type too. If any one of input strings is NCHAR type, then the result is NCHAR. -**Applicable data types**: BINARY, NCHAR. Can't be used on tag columns. At least 3 input strings are requird, and at most 9 input strings are allowed. +**Applicable data types**: VARCHAR, NCHAR. Can't be used on tag columns. At least 3 input strings are requird, and at most 9 input strings are allowed. **Applicable table types**: table, STable @@ -339,8 +317,7 @@ SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] **Return value type**: Integer -**Applicable data types**: BINARY or NCHAR, can't be used on tags - +**Applicable data types**: VARCHAR or NCHAR **Applicable table types**: table, STable **Applicable nested query**: Inner query and Outer query @@ -359,7 +336,7 @@ SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause] **Return value type**: Same as input -**Applicable data types**: BINARY or NCHAR, can't be used on tags +**Applicable data types**: VARCHAR or NCHAR **Applicable table types**: table, STable @@ -379,7 +356,7 @@ SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] **Return value type**: Same as input -**Applicable data types**: BINARY or NCHAR, can't be used on tags +**Applicable data types**: VARCHAR or NCHAR **Applicable table types**: table, STable @@ -399,7 +376,7 @@ SELECT RTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] **Return value type**: Same as input -**Applicable data types**: BINARY or NCHAR, can't be used on tags +**Applicable data types**: VARCHAR or NCHAR **Applicable table types**: table, STable @@ -419,7 +396,7 @@ SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause] **Return value type**: Same as input -**Applicable data types**: BINARY or NCHAR, can't be used on tags +**Applicable data types**: VARCHAR or NCHAR **Applicable table types**: table, STable @@ -441,7 +418,7 @@ SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause] **Return value type**: Same as input -**Applicable data types**: BINARY or NCHAR, can't be used on tags +**Applicable data types**: VARCHAR or NCHAR **Applicable table types**: table, STable @@ -461,21 +438,21 @@ This kind of functions convert from one data type to another one. SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] ``` -**Description**: It's used for type casting. The input parameter `expression` can be data columns, constants, scalar functions or arithmetic between them. Can't be used with tags, and can only be used in `select` clause. +**Description**: It's used for type casting. The input parameter `expression` can be data columns, constants, scalar functions or arithmetic between them. **Return value type**: The type specified by parameter `type_name` **Applicable data types**: -- Parameter `expression` can be any data type except for JSON, more specifically it can be any of BOOL/TINYINT/SMALLINT/INT/BIGINT/FLOAT/DOUBLE/BINARY(M)/TIMESTAMP/NCHAR(M)/TINYINT UNSIGNED/SMALLINT UNSIGNED/INT UNSIGNED/BIGINT UNSIGNED -- The output data type specified by `type_name` can only be one of BIGINT/BINARY(N)/TIMESTAMP/NCHAR(N)/BIGINT UNSIGNED +- Parameter `expression` can be any data type except for JSON +- The output data type specified by `type_name` can only be one of BIGINT/VARCHAR(N)/TIMESTAMP/NCHAR(N)/BIGINT UNSIGNED **More explanations**: - Error will be reported for unsupported type casting - NULL will be returned if the input value is NULL - Some values of some supported data types may not be casted, below are known issues: - 1）When casting BINARY/NCHAR to BIGINT/BIGINT UNSIGNED, some characters may be treated as illegal, for example "a" may be converted to 0. + 1）When casting VARCHAR/NCHAR to BIGINT/BIGINT UNSIGNED, some characters may be treated as illegal, for example "a" may be converted to 0. 2）There may be overflow when casting singed integer or TIMESTAMP to unsigned BIGINT 3）There may be overflow when casting unsigned BIGINT to BIGINT 4）There may be overflow when casting FLOAT/DOUBLE to BIGINT or UNSIGNED BIGINT @@ -488,7 +465,7 @@ SELECT TO_ISO8601(ts_val | ts_col) FROM { tb_name | stb_name } [WHERE clause]; **Description**: The ISO8601 date/time format converted from a UNIX timestamp, plus the timezone of the client side system -**Return value type**: BINARY +**Return value type**: VARCHAR **Applicable column types**: TIMESTAMP, constant or a column @@ -525,7 +502,7 @@ SELECT TO_UNIXTIMESTAMP(datetime_string | ts_col) FROM { tb_name | stb_name } [W **Return value type**: Long integer -**Applicable column types**: Constant or column of BINARY/NCHAR +**Applicable column types**: Constant or column of VARCHAR/NCHAR **Applicable table types**: table, STable @@ -608,7 +585,7 @@ SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause]; **Description**: The timezone of the client side system -**Return value type**: BINARY +**Return value type**: VARCHAR **Applicable column types**: None @@ -650,7 +627,7 @@ SELECT AVG(field_name) FROM tb_name [WHERE clause]; **Return value type**: Double precision floating number -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric type **Applicable table types**: table, STable @@ -708,7 +685,7 @@ SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause] **Return value type**: A string in the format of "(slope, intercept)" -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table only @@ -718,7 +695,7 @@ SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause] SELECT MODE(field_name) FROM tb_name [WHERE clause]; ``` -**Description**:The value which has the highest frequency of occurrence. NULL is returned if there are multiple values which have highest frequency of occurrence. It can't be used on timestamp column or tags. +**Description**:The value which has the highest frequency of occurrence. NULL is returned if there are multiple values which have highest frequency of occurrence. It can't be used on timestamp column. **Return value type**:Same as the data type of the column being operated upon @@ -736,7 +713,7 @@ SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; **Return value type**: Double precision floating point -**Applicable column types**: Data types except for binary, nchar, and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -752,7 +729,7 @@ SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; **Return value type**: Double precision floating number -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -766,7 +743,7 @@ SELECT SUM(field_name) FROM tb_name [WHERE clause]; **Return value type**: Double precision floating number or long integer -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -836,7 +813,7 @@ FROM { tb_name | stb_name } [WHERE clause] **Return value type**: Double precision floating point -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -858,7 +835,7 @@ SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; **Return value type**: Same as the column being operated upon -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -961,7 +938,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; **Return value type**: Same as the data type of the column being operated upon -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -975,7 +952,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; **Return value type**: Same as the data type of the column being operated upon -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -989,7 +966,7 @@ SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; **Return value type**: Double precision floating point -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table @@ -1019,7 +996,7 @@ SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; **Return value type**: Same as the column being operated upon -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -1060,15 +1037,13 @@ TDengine provides a set of time-series specific functions to better meet the req **Return value type**: Long integer for integers; Double for floating points. Timestamp is returned for each row. -**Applicable data types**: Data types except for timestamp, binary, nchar, and bool +**Applicable data types**: Numeric types **Applicable table types**: table, STable **Applicable nested query**: Inner query and Outer query **More explanations**: - -- Can't be used on tags when it's used on STable - Arithmetic operation can't be performed on the result of `csum` function - Can only be used with aggregate functions - `Group by tbname` must be used together on a STable to force the result on a single timeline @@ -1083,7 +1058,7 @@ SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHER **Return value type**: Double precision floating point -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -1102,7 +1077,7 @@ SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHER **Return value type**: Same as the column being operated upon -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -1121,7 +1096,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause; **Return value type**: Double precision floating number -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable @@ -1139,7 +1114,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause; **Return value type**: Double precision floating point -**Applicable data types**: Data types except for timestamp, binary, nchar, and bool +**Applicable data types**: Numeric types **Applicable nested query**: Inner query and Outer query @@ -1148,7 +1123,6 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause; **More explanations**: - Arithmetic operation can't be performed on the result of `MAVG`. -- Can only be used with data columns, can't be used with tags. - Can't be used with aggregate functions. - Must be used with `GROUP BY tbname` when it's used on a STable to force the result on each single timeline. @@ -1188,7 +1162,7 @@ SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clau **Return value type**: Integer -**Applicable data types**: Data types excpet for timestamp, binary, nchar, bool +**Applicable data types**: Numeric types **Applicable table types**: table, STable @@ -1215,7 +1189,7 @@ SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [W **Return value type**: Integer -**Applicable data types**: Data types excpet for timestamp, binary, nchar, bool +**Applicable data types**: Numeric types **Applicable table types**: table, STable @@ -1236,7 +1210,7 @@ SELECT TWA(field_name) FROM tb_name WHERE clause; **Return value type**: Double precision floating number -**Applicable column types**: Data types except for timestamp, binary, nchar and bool +**Applicable column types**: Numeric types **Applicable table types**: table, STable From 15b25eeec5f1211e53faf3d9393d303e2abbcde0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 1 Jun 2022 15:06:12 +0800 Subject: [PATCH 56/57] feat: tsma/rsma refactor --- include/common/tmsg.h | 1 + include/util/taoserror.h | 1 + source/common/src/tmsg.c | 2 ++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 23 +++++++++++-- source/dnode/vnode/inc/vnode.h | 2 ++ source/dnode/vnode/src/inc/vnodeInt.h | 7 ++-- source/dnode/vnode/src/sma/smaOpen.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/dnode/vnode/src/vnd/vnodeCfg.c | 3 ++ source/dnode/vnode/src/vnd/vnodeCommit.c | 3 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 38 ++++++++++++++------- source/util/src/terror.c | 1 + 13 files changed, 63 insertions(+), 24 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e926e2fb31..1eae1835b4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2296,6 +2296,7 @@ typedef struct { int8_t intervalUnit; // MACRO: TIME_UNIT_XXX int8_t slidingUnit; // MACRO: TIME_UNIT_XXX int8_t timezoneInt; // sma data expired if timezone changes. + int32_t dstVgId; char indexName[TSDB_INDEX_NAME_LEN]; int32_t exprLen; int32_t tagsFilterLen; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 65cfe8de0b..412192e8a8 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -70,6 +70,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_NEED_RETRY TAOS_DEF_ERROR_CODE(0, 0x0028) #define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE TAOS_DEF_ERROR_CODE(0, 0x0029) #define TSDB_CODE_INVALID_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x0030) +#define TSDB_CODE_MSG_DECODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0031) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0040) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0041) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 233623c616..2fc60320dd 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3654,6 +3654,7 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (tEncodeI8(pCoder, pSma->intervalUnit) < 0) return -1; if (tEncodeI8(pCoder, pSma->slidingUnit) < 0) return -1; if (tEncodeI8(pCoder, pSma->timezoneInt) < 0) return -1; + if (tEncodeI32(pCoder, pSma->dstVgId) < 0) return -1; if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1; if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1; if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1; @@ -3676,6 +3677,7 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { if (tDecodeI8(pCoder, &pSma->version) < 0) return -1; if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1; if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1; + if (tDecodeI32(pCoder, &pSma->dstVgId) < 0) return -1; if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1; if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1; if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 27ea19a5dc..dda59b2396 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -150,8 +150,13 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->tsdbCfg.minRows = pCreate->minRows; pCfg->tsdbCfg.maxRows = pCreate->maxRows; for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) { - memcpy(&pCfg->tsdbCfg.retentions[i], taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention)); + SRetention *pRetention = &pCfg->tsdbCfg.retentions[i]; + memcpy(pRetention, taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention)); + if (i == 0) { + if ((pRetention->freq > 0 && pRetention->keep > 0)) pCfg->isRsma = 1; + } } + pCfg->walCfg.vgId = pCreate->vgId; pCfg->hashBegin = pCreate->hashBegin; pCfg->hashEnd = pCreate->hashEnd; @@ -218,9 +223,20 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl); if (code != 0) { dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); + code = terrno; goto _OVER; } + if (createReq.isTsma) { + SMsgHead *smaMsg = createReq.pTsma; + uint32_t contLen = (uint32_t)(htonl(smaMsg->contLen) - sizeof(SMsgHead)); + if (vnodeProcessCreateTSma(pImpl, POINTER_SHIFT(smaMsg, sizeof(SMsgHead)), contLen) < 0) { + dError("vgId:%d, failed to create tsma since %s", createReq.vgId, terrstr()); + code = terrno; + goto _OVER; + }; + } + code = vnodeStart(pImpl); if (code != 0) { dError("vgId:%d, failed to start sync since %s", createReq.vgId, terrstr()); @@ -228,7 +244,10 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } code = vmWriteVnodeListToFile(pMgmt); - if (code != 0) goto _OVER; + if (code != 0) { + code = terrno; + goto _OVER; + } _OVER: if (code != 0) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6be770cfa7..a274de24af 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -68,6 +68,7 @@ void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever); int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader); int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData); +int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); // meta typedef struct SMeta SMeta; // todo: remove @@ -172,6 +173,7 @@ struct SVnodeCfg { bool isHeap; bool isWeak; int8_t isTsma; + int8_t isRsma; int8_t hashMethod; STsdbCfg tsdbCfg; SWalCfg walCfg; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index e3a0c94ccc..6c066acff2 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -239,6 +239,8 @@ struct SVnode { #define VND_RSMA1(vnd) ((vnd)->pSma->pRSmaTsdb1) #define VND_RSMA2(vnd) ((vnd)->pSma->pRSmaTsdb2) #define VND_RETENTIONS(vnd) (&(vnd)->config.tsdbCfg.retentions) +#define VND_IS_RSMA(v) ((v)->config.isRsma == 1) +#define VND_IS_TSMA(v) ((v)->config.isTsma == 1) struct STbUidStore { tb_uid_t suid; @@ -271,11 +273,6 @@ struct SSma { #define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb1) #define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb2) -static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) { - SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]); - return (pRetention->freq > 0 && pRetention->keep > 0); -} - // sma void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 2a74fe78cb..dde6578054 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -104,7 +104,7 @@ int32_t smaOpen(SVnode *pVnode) { taosThreadMutexInit(&pSma->mutex, NULL); pSma->locked = false; - if (vnodeIsRollup(pVnode)) { + if (VND_IS_RSMA(pVnode)) { STsdbKeepCfg keepCfg = {0}; for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { if (i == TSDB_RETENTION_L0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 62125b6dc7..f9c5fac536 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -333,7 +333,7 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData } static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) { - if (vnodeIsRollup(pVnode)) { + if (VND_IS_RSMA(pVnode)) { int level = 0; int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 56cc3d6374..e8fa2ed3c1 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -57,6 +57,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; @@ -133,6 +134,8 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if(code < 0) return -1; tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code); if(code < 0) return -1; + tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code); + if(code < 0) return -1; tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code); if(code < 0) return -1; tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code); diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index b4fbd01c63..a0db3cfe2d 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -230,7 +230,7 @@ int vnodeCommit(SVnode *pVnode) { return -1; } - if(vnodeIsRollup(pVnode)) { + if (VND_IS_RSMA(pVnode)) { if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { ASSERT(0); return -1; @@ -250,7 +250,6 @@ int vnodeCommit(SVnode *pVnode) { } } - if (tqCommit(pVnode->pTq) < 0) { ASSERT(0); return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 178ef28e5d..43b4d6c77d 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -97,7 +97,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { } // open tsdb - if (!vnodeIsRollup(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL) < 0) { + if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL) < 0) { vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4b237bc703..149c45ba6f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -776,8 +776,7 @@ _exit: // TODO: the partial success scenario and the error case // TODO: refactor - if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) && - (pRsp->code == TSDB_CODE_SUCCESS)) { + if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) { tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); } @@ -788,16 +787,19 @@ static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq SVCreateTSmaReq req = {0}; SDecoder coder; - pRsp->msgType = TDMT_VND_CREATE_SMA_RSP; - pRsp->code = TSDB_CODE_SUCCESS; - pRsp->pCont = NULL; - pRsp->contLen = 0; + if (pRsp) { + pRsp->msgType = TDMT_VND_CREATE_SMA_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + } // decode and process req tDecoderInit(&coder, pReq, len); if (tDecodeSVCreateTSmaReq(&coder, &req) < 0) { - pRsp->code = terrno; + terrno = TSDB_CODE_MSG_DECODE_ERROR; + if (pRsp) pRsp->code = terrno; goto _err; } @@ -805,18 +807,30 @@ static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq req.timezoneInt = tsTimezone; if (tdProcessTSmaCreate(pVnode->pSma, version, (const char *)&req) < 0) { - pRsp->code = terrno; + if (pRsp) pRsp->code = terrno; goto _err; } tDecoderClear(&coder); - vDebug("vgId:%d success to create tsma %s:%" PRIi64 " for table %" PRIi64, TD_VID(pVnode), req.indexName, - req.indexUid, req.tableUid); + vDebug("vgId:%d success to create tsma %s:%" PRIi64 " version %" PRIi64 " for table %" PRIi64, TD_VID(pVnode), + req.indexName, req.indexUid, version, req.tableUid); return 0; _err: tDecoderClear(&coder); - vError("vgId:%d failed to create tsma %s:%" PRIi64 " for table %" PRIi64 " since %s", TD_VID(pVnode), req.indexName, - req.indexUid, req.tableUid, terrstr(terrno)); + vError("vgId:%d failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s", + TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr(terrno)); return -1; } + +/** + * @brief specific for smaDstVnode + * + * @param pVnode + * @param pCont + * @param contLen + * @return int32_t + */ +int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) { + return vnodeProcessCreateTSmaReq(pVnode, 1, pCont, contLen, NULL); +} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 575ae10bbb..a57f942f74 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -75,6 +75,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DUP_KEY, "Cannot add duplicate TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY, "Retry needed") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE, "Out of memory in rpc queue") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP, "Invalid timestamp format") +TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") From d948a43f270b3eca875d0e0b93e536becafc8dd7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 1 Jun 2022 15:19:52 +0800 Subject: [PATCH 57/57] chore: prepare merge example and examples for 3.0 (#13370) --- CMakeLists.txt | 2 +- examples/c/CMakeLists.txt | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8fe6cc69e0..b78b896908 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ endif(${BUILD_TEST}) add_subdirectory(source) add_subdirectory(tools) add_subdirectory(tests) -add_subdirectory(example) +add_subdirectory(examples/c) # docs add_subdirectory(docs) diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index 17a9257c49..8bc86abbb1 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -3,20 +3,20 @@ PROJECT(TDengine) IF (TD_LINUX) INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) AUX_SOURCE_DIRECTORY(. SRC) - ADD_EXECUTABLE(demo apitest.c) - TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) - ADD_EXECUTABLE(sml schemaless.c) - TARGET_LINK_LIBRARIES(sml taos_static trpc tutil pthread ) - ADD_EXECUTABLE(subscribe subscribe.c) - TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread ) - ADD_EXECUTABLE(epoll epoll.c) - TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) + # ADD_EXECUTABLE(demo apitest.c) + #TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) + #ADD_EXECUTABLE(sml schemaless.c) + #TARGET_LINK_LIBRARIES(sml taos_static trpc tutil pthread ) + #ADD_EXECUTABLE(subscribe subscribe.c) + #TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread ) + #ADD_EXECUTABLE(epoll epoll.c) + #TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) ENDIF () IF (TD_DARWIN) INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) AUX_SOURCE_DIRECTORY(. SRC) - ADD_EXECUTABLE(demo demo.c) - TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua) - ADD_EXECUTABLE(epoll epoll.c) - TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) + #ADD_EXECUTABLE(demo demo.c) + #TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua) + #ADD_EXECUTABLE(epoll epoll.c) + #TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) ENDIF ()