diff --git a/docs/en/14-reference/03-connector/java.mdx b/docs/en/14-reference/03-connector/java.mdx index 310e0a15c6..22f99bb9ae 100644 --- a/docs/en/14-reference/03-connector/java.mdx +++ b/docs/en/14-reference/03-connector/java.mdx @@ -91,7 +91,7 @@ Add following dependency in the `pom.xml` file of your Maven project: You can build Java connector from source code after cloning the TDengine project: ``` -git clone https://github.com/taosdata/taos-connector-jdbc.git +git clone https://github.com/taosdata/taos-connector-jdbc.git --branch 2.0 cd taos-connector-jdbc mvn clean install -Dmaven.test.skip=true ``` @@ -140,34 +140,34 @@ When you use a JDBC native connection to connect to a TDengine cluster, you can 1. Do not specify hostname and port in Java applications. - ```java - public Connection getConn() throws Exception{ - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; - Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); - return conn; - } - ``` + ```java + public Connection getConn() throws Exception{ + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + return conn; + } + ``` 2. specify the firstEp and the secondEp in the configuration file taos.cfg - ```shell - # first fully qualified domain name (FQDN) for TDengine system - firstEp cluster_node1:6030 + ```shell + # first fully qualified domain name (FQDN) for TDengine system + firstEp cluster_node1:6030 - # second fully qualified domain name (FQDN) for TDengine system, for cluster only - secondEp cluster_node2:6030 + # second fully qualified domain name (FQDN) for TDengine system, for cluster only + secondEp cluster_node2:6030 - # default system charset - # charset UTF-8 + # default system charset + # charset UTF-8 - # system locale - # locale en_US.UTF-8 - ``` + # system locale + # locale en_US.UTF-8 + ``` In the above example, JDBC uses the client's configuration file to establish a connection to a hostname `cluster_node1`, port 6030, and a database named `test`. When the firstEp node in the cluster fails, JDBC attempts to connect to the cluster using secondEp. diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index 838fa2eff8..88a3674671 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -93,7 +93,7 @@ Maven 项目中，在 pom.xml 中添加以下依赖： 可以通过下载 TDengine 的源码，自己编译最新版本的 Java connector ```shell -git clone https://github.com/taosdata/taos-connector-jdbc.git +git clone https://github.com/taosdata/taos-connector-jdbc.git --branch 2.0 cd taos-connector-jdbc mvn clean install -Dmaven.test.skip=true ``` diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 516df71b0b..9bfee56e29 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -34,11 +34,10 @@ typedef enum { TSDB_SUPER_TABLE = 1, // super table TSDB_CHILD_TABLE = 2, // table created from super table TSDB_NORMAL_TABLE = 3, // ordinary table - TSDB_STREAM_TABLE = 4, // table created from stream computing - TSDB_TEMP_TABLE = 5, // temp table created by nest query - TSDB_SYSTEM_TABLE = 6, - TSDB_TSMA_TABLE = 7, // time-range-wise sma - TSDB_TABLE_MAX = 8 + TSDB_TEMP_TABLE = 4, // temp table created by nest query + TSDB_SYSTEM_TABLE = 5, + TSDB_TSMA_TABLE = 6, // time-range-wise sma + TSDB_TABLE_MAX = 7 } ETableType; typedef enum { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index a4f5904018..22aac46560 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -184,8 +184,8 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, - const SColumnInfoData* pSource, uint32_t numOfRow2); +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, + const SColumnInfoData* pSource, int32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 77b6ca1833..45be3a8507 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2537,6 +2537,15 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) return (void*)buf; } +typedef struct { + char topic[TSDB_TOPIC_FNAME_LEN]; + int64_t ntbUid; + SArray* colIdList; // SArray +} SCheckAlterInfo; + +int32_t tEncodeSCheckAlterInfo(SEncoder* pEncoder, const SCheckAlterInfo* pInfo); +int32_t tDecodeSCheckAlterInfo(SDecoder* pDecoder, SCheckAlterInfo* pInfo); + typedef struct { int32_t vgId; int64_t offset; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a9d8871c59..56e0935ce1 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -186,6 +186,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_COMMIT_OFFSET, "vnode-commit-offset", STqOffset, STqOffset) + TD_DEF_MSG_TYPE(TDMT_VND_CHECK_ALTER_INFO, "vnode-alter-check-info", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TOPIC, "vnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 65244ec11a..65e20336cc 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -64,7 +64,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); * @param SReadHandle * @return */ -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchemaWrapper); +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, + SSchemaWrapper** pSchema); /** * Set the input data block for the stream scan. @@ -195,6 +196,8 @@ int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem); int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer); +STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index a11ea0b2cd..e382fa4efd 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -111,6 +111,7 @@ typedef struct SAggLogicNode { SNodeList* pGroupKeys; SNodeList* pAggFuncs; bool hasLastRow; + bool hasTimeLineFunc; } SAggLogicNode; typedef struct SProjectLogicNode { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 89b0e6f952..431abe034f 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -332,6 +332,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_INVALID_TABLE_ACTION TAOS_DEF_ERROR_CODE(0, 0x0519) #define TSDB_CODE_VND_COL_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x051a) #define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x051b) +#define TSDB_CODE_VND_COL_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x051c) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 03d2b48134..ab9d970dd6 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(clientTest clientTests.cpp) TARGET_LINK_LIBRARIES( clientTest - PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom executor ) ADD_EXECUTABLE(tmqTest tmqTest.cpp) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 2caf81317d..e7ae3917f9 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -27,6 +27,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "taos.h" +#include "executor.h" namespace { void showDB(TAOS* pConn) { @@ -823,6 +824,17 @@ TEST(testCase, async_api_test) { TEST(testCase, update_test) { + + SInterval interval = {0}; + interval.offset = 8000; + interval.interval = 10000; + interval.sliding = 4000; + interval.intervalUnit = 's'; + interval.offsetUnit = 's'; + interval.slidingUnit = 's'; +// STimeWindow w = getAlignQueryTimeWindow(&interval, 0, 1630000000000); + STimeWindow w = getAlignQueryTimeWindow(&interval, 0, 1629999999999); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f7b0da0014..f64a0df36f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -214,8 +214,8 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, - const SColumnInfoData* pSource, uint32_t numOfRow2) { +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, + const SColumnInfoData* pSource, int32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRow2 == 0) { return numOfRow1; @@ -263,7 +263,8 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui pColumnInfoData->varmeta.length = len + oldLen; } else { if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { - ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); + // all data may be null, when the pColumnInfoData->info.type == 0, bytes == 0; +// ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 76ec5b2d22..a73808f2ed 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -183,7 +183,7 @@ bool tsStartUdfd = true; int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; int32_t tsTtlUnit = 86400; -int32_t tsTtlPushInterval = 60; +int32_t tsTtlPushInterval = 86400; int32_t tsGrantHBInterval = 60; void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { @@ -226,8 +226,17 @@ static int32_t taosSetTfsCfg(SConfig *pCfg) { } if (tsDataDir[0] == 0) { - uError("datadir not set"); - return -1; + if (pItem->str != NULL) { + taosAddDataDir(0, pItem->str, 0, 1); + tstrncpy(tsDataDir, pItem->str, PATH_MAX); + if (taosMulMkDir(tsDataDir) != 0) { + uError("failed to create dataDir:%s since %s", tsDataDir, terrstr()); + return -1; + } + } else { + uError("datadir not set"); + return -1; + } } return 0; @@ -466,7 +475,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 10000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, 1) != 0) return -1; if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; return 0; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3e733d291b..2a3452c534 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5623,6 +5623,33 @@ int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) { return 0; } +int32_t tEncodeSCheckAlterInfo(SEncoder *pEncoder, const SCheckAlterInfo *pInfo) { + if (tEncodeCStr(pEncoder, pInfo->topic) < 0) return -1; + if (tEncodeI64(pEncoder, pInfo->ntbUid) < 0) return -1; + int32_t sz = taosArrayGetSize(pInfo->colIdList); + if (tEncodeI32(pEncoder, sz) < 0) return -1; + for (int32_t i = 0; i < sz; i++) { + int16_t colId = *(int16_t *)taosArrayGet(pInfo->colIdList, i); + if (tEncodeI16(pEncoder, colId) < 0) return -1; + } + return pEncoder->pos; +} + +int32_t tDecodeSCheckAlterInfo(SDecoder *pDecoder, SCheckAlterInfo *pInfo) { + if (tDecodeCStrTo(pDecoder, pInfo->topic) < 0) return -1; + if (tDecodeI64(pDecoder, &pInfo->ntbUid) < 0) return -1; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + pInfo->colIdList = taosArrayInit(sz, sizeof(int16_t)); + if (pInfo->colIdList == NULL) return -1; + for (int32_t i = 0; i < sz; i++) { + int16_t colId; + if (tDecodeI16(pDecoder, &colId) < 0) return -1; + taosArrayPush(pInfo->colIdList, &colId); + } + return 0; +} + int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) { int32_t nUid = taosArrayGetSize(pRes->uidList); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index b1e4321053..77b45b6df1 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -815,20 +815,17 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio if (pInterval->offset > 0) { start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision); - if (start > t) { - start = taosTimeAdd(start, -pInterval->interval, pInterval->intervalUnit, precision); - } else { - // try to move current window to the left-hande-side, due to the offset effect. - int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - int64_t newEnd = end; - while(newEnd >= t) { - end = newEnd; - newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); - } + // try to move current window to the left-hande-side, due to the offset effect. + int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; + int64_t newEnd = end; + while (newEnd >= t) { + end = newEnd; + newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); } + + start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; } return start; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 8db43a460e..22a53f07f6 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -224,6 +224,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CHECK_ALTER_INFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 9571a83116..0471e2b850 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -352,6 +352,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CHECK_ALTER_INFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 977f33de49..f6c92c3929 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -144,33 +144,33 @@ typedef enum { } ECsmUpdateType; typedef struct { - int32_t id; - ETrnStage stage; - ETrnPolicy policy; - ETrnConflct conflict; - ETrnExec exec; - EOperType oper; - int32_t code; - int32_t failedTimes; - void* rpcRsp; - int32_t rpcRspLen; - int32_t redoActionPos; - SArray* redoActions; - SArray* undoActions; - SArray* commitActions; - int64_t createdTime; - int64_t lastExecTime; - int32_t lastAction; - int32_t lastErrorNo; - tmsg_t lastMsgType; - SEpSet lastEpset; - char dbname1[TSDB_DB_FNAME_LEN]; - char dbname2[TSDB_DB_FNAME_LEN]; - int32_t startFunc; - int32_t stopFunc; - int32_t paramLen; - void* param; - SArray* pRpcArray; + int32_t id; + ETrnStage stage; + ETrnPolicy policy; + ETrnConflct conflict; + ETrnExec exec; + EOperType oper; + int32_t code; + int32_t failedTimes; + void* rpcRsp; + int32_t rpcRspLen; + int32_t redoActionPos; + SArray* redoActions; + SArray* undoActions; + SArray* commitActions; + int64_t createdTime; + int64_t lastExecTime; + int32_t lastAction; + int32_t lastErrorNo; + tmsg_t lastMsgType; + SEpSet lastEpset; + char dbname1[TSDB_DB_FNAME_LEN]; + char dbname2[TSDB_DB_FNAME_LEN]; + int32_t startFunc; + int32_t stopFunc; + int32_t paramLen; + void* param; + SArray* pRpcArray; } STrans; typedef struct { @@ -477,6 +477,10 @@ typedef struct { char* physicalPlan; SSchemaWrapper schema; int64_t stbUid; + // forbid condition + int64_t ntbUid; + SArray* ntbColIds; + int64_t ctbStbUid; } SMqTopicObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index 4becad6da2..7eb53dbdbb 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -37,7 +37,7 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); -int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId); +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char* stbname, int64_t suid, col_id_t colId); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 1306a0252f..682e78acc0 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1144,7 +1144,7 @@ static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj } col_id_t colId = pOld->pTags[tag].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { return -1; } @@ -1179,7 +1179,7 @@ static int32_t mndAlterStbTagName(SMnode *pMnode, const SStbObj *pOld, SStbObj * } col_id_t colId = pOld->pTags[tag].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { return -1; } @@ -1213,7 +1213,7 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj } col_id_t colId = pOld->pTags[tag].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { return -1; } @@ -1295,7 +1295,7 @@ static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStb } col_id_t colId = pOld->pColumns[col].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { return -1; } @@ -1329,7 +1329,7 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO } col_id_t colId = pOld->pColumns[col].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 268cbaa55c..7acfc95bfc 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -57,6 +57,7 @@ int32_t mndInitTopic(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq); mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_CHECK_ALTER_INFO_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic); @@ -71,14 +72,16 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) { return strchr(topic, '.') + 1; } -int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId) { +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char* stbname, int64_t suid, col_id_t colId) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; - bool found = false; while (1) { SMqTopicObj *pTopic = NULL; pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); if (pIter == NULL) break; + + mDebug("topic:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, subType:%d sql:%s", + pTopic->name, stbname, suid, colId, pTopic->subType, pTopic->sql); if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { sdbRelease(pSdb, pTopic); continue; @@ -95,21 +98,25 @@ int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId SNode *pNode = NULL; FOREACH(pNode, pNodeList) { SColumnNode *pCol = (SColumnNode *)pNode; - if (pCol->tableId != suid) goto NEXT; - if (pCol->colId > 0 && pCol->colId == colId) { - found = true; + mDebug("topic:%s, check colId:%d tableId:%" PRId64 " ctbStbUid:%" PRId64, pTopic->name, pCol->colId, pCol->tableId, pTopic->ctbStbUid); + + if (pCol->tableId != suid && pTopic->ctbStbUid != suid) { + mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); goto NEXT; } - mTrace("topic:%s, colId:%d is used", pTopic->name, pCol->colId); + if (pCol->colId > 0 && pCol->colId == colId) { + sdbRelease(pSdb, pTopic); + nodesDestroyNode(pAst); + terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; + mError("topic:%s, check colId:%d conflicted", pTopic->name, pCol->colId); + return -1; + } + mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); } NEXT: sdbRelease(pSdb, pTopic); nodesDestroyNode(pAst); - if (found) { - terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; - return -1; - } } return 0; @@ -127,8 +134,10 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { if (pTopic->schema.nCols) { schemaLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema); } - int32_t size = - sizeof(SMqTopicObj) + physicalPlanLen + pTopic->sqlLen + pTopic->astLen + schemaLen + MND_TOPIC_RESERVE_SIZE; + int32_t ntbColLen = taosArrayGetSize(pTopic->ntbColIds) * sizeof(int16_t); + + int32_t size = sizeof(SMqTopicObj) + physicalPlanLen + pTopic->sqlLen + pTopic->astLen + schemaLen + ntbColLen + + MND_TOPIC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); if (pRaw == NULL) goto TOPIC_ENCODE_OVER; @@ -164,6 +173,16 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema); SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER); } + SDB_SET_INT64(pRaw, dataPos, pTopic->ntbUid, TOPIC_ENCODE_OVER); + if (pTopic->ntbUid != 0) { + int32_t sz = taosArrayGetSize(pTopic->ntbColIds); + SDB_SET_INT32(pRaw, dataPos, sz, TOPIC_ENCODE_OVER); + for (int32_t i = 0; i < sz; i++) { + int16_t colId = *(int16_t *)taosArrayGet(pTopic->ntbColIds, i); + SDB_SET_INT16(pRaw, dataPos, colId, TOPIC_ENCODE_OVER); + } + } + SDB_SET_INT64(pRaw, dataPos, pTopic->ctbStbUid, TOPIC_ENCODE_OVER); SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER); @@ -259,6 +278,20 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { pTopic->schema.version = 0; pTopic->schema.pSchema = NULL; } + SDB_GET_INT64(pRaw, dataPos, &pTopic->ntbUid, TOPIC_DECODE_OVER); + if (pTopic->ntbUid != 0) { + int32_t ntbColNum; + SDB_GET_INT32(pRaw, dataPos, &ntbColNum, TOPIC_DECODE_OVER); + pTopic->ntbColIds = taosArrayInit(ntbColNum, sizeof(int16_t)); + if (pTopic->ntbColIds == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto TOPIC_DECODE_OVER; + } + int16_t colId; + SDB_GET_INT16(pRaw, dataPos, &colId, TOPIC_DECODE_OVER); + taosArrayPush(pTopic->ntbColIds, &colId); + } + SDB_GET_INT64(pRaw, dataPos, &pTopic->ctbStbUid, TOPIC_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER); @@ -346,6 +379,26 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { return 0; } +static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) { + SNodeList *pNodeList = NULL; + nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); + int64_t suid = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->suid; + int8_t tableType = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->tableType; + if (tableType == TSDB_CHILD_TABLE) { + pTopic->ctbStbUid = suid; + } else if (tableType == TSDB_NORMAL_TABLE) { + SNode *pNode = NULL; + FOREACH(pNode, pNodeList) { + SColumnNode *pCol = (SColumnNode *)pNode; + if (pCol->tableType == TSDB_NORMAL_TABLE) { + pTopic->ntbUid = pCol->tableId; + taosArrayPush(pTopic->ntbColIds, &pCol->colId); + } + } + } + return 0; +} + static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { mDebug("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; @@ -386,6 +439,19 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * return -1; } + int64_t ntbUid; + topicObj.ntbColIds = taosArrayInit(0, sizeof(int16_t)); + if (topicObj.ntbColIds == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + extractTopicTbInfo(pAst, &topicObj); + + if (topicObj.ntbUid == 0) { + taosArrayDestroy(topicObj.ntbColIds); + topicObj.ntbColIds = NULL; + } + if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); taosMemoryFree(topicObj.ast); @@ -433,6 +499,60 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + if (topicObj.ntbUid != 0) { + SCheckAlterInfo info; + memcpy(info.topic, topicObj.name, TSDB_TOPIC_FNAME_LEN); + info.ntbUid = topicObj.ntbUid; + info.colIdList = topicObj.ntbColIds; + // broadcast forbid alter info + void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + while (1) { + // iterate vg + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (!mndVgroupInDb(pVgroup, topicObj.dbUid)) { + sdbRelease(pSdb, pVgroup); + continue; + } + + // encoder check alter info + int32_t len; + int32_t code; + tEncodeSize(tEncodeSCheckAlterInfo, &info, len, code); + if (code < 0) { + sdbRelease(pSdb, pVgroup); + mndTransDrop(pTrans); + ASSERT(0); + return -1; + } + void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + if (tEncodeSCheckAlterInfo(&encoder, &info) < 0) { + sdbRelease(pSdb, pVgroup); + mndTransDrop(pTrans); + return -1; + } + tEncoderClear(&encoder); + ((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId); + // add redo action + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = buf; + action.contLen = sizeof(SMsgHead) + len; + action.msgType = TDMT_VND_CHECK_ALTER_INFO; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + sdbRelease(pSdb, pVgroup); + mndTransDrop(pTrans); + return -1; + } + } + } + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); taosMemoryFreeClear(topicObj.physicalPlan); @@ -442,7 +562,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * taosMemoryFreeClear(topicObj.physicalPlan); mndTransDrop(pTrans); - return 0; + return TSDB_CODE_ACTION_IN_PROGRESS; } static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f2b791def6..4dac6ccfd2 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -89,6 +89,7 @@ void metaReaderClear(SMetaReader *pReader); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *tagVal); +int metaGetTableNameByUid(void* meta, uint64_t uid, char* tbName); typedef struct SMetaFltParam { tb_uid_t suid; diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 4673b01d27..217d40e3aa 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -67,8 +67,6 @@ struct STSmaStat { struct SRSmaStat { SSma *pSma; int64_t commitAppliedVer; // vnode applied version for async commit - int64_t commitSubmitVer; // rsma submit version for async commit - int64_t submitVer; // latest submit version int64_t refId; // shared by fetch tasks int8_t triggerStat; // shared by fetch tasks int8_t commitStat; // 0 not in committing, 1 in committing @@ -91,7 +89,6 @@ struct SSmaStat { #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) #define RSMA_COMMIT_STAT(r) (&(r)->commitStat) #define RSMA_REF_ID(r) ((r)->refId) -#define RSMA_SUBMIT_VER(r) ((r)->submitVer) struct SRSmaInfoItem { void *taskInfo; // qTaskInfo_t @@ -223,13 +220,6 @@ struct STFInfo { uint32_t ftype; uint32_t fver; int64_t fsize; - - // specific fields - union { - struct { - int64_t submitVer; - } qTaskInfo; - }; }; enum { diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 93c70e1109..b063e552f6 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -89,7 +89,7 @@ typedef struct { STqExecDb execDb; }; int32_t numOfCols; // number of out pout column, temporarily used - SSchemaWrapper *pSchemaWrapper; // columns that are involved in query + SSchemaWrapper* pSchemaWrapper; // columns that are involved in query } STqExecHandle; typedef struct { @@ -110,9 +110,6 @@ typedef struct { // exec STqExecHandle execHandle; - // prevent drop - int64_t ntbUid; - SArray* colIdList; // SArray } STqHandle; struct STQ { @@ -120,9 +117,9 @@ struct STQ { SHashObj* pushMgr; // consumerId -> STqHandle* SHashObj* handles; // subKey -> STqHandle SHashObj* pStreamTasks; // taksId -> SStreamTask + SHashObj* pAlterInfo; // topic -> SAlterCheckInfo STqOffsetStore* pOffsetStore; SVnode* pVnode; - SWal* pWal; TDB* pMetaStore; TTB* pExecStore; }; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 5e87e35d68..b7c23c8527 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -137,12 +137,13 @@ STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableL // tq int tqInit(); void tqCleanUp(); -STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); +STQ* tqOpen(const char* path, SVnode* pVnode); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); -int32_t tqCheckColModifiable(STQ* pTq, int32_t colId); +int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); +int32_t tqProcessCheckAlterInfoReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen); @@ -177,7 +178,6 @@ int32_t smaAsyncPostCommit(SSma* pSma); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); -int64_t tdRSmaGetMaxSubmitVer(SSma* pSma, int8_t level); int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq); int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 776a3c5b7f..da4399a609 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -98,6 +98,17 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { return uid; } +int metaGetTableNameByUid(void* meta, uint64_t uid, char* tbName) { + SMetaReader mr = {0}; + metaReaderInit(&mr, (SMeta*)meta, 0); + metaGetTableEntryByUid(&mr, uid); + + STR_TO_VARSTR(tbName, mr.me.name); + metaReaderClear(&mr); + + return 0; +} + int metaReadNext(SMetaReader *pReader) { SMeta *pMeta = pReader->pMeta; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 5255724b58..fc55c642e5 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -438,12 +438,15 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi } int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { - metaWLock(pMeta); int ret = metaTtlSmaller(pMeta, ttl, tbUids); if (ret != 0) { - metaULock(pMeta); return ret; } + if (taosArrayGetSize(tbUids) == 0){ + return 0; + } + + metaWLock(pMeta); for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i); metaDropTableByUid(pMeta, *uid, NULL); @@ -637,6 +640,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; goto _err; } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } pSchema->version++; tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); if (tlen) { @@ -653,6 +660,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; goto _err; } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } pSchema->version++; pColumn->bytes = pAlterTbReq->colModBytes; break; @@ -661,6 +672,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS; goto _err; } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } pSchema->version++; strcpy(pColumn->name, pAlterTbReq->colNewName); break; diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 2174a479e7..a0e3932245 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -146,7 +146,6 @@ static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) { // step 3: perform persist task for qTaskInfo pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; - pRSmaStat->commitSubmitVer = pRSmaStat->submitVer; tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma)); @@ -317,7 +316,6 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // step 4: others pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; - pRSmaStat->commitSubmitVer = pRSmaStat->submitVer; return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index e7293da60b..a6fde1e2d2 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -560,17 +560,6 @@ static void tdDestroySDataBlockArray(SArray *pArray) { taosArrayDestroy(pArray); } -int64_t tdRSmaGetMaxSubmitVer(SSma *pSma, int8_t level) { - if (level == TSDB_RETENTION_L0) { - return pSma->pVnode->state.applied; - } - - SSmaEnv *pRSmaEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pRSmaStat = (SRSmaStat *)(SMA_ENV_STAT(pRSmaEnv)); - - return atomic_load_64(&pRSmaStat->submitVer); -} - static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, int8_t blkType) { SArray *pResult = NULL; @@ -615,13 +604,16 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche goto _err; } - if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pStat->submitVer, 1), pReq) < 0) { + if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { taosMemoryFreeClear(pReq); smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr()); goto _err; } + smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " version:%"PRIi64, SMA_VID(pSma), + suid, pItem->level, output->info.version); + taosMemoryFreeClear(pReq); taosArrayClear(pResult); } else if (terrno == 0) { @@ -908,12 +900,8 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed) { goto _err; } - ASSERT(tFileInfo.qTaskInfo.submitVer > 0); - SSmaEnv *pRSmaEnv = pSma->pRSmaEnv; SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pRSmaEnv); - atomic_store_64(&pRSmaStat->submitVer, tFileInfo.qTaskInfo.submitVer); - smaDebug("%s:%d tFileInfo.qTaskInfo.submitVer = %" PRIi64, __func__, __LINE__, tFileInfo.qTaskInfo.submitVer); SRSmaQTaskInfoIter fIter = {0}; if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { @@ -1266,7 +1254,6 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { } if (isFileCreated) { - tFile.info.qTaskInfo.submitVer = atomic_load_64(&pRSmaStat->commitSubmitVer); if (tdUpdateTFileHeader(&tFile) < 0) { smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile), tstrerror(terrno)); @@ -1346,6 +1333,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); tdUnRefRSmaInfo(pSma, pRSmaInfo); + // atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); + // taosTmrReset(tdRSmaFetchTrigger, 5000, pItem, smaMgmt.tmrHandle, &pItem->tmrId); } break; case TASK_TRIGGER_STAT_PAUSED: { smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused", diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 13befabed5..9dce166afa 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -32,9 +32,6 @@ static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) { tlen += taosEncodeFixedU32(buf, pInfo->ftype); tlen += taosEncodeFixedU32(buf, pInfo->fver); tlen += taosEncodeFixedI64(buf, pInfo->fsize); - if (pInfo->ftype == TD_FTYPE_RSMA_QTASKINFO) { - tlen += taosEncodeFixedI64(buf, pInfo->qTaskInfo.submitVer); - } return tlen; } @@ -44,10 +41,6 @@ static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->ftype)); buf = taosDecodeFixedU32(buf, &(pInfo->fver)); buf = taosDecodeFixedI64(buf, &(pInfo->fsize)); - // specific - if (pInfo->ftype == TD_FTYPE_RSMA_QTASKINFO) { - buf = taosDecodeFixedI64(buf, &(pInfo->qTaskInfo.submitVer)); - } return buf; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3739897ec0..002b629660 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -51,7 +51,7 @@ void tqCleanUp() { } } -STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { +STQ* tqOpen(const char* path, SVnode* pVnode) { STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -59,7 +59,6 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { } pTq->path = strdup(path); pTq->pVnode = pVnode; - pTq->pWal = pWal; pTq->handles = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); @@ -67,6 +66,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + pTq->pAlterInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); + if (tqMetaOpen(pTq) < 0) { ASSERT(0); } @@ -91,6 +92,7 @@ void tqClose(STQ* pTq) { } taosHashCleanup(pTq->pStreamTasks); taosHashCleanup(pTq->pushMgr); + taosHashCleanup(pTq->pAlterInfo); taosMemoryFree(pTq->path); tqMetaClose(pTq); taosMemoryFree(pTq); @@ -208,18 +210,18 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { return 0; } -int32_t tqCheckColModifiable(STQ* pTq, int32_t colId) { +int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { void* pIter = NULL; while (1) { - pIter = taosHashIterate(pTq->handles, pIter); + pIter = taosHashIterate(pTq->pAlterInfo, pIter); if (pIter == NULL) break; - STqHandle* pExec = (STqHandle*)pIter; - if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - int32_t sz = pExec->execHandle.pSchemaWrapper->nCols; + SCheckAlterInfo* pCheck = (SCheckAlterInfo*)pIter; + if (pCheck->ntbUid == tbUid) { + int32_t sz = taosArrayGetSize(pCheck->colIdList); for (int32_t i = 0; i < sz; i++) { - SSchema* pSchema = &pExec->execHandle.pSchemaWrapper->pSchema[i]; - if (pSchema->colId == colId) { - taosHashCancelIterate(pTq->handles, pIter); + int16_t forbidColId = *(int16_t*)taosArrayGet(pCheck->colIdList, i); + if (forbidColId == colId) { + taosHashCancelIterate(pTq->pAlterInfo, pIter); return -1; } } @@ -270,6 +272,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t code = 0; STqOffsetVal reqOffset = pReq->reqOffset; STqOffsetVal fetchOffsetNew; + SWalCkHead* pCkHead = NULL; // 1.find handle STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey)); @@ -459,6 +462,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } OVER: + if (pCkHead) taosMemoryFree(pCkHead); // TODO wrap in destroy func taosArrayDestroy(dataRsp.blockDataLen); taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree); @@ -488,6 +492,22 @@ int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) { return 0; } +int32_t tqProcessCheckAlterInfoReq(STQ* pTq, char* msg, int32_t msgLen) { + SCheckAlterInfo info = {0}; + SDecoder decoder; + tDecoderInit(&decoder, msg, msgLen); + if (tDecodeSCheckAlterInfo(&decoder, &info) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + tDecoderClear(&decoder); + if (taosHashPut(pTq->pAlterInfo, info.topic, strlen(info.topic), &info, sizeof(SCheckAlterInfo)) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { SMqRebVgReq req = {0}; tDecodeSMqRebVgReq(msg, &req); @@ -549,8 +569,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { taosArrayDestroy(tbUidList); } taosHashPut(pTq->handles, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); + tqDebug("try to persist handle %s consumer %ld", req.subKey, pHandle->consumerId); if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { // TODO + ASSERT(0); } } else { /*ASSERT(pExec->consumerId == req.oldConsumerId);*/ diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 4e2750d9f0..d8851c3775 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -201,10 +201,12 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR if (pRsp->withTbName) { int64_t uid = pExec->pExecReader->msgIter.uid; if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) { + blockDataFreeRes(&block); continue; } } tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock)); + blockDataFreeRes(&block); tqAddBlockSchemaToRsp(pExec, pRsp); pRsp->blockNum++; } @@ -220,10 +222,12 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR if (pRsp->withTbName) { int64_t uid = pExec->pExecReader->msgIter.uid; if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) { + blockDataFreeRes(&block); continue; } } tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock)); + blockDataFreeRes(&block); tqAddBlockSchemaToRsp(pExec, pRsp); pRsp->blockNum++; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 835ffb02fd..d282036fe3 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -89,8 +89,8 @@ int32_t tqMetaOpen(STQ* pTq) { .version = handle.snapshotVer, }; - handle.execHandle.execCol.task = - qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, &handle.execHandle.pSchemaWrapper); + handle.execHandle.execCol.task = qCreateQueueExecTaskInfo( + handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, &handle.execHandle.pSchemaWrapper); ASSERT(handle.execHandle.execCol.task); void* scanner = NULL; qExtractStreamScanner(handle.execHandle.execCol.task, &scanner); @@ -101,6 +101,7 @@ int32_t tqMetaOpen(STQ* pTq) { handle.execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); } + tqDebug("tq restore %s consumer %ld", handle.subKey, handle.consumerId); taosHashPut(pTq->handles, pKey, kLen, &handle, sizeof(STqHandle)); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 17842615c4..821b48275a 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -340,29 +340,30 @@ FAIL: void tqReaderSetColIdList(STqReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } -int tqReaderSetTbUidList(STqReader* pHandle, const SArray* tbUidList) { - if (pHandle->tbIdHash) { - taosHashClear(pHandle->tbIdHash); +int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { + if (pReader->tbIdHash) { + taosHashClear(pReader->tbIdHash); + } else { + pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); } - pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (pHandle->tbIdHash == NULL) { + if (pReader->tbIdHash == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); - taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); + taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } return 0; } -int tqReaderAddTbUidList(STqReader* pHandle, const SArray* tbUidList) { - if (pHandle->tbIdHash == NULL) { - pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (pHandle->tbIdHash == NULL) { +int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) { + if (pReader->tbIdHash == NULL) { + pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pReader->tbIdHash == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -370,18 +371,18 @@ int tqReaderAddTbUidList(STqReader* pHandle, const SArray* tbUidList) { for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); - taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); + taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } return 0; } -int tqReaderRemoveTbUidList(STqReader* pHandle, const SArray* tbUidList) { - ASSERT(pHandle->tbIdHash != NULL); +int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { + ASSERT(pReader->tbIdHash != NULL); for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); - taosHashRemove(pHandle->tbIdHash, pKey, sizeof(int64_t)); + taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)); } return 0; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 464a3a3ee1..850394b15d 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -110,7 +110,6 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo // TODO ret = rpcMallocCont(cap); ret->header.vgId = vgId; - ret->version = htonl(1); ret->length = sizeof(SSubmitReq); ret->numOfBlocks = htonl(sz); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 26ced6cf6a..cec714e0ee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2265,10 +2265,6 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion; - if (VND_IS_RSMA(pVnode)) { - return (SVersionRange){.minVer = startVer, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)}; - } - int64_t endVer = 0; if (pCond->endVersion == -1) { // user not specified end version, set current maximum version of vnode as the endVersion diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 0c8a8a5fd1..1207df8058 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -95,16 +95,19 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // save info to a vnode_tmp.json pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { + vError("failed to open info file: %s for write: %s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } if (taosWriteFile(pFile, data, strlen(data)) < 0) { + vError("failed to write info file: %s data: %s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } if (taosFsyncFile(pFile) < 0) { + vError("failed to fsync info file: %s error: %s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index fb2ac722a6..e273a41f48 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -135,7 +135,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open tq sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); taosRealPath(tdir, NULL, sizeof(tdir)); - pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal); + pVnode->pTq = tqOpen(tdir, pVnode); if (pVnode->pTq == NULL) { vError("vgId:%d, failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e8d51555d8..5463f42f6a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -112,8 +112,8 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { tEncodeSize(tEncodeDeleteRes, &res, size, ret); pCont = rpcMallocCont(size + sizeof(SMsgHead)); - ((SMsgHead *)pCont)->contLen = htonl(size + sizeof(SMsgHead)); - ((SMsgHead *)pCont)->vgId = htonl(TD_VID(pVnode)); + ((SMsgHead *)pCont)->contLen = size + sizeof(SMsgHead); + ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size); tEncodeDeleteRes(pCoder, &res); @@ -206,6 +206,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp goto _err; } break; + case TDMT_VND_CHECK_ALTER_INFO: + if (tqProcessCheckAlterInfoReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), + pMsg->contLen - sizeof(SMsgHead)) < 0) { + goto _err; + } + break; case TDMT_STREAM_TASK_DEPLOY: { if (tqProcessTaskDeployReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 625541e00a..23732a6f9a 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -108,6 +108,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); +int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); +size_t getTableTagsBufLen(const SNodeList* pGroups); + SArray* createSortInfo(SNodeList* pNodeList); SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, @@ -129,6 +132,6 @@ int32_t convertFillType(int32_t mode); int32_t resultrowComparAsc(const void* p1, const void* p2); -int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 21068c68a4..1ad17bbc76 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -52,11 +52,12 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0) -#define START_TS_COLUMN_INDEX 0 -#define END_TS_COLUMN_INDEX 1 -#define UID_COLUMN_INDEX 2 -#define GROUPID_COLUMN_INDEX UID_COLUMN_INDEX -#define DELETE_GROUPID_COLUMN_INDEX 2 +#define START_TS_COLUMN_INDEX 0 +#define END_TS_COLUMN_INDEX 1 +#define UID_COLUMN_INDEX 2 +#define GROUPID_COLUMN_INDEX 3 +#define CALCULATE_START_TS_COLUMN_INDEX 4 +#define CALCULATE_END_TS_COLUMN_INDEX 5 enum { // when this task starts to execute, this status will set @@ -175,6 +176,7 @@ typedef struct SExecTaskInfo { int64_t owner; // if it is in execution int32_t code; + int64_t version; // used for stream to record wal version SStreamTaskInfo streamInfo; SSchemaInfo schemaInfo; STableListInfo tableqinfoList; // this is a table list @@ -346,7 +348,6 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_UPDATERES, - STREAM_SCAN_FROM_DATAREADER, // todo(liuyao) delete it STREAM_SCAN_FROM_DATAREADER_RETRIEVE, STREAM_SCAN_FROM_DATAREADER_RANGE, } EStreamScanMode; @@ -366,7 +367,7 @@ typedef struct SStreamAggSupporter { char* pKeyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - SArray* pScanWindow; + SSDataBlock* pScanBlock; } SStreamAggSupporter; typedef struct SessionWindowSupporter { @@ -401,8 +402,6 @@ typedef struct SStreamScanInfo { uint64_t numOfExec; // execution times STqReader* tqReader; - int32_t tsArrayIndex; - SArray* tsArray; uint64_t groupId; SUpdateInfo* pUpdateInfo; @@ -419,10 +418,11 @@ typedef struct SStreamScanInfo { int32_t deleteDataIndex; STimeWindow updateWin; STimeWindowAggSupp twAggSup; - + SSDataBlock* pUpdateDataRes; // status for tmq // SSchemaWrapper schema; STqOffset offset; + SNodeList* pGroupTags; SNode* pTagCond; SNode* pTagIndexCond; } SStreamScanInfo; @@ -545,9 +545,10 @@ typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; SAggSupporter aggSup; SNode* pFilterNode; // filter info, which is push down by optimizer - SSDataBlock* existDataBlock; SArray* pPseudoColInfo; SLimitInfo limitInfo; + bool mergeDataBlocks; + SSDataBlock* pFinalRes; } SProjectOperatorInfo; typedef struct SIndefOperatorInfo { @@ -712,7 +713,6 @@ typedef struct SStreamStateAggOperatorInfo { SSDataBlock* pDelRes; SHashObj* pSeDeleted; void* pDelIterator; - SArray* pScanWindow; SArray* pChildren; // cache for children's result; bool ignoreExpiredData; } SStreamStateAggOperatorInfo; @@ -805,7 +805,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock, const char* idStr); @@ -869,7 +869,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* re SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, - STimeWindowAggSupp* pTwAggSup, SExecTaskInfo* pTaskInfo); + SExecTaskInfo* pTaskInfo); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); @@ -879,8 +879,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, /*SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, const SNodeListNode* pValNode, */SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); @@ -954,6 +953,7 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); +void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid); int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, @@ -970,7 +970,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); -SSDataBlock* createPullDataBlock(); +SSDataBlock* createSpecialDataBlock(EStreamType type); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 18bb8a57f4..a76253ab20 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -265,7 +265,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { int32_t code = TSDB_CODE_SUCCESS; SMetaReader mr = {0}; @@ -356,7 +356,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i); bool qualified = true; - code = isTableOk(info, pTagCond, metaHandle, &qualified); + code = isQualifiedTable(info, pTagCond, metaHandle, &qualified); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -379,6 +379,82 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, return code; } +size_t getTableTagsBufLen(const SNodeList* pGroups) { + size_t keyLen = 0; + + SNode* node; + FOREACH(node, pGroups) { + SExprNode* pExpr = (SExprNode*)node; + keyLen += pExpr->resType.bytes; + } + + keyLen += sizeof(int8_t) * LIST_LENGTH(pGroups); + return keyLen; +} + +int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) { + SMetaReader mr = {0}; + metaReaderInit(&mr, pMeta, 0); + metaGetTableEntryByUid(&mr, uid); + + SNodeList* groupNew = nodesCloneList(pGroupNode); + + nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr); + char* isNull = (char*)keyBuf; + char* pStart = (char*)keyBuf + sizeof(int8_t)*LIST_LENGTH(pGroupNode); + + SNode* pNode; + int32_t index = 0; + FOREACH(pNode, groupNew) { + SNode* pNew = NULL; + int32_t code = scalarCalculateConstants(pNode, &pNew); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(pNew); + } else { + taosMemoryFree(keyBuf); + nodesDestroyList(groupNew); + metaReaderClear(&mr); + return code; + } + + ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); + SValueNode* pValue = (SValueNode*)pNew; + + if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) { + isNull[index++] = 1; + continue; + } else { + isNull[index++] = 0; + char* data = nodesGetValueFromNode(pValue); + if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) { + if (tTagIsJson(data)) { + terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; + taosMemoryFree(keyBuf); + nodesDestroyList(groupNew); + metaReaderClear(&mr); + return terrno; + } + int32_t len = getJsonValueLen(data); + memcpy(pStart, data, len); + pStart += len; + } else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { + memcpy(pStart, data, varDataTLen(data)); + pStart += varDataTLen(data); + } else { + memcpy(pStart, data, pValue->node.resType.bytes); + pStart += pValue->node.resType.bytes; + } + } + } + + int32_t len = (int32_t)(pStart - (char*)keyBuf); + *pGroupId = calcGroupId(keyBuf, len); + + nodesDestroyList(groupNew); + metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; +} + SArray* extractPartitionColInfo(SNodeList* pNodeList) { if (!pNodeList) { return NULL; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index c5aa90e0eb..d8cd76d31e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,10 +14,21 @@ */ #include "executor.h" +#include "tref.h" #include "executorimpl.h" #include "planner.h" #include "tdatablock.h" #include "vnode.h" +#include "tudf.h" + +static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; +int32_t exchangeObjRefPool = -1; + +static void initRefPool() { exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); } +static void cleanupRefPool() { + int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0); + taosCloseRef(ref); +} static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, bool assignUid, char* id) { @@ -120,8 +131,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO return code; } -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, - SSchemaWrapper** pSchemaWrapper) { +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) { if (msg == NULL) { // TODO create raw scan return NULL; @@ -155,7 +165,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n } } - *pSchemaWrapper = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw); + *pSchema = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw); return pTaskInfo; } @@ -185,8 +195,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { return pTaskInfo; } -static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, - const char* idstr) { +static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); // let's discard the tables those are not created according to the queried super table. @@ -209,7 +218,7 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons if (pScanInfo->pTagCond != NULL) { bool qualified = false; STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid}; - code = isTableOk(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); + code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); if (code != TSDB_CODE_SUCCESS) { qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); continue; @@ -240,7 +249,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo int32_t code = 0; SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id - SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); + SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); @@ -248,17 +257,35 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo return code; } - // add to qTaskInfo // todo refactor STableList - for (int32_t i = 0; i < taosArrayGetSize(qa); ++i) { - uint64_t* uid = taosArrayGet(qa, i); - - qDebug("table %ld added to task info", *uid); + size_t bufLen = (pScanInfo->pGroupTags != NULL)? getTableTagsBufLen(pScanInfo->pGroupTags):0; + char* keyBuf = NULL; + if (bufLen > 0) { + keyBuf = taosMemoryMalloc(bufLen); + if (keyBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + for(int32_t i = 0; i < taosArrayGetSize(qa); ++i) { + uint64_t* uid = taosArrayGet(qa, i); STableKeyInfo keyInfo = {.uid = *uid, .groupId = 0}; + + if (bufLen > 0) { + code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, + &keyInfo.groupId); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo); } + if (keyBuf != NULL) { + taosMemoryFree(keyBuf); + } + taosArrayDestroy(qa); } else { // remove the table id in current list qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); @@ -292,3 +319,396 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table return 0; } + +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) { + assert(pSubplan != NULL); + SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; + + taosThreadOnce(&initPoolOnce, initRefPool); + atexit(cleanupRefPool); + + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; + code = dsDataSinkMgtInit(&cfg); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + if (handle) { + void* pSinkParam = NULL; + code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam); + } + + _error: + // if failed to add ref for all tables in this query, abort current query + return code; +} + +#ifdef TEST_IMPL +// wait moment +int waitMoment(SQInfo* pQInfo) { + if (pQInfo->sql) { + int ms = 0; + char* pcnt = strstr(pQInfo->sql, " count(*)"); + if (pcnt) return 0; + + char* pos = strstr(pQInfo->sql, " t_"); + if (pos) { + pos += 3; + ms = atoi(pos); + while (*pos >= '0' && *pos <= '9') { + pos++; + } + char unit_char = *pos; + if (unit_char == 'h') { + ms *= 3600 * 1000; + } else if (unit_char == 'm') { + ms *= 60 * 1000; + } else if (unit_char == 's') { + ms *= 1000; + } + } + if (ms == 0) return 0; + printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); + + if (ms < 1000) { + taosMsleep(ms); + } else { + int used_ms = 0; + while (used_ms < ms) { + taosMsleep(1000); + used_ms += 1000; + if (isTaskKilled(pQInfo)) { + printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); + break; + } + } + } + } + return 1; +} +#endif + +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int64_t threadId = taosGetSelfPthreadId(); + + *pRes = NULL; + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); + pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; + return pTaskInfo->code; + } + + if (pTaskInfo->cost.start == 0) { + pTaskInfo->cost.start = taosGetTimestampMs(); + } + + if (isTaskKilled(pTaskInfo)) { + qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); + return TSDB_CODE_SUCCESS; + } + + // error occurs, record the error code and return to client + int32_t ret = setjmp(pTaskInfo->env); + if (ret != TSDB_CODE_SUCCESS) { + pTaskInfo->code = ret; + cleanUpUdfs(); + qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + atomic_store_64(&pTaskInfo->owner, 0); + + return pTaskInfo->code; + } + + qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + + int64_t st = taosGetTimestampUs(); + + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); + uint64_t el = (taosGetTimestampUs() - st); + + pTaskInfo->cost.elapsedTime += el; + if (NULL == *pRes) { + *useconds = pTaskInfo->cost.elapsedTime; + } + + cleanUpUdfs(); + + int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; + uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; + + qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); + + atomic_store_64(&pTaskInfo->owner, 0); + return pTaskInfo->code; +} + +int32_t qKillTask(qTaskInfo_t qinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; + if (pTaskInfo == NULL) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qAsyncKillTask(qinfo); + + // Wait for the query executing thread being stopped/ + // Once the query is stopped, the owner of qHandle will be cleared immediately. + while (pTaskInfo->owner != 0) { + taosMsleep(100); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qAsyncKillTask(qTaskInfo_t qinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; + + if (pTaskInfo == NULL) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); + setTaskKilled(pTaskInfo); + return TSDB_CODE_SUCCESS; +} + +void qDestroyTask(qTaskInfo_t qTaskHandle) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle; + if (pTaskInfo == NULL) { + return; + } + + qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows); + + queryCostStatis(pTaskInfo); // print the query cost summary + doDestroyTask(pTaskInfo); +} + +int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int32_t capacity = 0; + + return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum); +} + +int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { + SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; + if (pTaskInfo->pRoot == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + int32_t nOptrWithVal = 0; + int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + taosMemoryFreeClear(*pOutput); + *len = 0; + } + return code; +} + +int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) { + SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; + + if (pTaskInfo == NULL || pInput == NULL || len == 0) { + return TSDB_CODE_INVALID_PARA; + } + + return decodeOperator(pTaskInfo->pRoot, pInput, len); +} + +int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + + while (1) { + uint8_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + *scanner = pOperator->info; + return 0; + } else { + ASSERT(pOperator->numOfDownstream == 1); + pOperator = pOperator->pDownstream[0]; + } + } +} + +#if 0 +int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + taosWriteQitem(pTaskInfo->streamInfo.inputQueue->queue, pItem); + return 0; +} +#endif + +int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + pTaskInfo->streamInfo.recoverStartVer = startVer; + pTaskInfo->streamInfo.recoverEndVer = endVer; + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE; + return 0; +} + +void* qExtractReaderFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return (void*)pInfo->tqReader; +} + +const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return pInfo->tqReader->pSchemaWrapper; +} + +const STqOffset* qExtractStatusFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return &pInfo->offset; +} + +void* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + return pTaskInfo->streamInfo.metaBlk; +} + +int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); + return 0; +} + +int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + pTaskInfo->streamInfo.prepareStatus = *pOffset; + if (!tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) { + while (1) { + uint8_t type = pOperator->operatorType; + pOperator->status = OP_OPENED; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pInfo = pOperator->info; + if (pOffset->type == TMQ_OFFSET__LOG) { + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + tsdbReaderClose(pTSInfo->dataReader); + pTSInfo->dataReader = NULL; +#if 0 + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) && + pInfo->tqReader->pWalReader->curVersion != pOffset->version) { + qError("prepare scan ver %ld actual ver %ld, last %ld", pOffset->version, + pInfo->tqReader->pWalReader->curVersion, pTaskInfo->streamInfo.lastStatus.version); + ASSERT(0); + } +#endif + if (tqSeekVer(pInfo->tqReader, pOffset->version + 1) < 0) { + return -1; + } + ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version + 1); + } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { + /*pInfo->blockType = STREAM_INPUT__TABLE_SCAN;*/ + int64_t uid = pOffset->uid; + int64_t ts = pOffset->ts; + + if (uid == 0) { + if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); + uid = pTableInfo->uid; + ts = INT64_MIN; + } else { + return -1; + } + } + + /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ + /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList); + +#ifndef NDEBUG + + qDebug("switch to next table %ld (cursor %d), %ld rows returned", uid, pTableScanInfo->currentTable, + pInfo->pTableScanOp->resultInfo.totalRows); + pInfo->pTableScanOp->resultInfo.totalRows = 0; +#endif + + bool found = false; + for (int32_t i = 0; i < tableSz; i++) { + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i); + if (pTableInfo->uid == uid) { + found = true; + pTableScanInfo->currentTable = i; + break; + } + } + + // TODO after dropping table, table may be not found + ASSERT(found); + + if (pTableScanInfo->dataReader == NULL) { + if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, + pTaskInfo->tableqinfoList.pTableList, &pTableScanInfo->dataReader, NULL) < 0 || + pTableScanInfo->dataReader == NULL) { + ASSERT(0); + } + } + + tsdbSetTableId(pTableScanInfo->dataReader, uid); + int64_t oldSkey = pTableScanInfo->cond.twindows.skey; + pTableScanInfo->cond.twindows.skey = ts + 1; + tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + pTableScanInfo->cond.twindows.skey = oldSkey; + pTableScanInfo->scanTimes = 0; + + qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, + pTableScanInfo->currentTable, tableSz); + /*}*/ + + } else { + ASSERT(0); + } + return 0; + } else { + ASSERT(pOperator->numOfDownstream == 1); + pOperator = pOperator->pDownstream[0]; + } + } + } + return 0; +} + + +#if 0 +int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + + if (uid == 0) { + if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); + uid = pTableInfo->uid; + ts = INT64_MIN; + } + } + + return doPrepareScan(pTaskInfo->pRoot, uid, ts); +} + +int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + + return doGetScanStatus(pTaskInfo->pRoot, uid, ts); +} +#endif + diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c deleted file mode 100644 index e920f58560..0000000000 --- a/source/libs/executor/src/executorMain.c +++ /dev/null @@ -1,425 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "dataSinkMgt.h" -#include "os.h" -#include "tmsg.h" -#include "tref.h" -#include "tudf.h" - -#include "executor.h" -#include "executorimpl.h" -#include "query.h" - -static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; -int32_t exchangeObjRefPool = -1; - -static void initRefPool() { exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); } -static void cleanupRefPool() { - int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0); - taosCloseRef(ref); -} - -int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) { - assert(pSubplan != NULL); - SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - - taosThreadOnce(&initPoolOnce, initRefPool); - atexit(cleanupRefPool); - - int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; - code = dsDataSinkMgtInit(&cfg); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - if (handle) { - void* pSinkParam = NULL; - code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam); - } - -_error: - // if failed to add ref for all tables in this query, abort current query - return code; -} - -#ifdef TEST_IMPL -// wait moment -int waitMoment(SQInfo* pQInfo) { - if (pQInfo->sql) { - int ms = 0; - char* pcnt = strstr(pQInfo->sql, " count(*)"); - if (pcnt) return 0; - - char* pos = strstr(pQInfo->sql, " t_"); - if (pos) { - pos += 3; - ms = atoi(pos); - while (*pos >= '0' && *pos <= '9') { - pos++; - } - char unit_char = *pos; - if (unit_char == 'h') { - ms *= 3600 * 1000; - } else if (unit_char == 'm') { - ms *= 60 * 1000; - } else if (unit_char == 's') { - ms *= 1000; - } - } - if (ms == 0) return 0; - printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); - - if (ms < 1000) { - taosMsleep(ms); - } else { - int used_ms = 0; - while (used_ms < ms) { - taosMsleep(1000); - used_ms += 1000; - if (isTaskKilled(pQInfo)) { - printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); - break; - } - } - } - } - return 1; -} -#endif - -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int64_t threadId = taosGetSelfPthreadId(); - - *pRes = NULL; - int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { - qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); - pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; - return pTaskInfo->code; - } - - if (pTaskInfo->cost.start == 0) { - pTaskInfo->cost.start = taosGetTimestampMs(); - } - - if (isTaskKilled(pTaskInfo)) { - qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); - return TSDB_CODE_SUCCESS; - } - - // error occurs, record the error code and return to client - int32_t ret = setjmp(pTaskInfo->env); - if (ret != TSDB_CODE_SUCCESS) { - pTaskInfo->code = ret; - cleanUpUdfs(); - qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); - return pTaskInfo->code; - } - - qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); - - int64_t st = taosGetTimestampUs(); - - *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); - uint64_t el = (taosGetTimestampUs() - st); - - pTaskInfo->cost.elapsedTime += el; - if (NULL == *pRes) { - *useconds = pTaskInfo->cost.elapsedTime; - } - - cleanUpUdfs(); - - int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; - uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; - - qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", - GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); - - atomic_store_64(&pTaskInfo->owner, 0); - return pTaskInfo->code; -} - -int32_t qKillTask(qTaskInfo_t qinfo) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; - - if (pTaskInfo == NULL) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - qDebug("%s execTask killed", GET_TASKID(pTaskInfo)); - setTaskKilled(pTaskInfo); - - // Wait for the query executing thread being stopped/ - // Once the query is stopped, the owner of qHandle will be cleared immediately. - while (pTaskInfo->owner != 0) { - taosMsleep(100); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t qAsyncKillTask(qTaskInfo_t qinfo) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; - - if (pTaskInfo == NULL) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); - setTaskKilled(pTaskInfo); - - return TSDB_CODE_SUCCESS; -} - -void qDestroyTask(qTaskInfo_t qTaskHandle) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle; - if (pTaskInfo == NULL) { - return; - } - - qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows); - - queryCostStatis(pTaskInfo); // print the query cost summary - doDestroyTask(pTaskInfo); -} - -int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int32_t capacity = 0; - - return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum); -} - -int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { - SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; - if (pTaskInfo->pRoot == NULL) { - return TSDB_CODE_INVALID_PARA; - } - - int32_t nOptrWithVal = 0; - int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); - if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { - taosMemoryFreeClear(*pOutput); - *len = 0; - } - return code; -} - -int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) { - SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; - - if (pTaskInfo == NULL || pInput == NULL || len == 0) { - return TSDB_CODE_INVALID_PARA; - } - - return decodeOperator(pTaskInfo->pRoot, pInput, len); -} - -int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - SOperatorInfo* pOperator = pTaskInfo->pRoot; - - while (1) { - uint8_t type = pOperator->operatorType; - if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - *scanner = pOperator->info; - return 0; - } else { - ASSERT(pOperator->numOfDownstream == 1); - pOperator = pOperator->pDownstream[0]; - } - } -} - -#if 0 -int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - taosWriteQitem(pTaskInfo->streamInfo.inputQueue->queue, pItem); - return 0; -} -#endif - -int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.recoverStartVer = startVer; - pTaskInfo->streamInfo.recoverEndVer = endVer; - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE; - return 0; -} - -void* qExtractReaderFromStreamScanner(void* scanner) { - SStreamScanInfo* pInfo = scanner; - return (void*)pInfo->tqReader; -} - -const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) { - SStreamScanInfo* pInfo = scanner; - return pInfo->tqReader->pSchemaWrapper; -} - -const STqOffset* qExtractStatusFromStreamScanner(void* scanner) { - SStreamScanInfo* pInfo = scanner; - return &pInfo->offset; -} - -void* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - return pTaskInfo->streamInfo.metaBlk; -} - -int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); - return 0; -} - -int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - SOperatorInfo* pOperator = pTaskInfo->pRoot; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - pTaskInfo->streamInfo.prepareStatus = *pOffset; - if (!tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) { - while (1) { - uint8_t type = pOperator->operatorType; - pOperator->status = OP_OPENED; - if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - SStreamScanInfo* pInfo = pOperator->info; - if (pOffset->type == TMQ_OFFSET__LOG) { - STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->dataReader); - pTSInfo->dataReader = NULL; -#if 0 - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) && - pInfo->tqReader->pWalReader->curVersion != pOffset->version) { - qError("prepare scan ver %ld actual ver %ld, last %ld", pOffset->version, - pInfo->tqReader->pWalReader->curVersion, pTaskInfo->streamInfo.lastStatus.version); - ASSERT(0); - } -#endif - if (tqSeekVer(pInfo->tqReader, pOffset->version + 1) < 0) { - return -1; - } - ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version + 1); - } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - /*pInfo->blockType = STREAM_INPUT__TABLE_SCAN;*/ - int64_t uid = pOffset->uid; - int64_t ts = pOffset->ts; - - if (uid == 0) { - if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { - STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); - uid = pTableInfo->uid; - ts = INT64_MIN; - } else { - return -1; - } - } - - /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ - /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList); - -#ifndef NDEBUG - - qDebug("switch to next table %ld (cursor %d), %ld rows returned", uid, pTableScanInfo->currentTable, - pInfo->pTableScanOp->resultInfo.totalRows); - pInfo->pTableScanOp->resultInfo.totalRows = 0; -#endif - - bool found = false; - for (int32_t i = 0; i < tableSz; i++) { - STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i); - if (pTableInfo->uid == uid) { - found = true; - pTableScanInfo->currentTable = i; - break; - } - } - - // TODO after dropping table, table may be not found - ASSERT(found); - - if (pTableScanInfo->dataReader == NULL) { - if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, - pTaskInfo->tableqinfoList.pTableList, &pTableScanInfo->dataReader, NULL) < 0 || - pTableScanInfo->dataReader == NULL) { - ASSERT(0); - } - } - - tsdbSetTableId(pTableScanInfo->dataReader, uid); - int64_t oldSkey = pTableScanInfo->cond.twindows.skey; - pTableScanInfo->cond.twindows.skey = ts + 1; - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); - pTableScanInfo->cond.twindows.skey = oldSkey; - pTableScanInfo->scanTimes = 0; - - qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, - pTableScanInfo->currentTable, tableSz); - /*}*/ - - } else { - ASSERT(0); - } - return 0; - } else { - ASSERT(pOperator->numOfDownstream == 1); - pOperator = pOperator->pDownstream[0]; - } - } - } - return 0; -} - -#if 0 -int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - - if (uid == 0) { - if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { - STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); - uid = pTableInfo->uid; - ts = INT64_MIN; - } - } - - return doPrepareScan(pTaskInfo->pRoot, uid, ts); -} - -int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - - return doGetScanStatus(pTaskInfo->pRoot, uid, ts); -} -#endif diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 44a78f2a0b..7bac828a53 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -521,7 +521,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt // NOTE: the last parameter is the primary timestamp column // todo: refactor this - if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { + if (fmIsImplicitTsFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { pInput->pPTS = pInput->pData[j]; // in case of merge function, this is not always the ts column data. // ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP); } @@ -1333,7 +1333,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) { if (pFilterNode == NULL || pBlock->info.rows == 0) { return; } @@ -1354,6 +1354,20 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { filterFreeInfo(filter); extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); + + if (pColMatchInfo != NULL) { + for(int32_t i = 0; i < taosArrayGetSize(pColMatchInfo); ++i) { + SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, i); + if (pInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pInfo->targetSlotId); + if (pColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + blockDataUpdateTsWindow(pBlock, pInfo->targetSlotId); + break; + } + } + } + } + taosMemoryFree(rowRes); } @@ -1602,6 +1616,9 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG SSDataBlock* pBlock = pbInfo->pRes; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + // set output datablock version + pBlock->info.version = pTaskInfo->version; + blockDataCleanup(pBlock); if (!hasDataInGroupInfo(pGroupResInfo)) { return; @@ -3040,7 +3057,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pAggInfo->pCondition, pInfo->pRes); + doFilter(pAggInfo->pCondition, pInfo->pRes, NULL); if (!hasDataInGroupInfo(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -3206,6 +3223,7 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa pLimitInfo->currentGroupId = pBlock->info.groupId; } + // here check for a new group data, we need to handle the data of the previous group. if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { pLimitInfo->numOfOutputGroups += 1; if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { @@ -3218,6 +3236,11 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa // reset the value for a new group data pLimitInfo->numOfOutputRows = 0; pLimitInfo->remainOffset = pLimitInfo->limit.offset; + + // existing rows that belongs to previous group. + if (pBlock->info.rows > 0) { + return PROJECT_RETRIEVE_DONE; + } } // here we reach the start position, according to the limit/offset requirements. @@ -3262,7 +3285,9 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SSDataBlock* pRes = pInfo->pRes; - blockDataCleanup(pRes); + SSDataBlock* pFinalRes = pProjectInfo->pFinalRes; + + blockDataCleanup(pFinalRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->status == OP_EXEC_DONE) { @@ -3273,24 +3298,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { return NULL; } -#if 0 - if (pProjectInfo->existDataBlock) { // TODO refactor - SSDataBlock* pBlock = pProjectInfo->existDataBlock; - pProjectInfo->existDataBlock = NULL; - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); - - blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); - projectApplyFunctions(pOperator->exprSupp.pExprInfo, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->exprSupp.numOfExprs); - if (pRes->info.rows >= pProjectInfo->binfo.capacity * 0.8) { - copyTsColoum(pRes, pInfo->pCtx, pOperator->exprSupp.numOfExprs); - resetResultRowEntryResult(pInfo->pCtx, pOperator->exprSupp.numOfExprs); - return pRes; - } - } -#endif - int64_t st = 0; int32_t order = 0; int32_t scanFlag = 0; @@ -3300,67 +3307,132 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } SOperatorInfo* downstream = pOperator->pDownstream[0]; + SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo; - while (1) { - // The downstream exec may change the value of the newgroup, so use a local variable instead. - qDebug("projection call next"); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - qDebug("projection get null"); + while(1) { + while (1) { + blockDataCleanup(pRes); - /*if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH) {*/ - doSetOperatorCompleted(pOperator); - /*} else if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) {*/ - /*pOperator->status = OP_RES_TO_RETURN;*/ - /*}*/ - break; - } - if (pBlock->info.type == STREAM_RETRIEVE) { - // for stream interval - return pBlock; - } + // The downstream exec may change the value of the newgroup, so use a local variable instead. + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + break; + } - // the pDataBlock are always the same one, no need to call this again - int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + if (pBlock->info.type == STREAM_RETRIEVE) { + // for stream interval + return pBlock; + } - setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false); - blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); + if (pLimitInfo->remainGroupOffset > 0) { + if (pLimitInfo->currentGroupId == 0 || pLimitInfo->currentGroupId == pBlock->info.groupId) { // it is the first group + pLimitInfo->currentGroupId = pBlock->info.groupId; + continue; + } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { + // now it is the data from a new group + pLimitInfo->remainGroupOffset -= 1; + pLimitInfo->currentGroupId = pBlock->info.groupId; - code = projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, - pProjectInfo->pPseudoColInfo); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + // ignore data block in current group + if (pLimitInfo->remainGroupOffset > 0) { + continue; + } + } - int32_t status = handleLimitOffset(pOperator, &pProjectInfo->limitInfo, pInfo->pRes, true); + // set current group id of the project operator + pLimitInfo->currentGroupId = pBlock->info.groupId; + } - // filter shall be applied after apply functions and limit/offset on the result - doFilter(pProjectInfo->pFilterNode, pInfo->pRes); + // remainGroupOffset == 0 + // here check for a new group data, we need to handle the data of the previous group. + if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { + pLimitInfo->numOfOutputGroups += 1; + if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { + doSetOperatorCompleted(pOperator); + break; + } - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + // reset the value for a new group data + // existing rows that belongs to previous group. + pLimitInfo->numOfOutputRows = 0; + pLimitInfo->remainOffset = pLimitInfo->limit.offset; + } + + // the pDataBlock are always the same one, no need to call this again + int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false); + blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); + + code = projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, + pProjectInfo->pPseudoColInfo); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + // set current group id + pLimitInfo->currentGroupId = pBlock->info.groupId; + + if (pLimitInfo->remainOffset >= pInfo->pRes->info.rows) { + pLimitInfo->remainOffset -= pInfo->pRes->info.rows; + blockDataCleanup(pInfo->pRes); + continue; + } else if (pLimitInfo->remainOffset < pInfo->pRes->info.rows && pLimitInfo->remainOffset > 0) { + blockDataTrimFirstNRows(pInfo->pRes, pLimitInfo->remainOffset); + pLimitInfo->remainOffset = 0; + } + + // check for the limitation in each group + if (pLimitInfo->limit.limit >= 0 && + pLimitInfo->numOfOutputRows + pInfo->pRes->info.rows >= pLimitInfo->limit.limit) { + int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); + blockDataKeepFirstNRows(pInfo->pRes, keepRows); + if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { + pOperator->status = OP_EXEC_DONE; + } + } + + pLimitInfo->numOfOutputRows += pInfo->pRes->info.rows; break; } - if (status == PROJECT_RETRIEVE_CONTINUE || pInfo->pRes->info.rows == 0) { - continue; - } else if (status == PROJECT_RETRIEVE_DONE) { + // no results generated + if (pInfo->pRes->info.rows == 0 || (!pProjectInfo->mergeDataBlocks)) { + break; + } + + if (pProjectInfo->mergeDataBlocks) { + pFinalRes->info.groupId = pInfo->pRes->info.groupId; + pFinalRes->info.version = pInfo->pRes->info.version; + + // continue merge data, ignore the group id + blockDataMerge(pFinalRes, pInfo->pRes); + + if (pFinalRes->info.rows + pInfo->pRes->info.rows <= pOperator->resultInfo.threshold) { + continue; + } + } + + // do apply filter + SSDataBlock* p = pProjectInfo->mergeDataBlocks ? pFinalRes : pRes; + doFilter(pProjectInfo->pFilterNode, p, NULL); + if (p->info.rows > 0) { break; } } - size_t rows = pInfo->pRes->info.rows; - pProjectInfo->limitInfo.numOfOutputRows += rows; - - pOperator->resultInfo.totalRows += rows; + SSDataBlock* p = pProjectInfo->mergeDataBlocks ? pFinalRes : pRes; + pOperator->resultInfo.totalRows += p->info.rows; if (pOperator->cost.openCost == 0) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; } - return (rows > 0) ? pInfo->pRes : NULL; + return (p->info.rows > 0) ? p : NULL; } static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, @@ -3489,7 +3561,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { break; } - doFilter(pInfo->pCondition, fillResult); + doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo); if (fillResult->info.rows > 0) { break; } @@ -3752,6 +3824,7 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { cleanupAggSup(&pInfo->aggSup); taosArrayDestroy(pInfo->pPseudoColInfo); + blockDataDestroy(pInfo->pFinalRes); taosMemoryFreeClear(param); } @@ -3811,7 +3884,10 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo); pInfo->binfo.pRes = pResBlock; + pInfo->pFinalRes = createOneDataBlock(pResBlock, false); + pInfo->pFilterNode = pProjPhyNode->node.pConditions; + pInfo->mergeDataBlocks = pProjPhyNode->mergeDataBlock; int32_t numOfRows = 4096; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; @@ -3947,7 +4023,7 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { } } - doFilter(pIndefInfo->pCondition, pInfo->pRes); + doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL); size_t rows = pInfo->pRes->info.rows; if (rows > 0 || pOperator->status == OP_EXEC_DONE) { break; @@ -4131,9 +4207,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT return pTaskInfo; } -static STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, const char* idstr); - static SArray* extractColumnInfo(SNodeList* pNodeList); SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); @@ -4174,9 +4247,11 @@ int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, } SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { - int32_t numOfCols = LIST_LENGTH(pScanNode->pScanCols); + int32_t numOfCols = LIST_LENGTH(pScanNode->pScanCols); + int32_t numOfTags = LIST_LENGTH(pScanNode->pScanPseudoCols); + SSchemaWrapper* pqSw = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - pqSw->pSchema = taosMemoryCalloc(numOfCols, sizeof(SSchema)); + pqSw->pSchema = taosMemoryCalloc(numOfCols + numOfTags, sizeof(SSchema)); for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanCols, i); @@ -4189,6 +4264,22 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } + // this the tags and pseudo function columns, we only keep the tag columns + for(int32_t i = 0; i < numOfTags; ++i) { + STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanPseudoCols, i); + + int32_t type = nodeType(pNode->pExpr); + if (type == QUERY_NODE_COLUMN) { + SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; + + SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; + pSchema->colId = pColNode->colId; + pSchema->type = pColNode->node.resType.type; + pSchema->type = pColNode->node.resType.bytes; + strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + } + } + return pqSw; } @@ -4290,69 +4381,15 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, int32_t groupNum = 0; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - SMetaReader mr = {0}; - metaReaderInit(&mr, pHandle->meta, 0); - metaGetTableEntryByUid(&mr, info->uid); - - SNodeList* groupNew = nodesCloneList(group); - - nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr); - char* isNull = (char*)keyBuf; - char* pStart = (char*)keyBuf + nullFlagSize; - - SNode* pNode; - int32_t index = 0; - FOREACH(pNode, groupNew) { - SNode* pNew = NULL; - int32_t code = scalarCalculateConstants(pNode, &pNew); - if (TSDB_CODE_SUCCESS == code) { - REPLACE_NODE(pNew); - } else { - taosMemoryFree(keyBuf); - nodesDestroyList(groupNew); - metaReaderClear(&mr); - return code; - } - - ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); - SValueNode* pValue = (SValueNode*)pNew; - - if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) { - isNull[index++] = 1; - continue; - } else { - isNull[index++] = 0; - char* data = nodesGetValueFromNode(pValue); - if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) { - if (tTagIsJson(data)) { - terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; - taosMemoryFree(keyBuf); - nodesDestroyList(groupNew); - metaReaderClear(&mr); - return terrno; - } - int32_t len = getJsonValueLen(data); - memcpy(pStart, data, len); - pStart += len; - } else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { - memcpy(pStart, data, varDataTLen(data)); - pStart += varDataTLen(data); - } else { - memcpy(pStart, data, pValue->node.resType.bytes); - pStart += pValue->node.resType.bytes; - } - } + int32_t code = getGroupIdFromTagsVal(pHandle->meta, info->uid, group, keyBuf, &info->groupId); + if (code != TSDB_CODE_SUCCESS) { + return code; } - int32_t len = (int32_t)(pStart - (char*)keyBuf); - uint64_t groupId = calcGroupId(keyBuf, len); - taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); - info->groupId = groupId; + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t)); groupNum++; - - nodesDestroyList(groupNew); - metaReaderClear(&mr); } + taosMemoryFree(keyBuf); if (pTableListInfo->needSortTableByGroupId) { @@ -4440,12 +4477,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - STimeWindowAggSupp aggSup = (STimeWindowAggSupp){ - .waterMark = pTableScanNode->watermark, - .calTrigger = pTableScanNode->triggerType, - .maxTs = INT64_MIN, - }; - if (pHandle->vnode) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, @@ -4465,7 +4496,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo #endif pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, &aggSup, pTaskInfo); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -5136,8 +5167,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF } pSup->valueSize = size; - pSup->pScanWindow = taosArrayInit(4, sizeof(STimeWindow)); - + pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR); int32_t pageSize = 4096; while (pageSize < pSup->resultRowSize * 4) { pageSize <<= 1u; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 20630fd6ff..c83206b730 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -299,7 +299,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; while(1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes); + doFilter(pInfo->pCondition, pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 497de8347c..2e6c9bd351 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -211,7 +211,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { break; } if (pJoinInfo->pCondAfterMerge != NULL) { - doFilter(pJoinInfo->pCondAfterMerge, pRes); + doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL); } if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index fc29eed455..e29beeb3f6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -25,7 +25,6 @@ #include "tdatablock.h" #include "tmsg.h" -#include "executorimpl.h" #include "query.h" #include "tcompare.h" #include "thash.h" @@ -265,7 +264,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); int64_t et = taosGetTimestampMs(); pTableScanInfo->readRecorder.filterTime += (et - st); @@ -274,6 +273,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->filterOutBlocks += 1; qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + } else { + qDebug("%s data block filter out, elapsed time:%"PRId64, GET_TASKID(pTaskInfo), (et - st)); } return TSDB_CODE_SUCCESS; @@ -812,6 +813,10 @@ static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; } +static bool isSlidingWindow(SStreamScanInfo* pInfo) { + return isIntervalWindow(pInfo) && pInfo->interval.interval != pInfo->interval.sliding; +} + static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) { uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t)); if (groupId) { @@ -834,17 +839,10 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) { } static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex); + uint64_t* groupCol = (uint64_t*)pColInfo->pData; ASSERT(rowIndex < pBlock->info.rows); - switch (pBlock->info.type) { - case STREAM_DELETE_DATA: - case STREAM_RETRIEVE: { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex); - uint64_t* groupCol = (uint64_t*)pColInfo->pData; - pInfo->groupId = groupCol[rowIndex]; - } break; - default: - break; - } + pInfo->groupId = groupCol[rowIndex]; } void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) { @@ -864,7 +862,17 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); TSKEY* endData = (TSKEY*)pEndTsCol->pData; STimeWindow win = {.skey = startData[*pRowIndex], .ekey = endData[*pRowIndex]}; + + SColumnInfoData* pCalStartTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + TSKEY* calStartData = (TSKEY*)pCalStartTsCol->pData; + SColumnInfoData* pCalEndTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + TSKEY* calEndData = (TSKEY*)pCalEndTsCol->pData; + setGroupId(pInfo, pBlock, GROUPID_COLUMN_INDEX, *pRowIndex); + if (isSlidingWindow(pInfo)) { + pInfo->updateWin.skey = calStartData[*pRowIndex]; + pInfo->updateWin.ekey = calEndData[*pRowIndex]; + } (*pRowIndex)++; for (; *pRowIndex < pBlock->info.rows; (*pRowIndex)++) { @@ -876,8 +884,8 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ win.skey = TMIN(win.skey, startData[*pRowIndex]); continue; } - ASSERT((win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) || - (isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0))); + ASSERT(!(win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) || + !(isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0))); break; } @@ -908,68 +916,6 @@ static STimeWindow getSlidingWindow(TSKEY* tsCol, SInterval* pInterval, SDataBlo win.ekey = endWin.ekey; } } -static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { - STimeWindow win = { - .skey = INT64_MIN, - .ekey = INT64_MAX, - }; - bool needRead = false; - if (!isStateWindow(pInfo) && (*pRowIndex) < pSDB->info.rows) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, tsColIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; - SResultRowInfo dumyInfo; - dumyInfo.cur.pageId = -1; - if (isSessionWindow(pInfo)) { - SStreamAggSupporter* pAggSup = pInfo->sessionSup.pStreamAggSup; - int64_t gap = pInfo->sessionSup.gap; - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup, tsCols[*pRowIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex); - win = pCurWin->win; - setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex); - (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL); - } else { - setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex); - pInfo->updateWin.skey = tsCols[*pRowIndex]; - win = getSlidingWindow(tsCols, &pInfo->interval, &pSDB->info, pRowIndex); - pInfo->updateWin.ekey = tsCols[*pRowIndex - 1]; - // win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, TSDB_ORDER_ASC); - // (*pRowIndex) += - // getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL, - // TSDB_ORDER_ASC); - } - needRead = true; - } else if (isStateWindow(pInfo)) { - SArray* pWins = pInfo->sessionSup.pStreamAggSup->pScanWindow; - int32_t size = taosArrayGetSize(pWins); - if (pInfo->scanWinIndex < size) { - win = *(STimeWindow*)taosArrayGet(pWins, pInfo->scanWinIndex); - pInfo->scanWinIndex++; - needRead = true; - } else { - pInfo->scanWinIndex = 0; - taosArrayClear(pWins); - } - } - if (!needRead) { - return false; - } - resetTableScanInfo(pInfo->pTableScanOp->info, &win); - return true; -} - -static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) { - for (int32_t j = 0; j < taosArrayGetSize(source->pDataBlock); j++) { - SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j); - SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j); - if (colDataIsNull_s(pSourceCol, sourceRowId)) { - colDataAppendNULL(pDestCol, dest->info.rows); - } else { - colDataAppend(pDestCol, dest->info.rows, colDataGetData(pSourceCol, sourceRowId), false); - } - } - dest->info.rows++; -} static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { while (1) { @@ -982,29 +928,6 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 if (!pResult) { blockDataCleanup(pSDB); *pRowIndex = 0; - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTableScanInfo->dataReader); - pTableScanInfo->dataReader = NULL; - return NULL; - } - - if (pResult->info.groupId == pInfo->groupId) { - return pResult; - } - } -} - -static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { - while (1) { - SSDataBlock* pResult = NULL; - pResult = doTableScan(pInfo->pTableScanOp); - if (pResult == NULL) { - if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) { - // scan next window data - pResult = doTableScan(pInfo->pTableScanOp); - } - } - if (!pResult) { pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; tsdbReaderClose(pTableScanInfo->dataReader); @@ -1017,77 +940,31 @@ static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_ return pResult; } } - - /* Todo(liuyao) for partition by column - SSDataBlock* pBlock = createOneDataBlock(pResult, true); - blockDataCleanup(pResult); - for (int32_t i = 0; i < pBlock->info.rows; i++) { - uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock->info.uid); - if (id == pInfo->groupId) { - copyOneRow(pResult, pBlock, i); - } - } - return pResult; - */ -} -static void generateIntervalTs(SStreamScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator, - SSDataBlock* pUpdateRes) { - if (pDelBlock->info.rows == 0) { - return; - } - blockDataCleanup(pUpdateRes); - blockDataEnsureCapacity(pUpdateRes, 64); - ASSERT(taosArrayGetSize(pDelBlock->pDataBlock) >= 3); - SColumnInfoData* pStartTsCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* startData = (TSKEY*)pStartTsCol->pData; - SColumnInfoData* pEndTsCol = taosArrayGet(pDelBlock->pDataBlock, END_TS_COLUMN_INDEX); - TSKEY* endData = (TSKEY*)pEndTsCol->pData; - SColumnInfoData* pGpCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX); - uint64_t* uidCol = (uint64_t*)pGpCol->pData; - - SColumnInfoData* pDestTsCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, GROUPID_COLUMN_INDEX); - for (int32_t i = pInfo->deleteDataIndex; - i < pDelBlock->info.rows && - i < pDelBlock->info.capacity - (endData[i] - startData[i]) / pInfo->interval.interval - 1; - i++) { - uint64_t groupId = getGroupId(pOperator, uidCol[i]); - for (TSKEY startTs = startData[i]; startTs <= endData[i];) { - colDataAppend(pDestTsCol, pUpdateRes->info.rows, (const char*)&startTs, false); - colDataAppend(pDestGpCol, pUpdateRes->info.rows, (const char*)&groupId, false); - pUpdateRes->info.rows++; - startTs = taosTimeAdd(startTs, pInfo->interval.interval, pInfo->interval.intervalUnit, pInfo->interval.precision); - } - pInfo->deleteDataIndex++; - } - - if (pInfo->deleteDataIndex > 0 && pInfo->deleteDataIndex == pDelBlock->info.rows) { - blockDataCleanup(pDelBlock); - pInfo->deleteDataIndex = 0; - } } -static void generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SOperatorInfo* pOperator, - SSDataBlock* pUpdateRes) { - if (pBlock->info.rows == 0) { - return; +static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + if (pSrcBlock->info.rows == 0) { + return TSDB_CODE_SUCCESS; } - blockDataCleanup(pUpdateRes); - blockDataEnsureCapacity(pUpdateRes, pBlock->info.rows); - ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3); - SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + blockDataCleanup(pDestBlock); + int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + ASSERT(taosArrayGetSize(pSrcBlock->pDataBlock) >= 3); + SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startData = (TSKEY*)pStartTsCol->pData; - SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); TSKEY* endData = (TSKEY*)pEndTsCol->pData; - SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); - uint64_t* uidCol = (uint64_t*)pGpCol->pData; + SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* uidCol = (uint64_t*)pUidCol->pData; - SColumnInfoData* pDestStartCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pDestEndCol = taosArrayGet(pUpdateRes->pDataBlock, END_TS_COLUMN_INDEX); - SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); int32_t dummy = 0; - for (int32_t i = 0; i < pBlock->info.rows; i++) { - uint64_t groupId = getGroupId(pOperator, uidCol[i]); + for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { + uint64_t groupId = getGroupId(pInfo->pTableScanOp, uidCol[i]); // gap must be 0. SResultWindowInfo* pStartWin = getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy); @@ -1101,46 +978,75 @@ static void generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SOper colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false); colDataAppend(pDestEndCol, i, (const char*)&pEndWin->win.ekey, false); colDataAppend(pDestGpCol, i, (const char*)&groupId, false); - pUpdateRes->info.rows++; + pDestBlock->info.rows++; } + return TSDB_CODE_SUCCESS; } -static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) { - blockDataCleanup(pUpdateBlock); - int32_t size = taosArrayGetSize(pInfo->tsArray); - if (pInfo->tsArrayIndex < size) { - SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); - ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - blockDataEnsureCapacity(pUpdateBlock, size); - int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); - pInfo->groupId = getGroupId(pInfo->pTableScanOp, pBlock->info.uid); - int32_t i = 0; - for (; i < size; i++) { - rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex); - uint64_t id = getGroupId(pInfo->pTableScanOp, pBlock->info.uid); - if (pInfo->groupId != id) { - break; - } - copyOneRow(pUpdateBlock, pBlock, rowId); - } - pUpdateBlock->info.rows = i; - pInfo->tsArrayIndex += i; - pUpdateBlock->info.groupId = pInfo->groupId; - pUpdateBlock->info.type = STREAM_CLEAR; - blockDataUpdateTsWindow(pUpdateBlock, 0); +static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + blockDataCleanup(pDestBlock); + int32_t rows = pSrcBlock->info.rows; + if (rows == 0) { + return TSDB_CODE_SUCCESS; + } + int32_t code = blockDataEnsureCapacity(pDestBlock, rows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + SColumnInfoData* pTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* uidCol = (uint64_t*)pUidCol->pData; + ASSERT(pTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); + TSKEY* tsCol = (TSKEY*)pTsCol->pData; + SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + uint64_t groupId = getGroupId(pInfo->pTableScanOp, uidCol[0]); + for (int32_t i = 0; i < rows; ) { + colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(tsCol + i), false); + STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i); + colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(tsCol + i - 1), false); + + colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); + colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false); + colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false); + pDestBlock->info.rows++; } // all rows have same group id - ASSERT(pInfo->tsArrayIndex >= size); - if (size > 0 && pInfo->tsArrayIndex == size) { - taosArrayClear(pInfo->tsArray); - } + pDestBlock->info.groupId = groupId; + return TSDB_CODE_SUCCESS; +} - if (size == 0) { - generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pUpdateBlock); +static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + int32_t code = TSDB_CODE_SUCCESS; + if (isIntervalWindow(pInfo)) { + code = generateIntervalScanRange(pInfo, pSrcBlock, pDestBlock); + } else { + code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock); } + pDestBlock->info.type = STREAM_CLEAR; + blockDataUpdateTsWindow(pDestBlock, 0); + return code; +} + +void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid) { + SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)pStartTs, false); + colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false); + colDataAppend(pUidCol, pBlock->info.rows, (const char*)pUid, false); + pBlock->info.rows++; } static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { + if (out) { + blockDataCleanup(pInfo->pUpdateDataRes); + blockDataEnsureCapacity(pInfo->pUpdateDataRes, pBlock->info.rows); + } SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; @@ -1151,9 +1057,13 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); if ((update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup))) && out) { - taosArrayPush(pInfo->tsArray, &rowId); + appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid); } } + if (out) { + blockDataUpdateTsWindow(pInfo->pUpdateDataRes, 0); + pInfo->pUpdateDataRes->info.type = STREAM_CLEAR; + } } static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t uidColIndex) { @@ -1176,6 +1086,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock pInfo->pRes->info.rows = pBlock->info.rows; pInfo->pRes->info.uid = pBlock->info.uid; pInfo->pRes->info.type = STREAM_NORMAL; + pInfo->pRes->info.version = pBlock->info.version; uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); if (groupIdPre) { @@ -1225,7 +1136,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } } - doFilter(pInfo->pCondition, pInfo->pRes); + doFilter(pInfo->pCondition, pInfo->pRes, NULL); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataFreeRes((SSDataBlock*)pBlock); return 0; @@ -1319,26 +1230,18 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { case STREAM_RETRIEVE: { pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE; - copyDataBlock(pInfo->pPullDataRes, pBlock); - pInfo->pullDataResIndex = 0; - prepareDataScan(pInfo, pInfo->pPullDataRes, START_TS_COLUMN_INDEX, &pInfo->pullDataResIndex); + copyDataBlock(pInfo->pUpdateRes, pBlock); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); } break; case STREAM_DELETE_DATA: { pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; pInfo->updateResIndex = 0; - if (isIntervalWindow(pInfo)) { - copyDataBlock(pInfo->pDeleteDataRes, pBlock); - generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pInfo->pUpdateRes); - prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex); - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; - } else { - generateScanRange(pInfo, pBlock, pInfo->pTableScanOp, pInfo->pUpdateRes); - prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; - } - pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA; - return pInfo->pUpdateRes; + generateScanRange(pInfo, pBlock, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + return pInfo->pDeleteDataRes; } break; default: break; @@ -1346,56 +1249,40 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return pBlock; } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) { qDebug("scan mode %d", pInfo->scanMode); - if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { - blockDataDestroy(pInfo->pUpdateRes); - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - return pInfo->pRes; - } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { - if (isStateWindow(pInfo)) { + switch (pInfo->scanMode) { + case STREAM_SCAN_FROM_RES: { + blockDataDestroy(pInfo->pUpdateRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - } else { - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; - prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); - } + return pInfo->pRes; + } break; + case STREAM_SCAN_FROM_UPDATERES: { + generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + return pInfo->pUpdateRes; + } break; + case STREAM_SCAN_FROM_DATAREADER_RANGE: + case STREAM_SCAN_FROM_DATAREADER_RETRIEVE: { + SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); + if (pSDB) { + pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; + checkUpdateData(pInfo, true, pSDB, false); + return pSDB; + } + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } break; + default: + break; + } + + SStreamAggSupporter* pSup = pInfo->sessionSup.pStreamAggSup; + if (isStateWindow(pInfo) && pSup->pScanBlock->info.rows > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + pInfo->updateResIndex = 0; + copyDataBlock(pInfo->pUpdateRes, pSup->pScanBlock); + blockDataCleanup(pSup->pScanBlock); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); return pInfo->pUpdateRes; - } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RETRIEVE) { - SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex); - if (pSDB != NULL) { - checkUpdateData(pInfo, true, pSDB, false); - pSDB->info.type = STREAM_PULL_DATA; - return pSDB; - } - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; - } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) { - SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); - if (pSDB) { - pSDB->info.type = STREAM_NORMAL; - checkUpdateData(pInfo, true, pSDB, false); - return pSDB; - } - setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); - if (pInfo->pUpdateRes->info.rows > 0) { - prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); - return pInfo->pUpdateRes; - } - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE) { - SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); - if (pSDB) { - pSDB->info.type = STREAM_NORMAL; - checkUpdateData(pInfo, true, pSDB, false); - return pSDB; - } - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - } else if (isStateWindow(pInfo)) { - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; - pInfo->updateResIndex = pInfo->pUpdateRes->info.rows; - if (prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) { - blockDataCleanup(pInfo->pUpdateRes); - // return empty data blcok - return pInfo->pUpdateRes; - } - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; @@ -1452,17 +1339,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); /*pOperator->status = OP_EXEC_DONE;*/ } else if (pInfo->pUpdateInfo) { - pInfo->tsArrayIndex = 0; checkUpdateData(pInfo, true, pInfo->pRes, true); - setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey); - if (pInfo->pUpdateRes->info.rows > 0) { - if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) { + if (pInfo->pUpdateDataRes->info.rows > 0) { + if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) { pInfo->updateResIndex = 0; pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; - } else if (pInfo->pUpdateRes->info.type == STREAM_INVERT) { + } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) { pInfo->scanMode = STREAM_SCAN_FROM_RES; - return pInfo->pUpdateRes; + return pInfo->pUpdateDataRes; } } } @@ -1511,7 +1396,8 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) { #if 1 if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) { STableScanInfo* pTableScanInfo = pStreamScan->pTableScanOp->info; - destroyTableScanOperatorInfo(pTableScanInfo, 1); + destroyTableScanOperatorInfo(pTableScanInfo, numOfOutput); + taosMemoryFreeClear(pStreamScan->pTableScanOp); } #endif if (pStreamScan->tqReader) { @@ -1525,13 +1411,13 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) { blockDataDestroy(pStreamScan->pUpdateRes); blockDataDestroy(pStreamScan->pPullDataRes); blockDataDestroy(pStreamScan->pDeleteDataRes); + blockDataDestroy(pStreamScan->pUpdateDataRes); taosArrayDestroy(pStreamScan->pBlockLists); - taosArrayDestroy(pStreamScan->tsArray); taosMemoryFree(pStreamScan); } SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, - STimeWindowAggSupp* pTwSup, SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo) { SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1544,8 +1430,12 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; pInfo->pTagCond = pTagCond; - - pInfo->twAggSup = *pTwSup; + pInfo->pGroupTags = pTableScanNode->pGroupTags; + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pTableScanNode->watermark, + .calTrigger = pTableScanNode->triggerType, + .maxTs = INT64_MIN, + }; int32_t numOfCols = 0; pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); @@ -1568,11 +1458,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } - pInfo->tsArray = taosArrayInit(4, sizeof(int32_t)); - if (pInfo->tsArray == NULL) { - goto _error; - } - if (pHandle->vnode) { SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info; @@ -1628,17 +1513,18 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } pInfo->pRes = createResDataBlock(pDescNode); - pInfo->pUpdateRes = createResDataBlock(pDescNode); + pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); pInfo->pCondition = pScanPhyNode->node.pConditions; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; pInfo->groupId = 0; - pInfo->pPullDataRes = createPullDataBlock(); + pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE); pInfo->pStreamScanOp = pOperator; pInfo->deleteDataIndex = 0; - pInfo->pDeleteDataRes = createPullDataBlock(); + pInfo->pDeleteDataRes = createSpecialDataBlock(STREAM_DELETE_DATA); pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX}; + pInfo->pUpdateDataRes = createSpecialDataBlock(STREAM_CLEAR); pOperator->name = "StreamScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; @@ -1761,55 +1647,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } - doFilter(pInfo->pCondition, pInfo->pRes); -#if 0 - SFilterInfo* filter = NULL; - - int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); - - SFilterColumnParam param1 = {.numOfCols = pInfo->pRes->info.numOfCols, .pDataBlock = pInfo->pRes->pDataBlock}; - code = filterSetDataFromSlotId(filter, ¶m1); - - int8_t* rowRes = NULL; - bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); - filterFreeInfo(filter); - - SSDataBlock* px = createOneDataBlock(pInfo->pRes, false); - blockDataEnsureCapacity(px, pInfo->pRes->info.rows); - - // TODO refactor - int32_t numOfRow = 0; - for (int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) { - SColumnInfoData* pDest = taosArrayGet(px->pDataBlock, i); - SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); - - if (keep) { - colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info); - numOfRow = pInfo->pRes->info.rows; - } else if (NULL != rowRes) { - numOfRow = 0; - for (int32_t j = 0; j < pInfo->pRes->info.rows; ++j) { - if (rowRes[j] == 0) { - continue; - } - - if (colDataIsNull_s(pSrc, j)) { - colDataAppendNULL(pDest, numOfRow); - } else { - colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); - } - - numOfRow += 1; - } - } else { - numOfRow = 0; - } - } - - px->info.rows = numOfRow; - pInfo->pRes = px; -#endif - + doFilter(pInfo->pCondition, pInfo->pRes, NULL); return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } @@ -2777,7 +2615,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc } int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); int64_t et = taosGetTimestampMs(); pTableScanInfo->readRecorder.filterTime += (et - st); @@ -2895,6 +2733,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t tableEndIdx = pInfo->tableEndIndex; STableListInfo* tableListInfo = pInfo->tableListInfo; + pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx, pInfo->dataReaders, GET_TASKID(pTaskInfo)); @@ -3067,7 +2906,13 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { const STableKeyInfo* info1 = p1; const STableKeyInfo* info2 = p2; - return info1->groupId - info2->groupId; + if (info1->groupId < info2->groupId) { + return -1; + } else if (info1->groupId > info2->groupId) { + return 1; + } else { + return 0; + } } SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, @@ -3109,7 +2954,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pColMatchInfo = pColList; pInfo->pResBlock = createResDataBlock(pDescNode); - pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); pInfo->pSortInfo = generateSortByTsInfo(pInfo->pColMatchInfo, pInfo->cond.order); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index f019ee94b8..26dd772292 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -216,7 +216,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { return NULL; } - doFilter(pInfo->pCondition, pBlock); + doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo); if (blockDataGetNumOfRows(pBlock) == 0) { continue; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b5966fc463..0f1272c964 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -772,6 +772,41 @@ int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_f return midPos; } +int32_t comparePullWinKey(void* pKey, void* data, int32_t index) { + SArray* res = (SArray*)data; + SPullWindowInfo* pos = taosArrayGet(res, index); + SPullWindowInfo* pData = (SPullWindowInfo*) pKey; + if (pData->window.skey == pos->window.skey) { + if (pData->groupId > pos->groupId) { + return 1; + } else if (pData->groupId < pos->groupId) { + return -1; + } + return 0; + } else if (pData->window.skey > pos->window.skey) { + return 1; + } + return -1; +} + +static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) { + int32_t size = taosArrayGetSize(pPullWins); + int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey); + if (index == -1) { + index = 0; + } else { + if (comparePullWinKey(pPullInfo, pPullWins, index) > 0) { + index++; + } else { + return TSDB_CODE_SUCCESS; + } + } + if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + int32_t compareResKey(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; SResKeyPos* pos = taosArrayGetP(res, index); @@ -1178,7 +1213,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -1219,7 +1254,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -1256,7 +1291,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBlock); + doFilter(pInfo->pCondition, pBlock, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -1373,8 +1408,10 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) { - SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* tsCols = (TSKEY*)pTsCol->pData; + SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData; + SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* endTsCols = (TSKEY*)pEndTsCol->pData; uint64_t* pGpDatas = NULL; if (pBlock->info.type == STREAM_RETRIEVE) { SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); @@ -1382,22 +1419,18 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* } int32_t step = 0; int32_t startPos = 0; - SResultRowInfo dumyInfo; - dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[0], pInterval, TSDB_ORDER_ASC); - while (1) { - step = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - uint64_t winGpId = pGpDatas ? pGpDatas[startPos] : pBlock->info.groupId; - bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput); - if (pUpWins && res) { - SWinRes winRes = {.ts = win.skey, .groupId = winGpId}; - taosArrayPush(pUpWins, &winRes); - } - int32_t prevEndPos = step - 1 + startPos; - startPos = getNextQualifiedWindow(pInterval, &win, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); - if (startPos < 0) { - break; + for (int32_t i = 0; i < pBlock->info.rows; i++) { + SResultRowInfo dumyInfo; + dumyInfo.cur.pageId = -1; + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); + while (win.ekey <= endTsCols[i]) { + uint64_t winGpId = pGpDatas ? pGpDatas[startPos] : pBlock->info.groupId; + bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput); + if (pUpWins && res) { + SWinRes winRes = {.ts = win.skey, .groupId = winGpId}; + taosArrayPush(pUpWins, &winRes); + } + getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); } } } @@ -1501,7 +1534,7 @@ static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlo } blockDataEnsureCapacity(pBlock, size - *index); SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, DELETE_GROUPID_COLUMN_INDEX); + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); for (int32_t i = *index; i < size; i++) { SWinRes* pWin = taosArrayGet(pWins, i); colDataAppend(pTsCol, pBlock->info.rows, (const char*)&pWin->ts, false); @@ -1562,6 +1595,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { continue; } + if (pBlock->info.type == STREAM_NORMAL) { + //set input version + pTaskInfo->version = pBlock->info.version; + } + if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); @@ -1645,6 +1683,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { } } nodesDestroyNode((SNode*)pInfo->pPhyNode); + colDataDestroy(&pInfo->twAggSup.timeWindowData); taosMemoryFreeClear(param); } @@ -1793,10 +1832,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes)); pInfo->delIndex = 0; - // pInfo->pDelRes = createPullDataBlock(); todo(liuyao) for delete - pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false); // todo(liuyao) for delete - pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; // todo(liuyao) for delete - + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); initResultRowInfo(&pInfo->binfo.resultRowInfo); pOperator->name = "TimeIntervalAggOperator"; @@ -1969,7 +2005,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -2013,7 +2049,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -2585,7 +2621,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) { SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; // add pull data request - taosArrayPush(pInfo->pPullWins, &pull); + savePullWindow(&pull, pInfo->pPullWins); int32_t size = taosArrayGetSize(pInfo->pChildren); addPullWindow(pInfo->pPullDataMap, &winRes, size); qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size); @@ -2598,14 +2634,6 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc chId = getChildIndex(pSDataBlock); index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); } - // if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) { - // qDebug("===stream===delete child id %d", chId); - // taosArrayRemove(chArray, index); - // if (taosArrayGetSize(chArray) == 0) { - // // pull data is over - // taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); - // } - // } if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) { ignore = false; } @@ -2681,14 +2709,6 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol blockDataUpdateTsWindow(pDest, 0); } -static bool needBreak(SStreamFinalIntervalOperatorInfo* pInfo) { - int32_t size = taosArrayGetSize(pInfo->pPullWins); - if (pInfo->pullIndex < size) { - return true; - } - return false; -} - static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) { clearSpecialDataBlock(pBlock); int32_t size = taosArrayGetSize(array); @@ -2697,16 +2717,18 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB } blockDataEnsureCapacity(pBlock, size - (*pIndex)); ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock)); + SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); for (; (*pIndex) < size; (*pIndex)++) { SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex)); - SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); - - SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); - - SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); + colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); + colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); pBlock->info.rows++; } if ((*pIndex) == size) { @@ -2753,6 +2775,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } else if (pOperator->status == OP_RES_TO_RETURN) { + doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); + if (pInfo->pPullDataRes->info.rows != 0) { + // process the rest of the data + ASSERT(IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->pPullDataRes; + } + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows == 0) { pOperator->status = OP_EXEC_DONE; @@ -2767,10 +2797,12 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->binfo.pRes; } else { - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pInfo->binfo.pRes->info.rows != 0) { - printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); - return pInfo->binfo.pRes; + if (!IS_FINAL_OP(pInfo)) { + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows != 0) { + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } } if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { pInfo->returnUpdate = false; @@ -2779,13 +2811,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { // process the rest of the data return pInfo->pUpdateRes; } - doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); - if (pInfo->pPullDataRes->info.rows != 0) { - // process the rest of the data - ASSERT(IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); - return pInfo->pPullDataRes; - } + // doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); + // if (pInfo->pPullDataRes->info.rows != 0) { + // // process the rest of the data + // ASSERT(IS_FINAL_OP(pInfo)); + // printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + // return pInfo->pPullDataRes; + // } doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows != 0) { // process the rest of the data @@ -2825,7 +2857,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { continue; } removeResults(pUpWins, pUpdated); - copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); + copyDataBlock(pInfo->pUpdateRes, pBlock); + // copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); pInfo->returnUpdate = true; taosArrayDestroy(pUpWins); break; @@ -2884,10 +2917,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); - - if (needBreak(pInfo)) { - break; - } } } @@ -2901,6 +2930,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + + doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); + if (pInfo->pPullDataRes->info.rows != 0) { + // process the rest of the data + ASSERT(IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->pPullDataRes; + } + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); @@ -2915,14 +2953,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pUpdateRes; } - doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); - if (pInfo->pPullDataRes->info.rows != 0) { - // process the rest of the data - ASSERT(IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); - return pInfo->pPullDataRes; - } - doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows != 0) { // process the rest of the data @@ -2933,15 +2963,16 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return NULL; } -SSDataBlock* createPullDataBlock() { +SSDataBlock* createSpecialDataBlock(EStreamType type) { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->info.hasVarCol = false; pBlock->info.groupId = 0; pBlock->info.rows = 0; - pBlock->info.type = STREAM_RETRIEVE; - pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t); + pBlock->info.type = type; + pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + + sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY); - pBlock->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; infoData.info.bytes = sizeof(TSKEY); @@ -2952,6 +2983,14 @@ SSDataBlock* createPullDataBlock() { infoData.info.type = TSDB_DATA_TYPE_UBIGINT; infoData.info.bytes = sizeof(uint64_t); + // uid + taosArrayPush(pBlock->pDataBlock, &infoData); + // group id + taosArrayPush(pBlock->pDataBlock, &infoData); + + // calculate start ts + taosArrayPush(pBlock->pDataBlock, &infoData); + // calculate end ts taosArrayPush(pBlock->pDataBlock, &infoData); return pBlock; @@ -3019,8 +3058,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, goto _error; } } - pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pInfo->pUpdateRes->info.type = STREAM_CLEAR; + pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); pInfo->returnUpdate = false; @@ -3042,11 +3080,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pullIndex = 0; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); - pInfo->pPullDataRes = createPullDataBlock(); + pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE); pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; - // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete - pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false); // todo(liuyao) for delete - pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; // todo(liuyao) for delete + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->delIndex = 0; pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes)); @@ -3061,7 +3097,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - + if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { + initIntervalDownStream(downstream, pPhyNode->type); + } code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -3086,6 +3124,7 @@ void destroyStreamAggSupporter(SStreamAggSupporter* pSup) { } taosHashCleanup(pSup->pResultRows); destroyDiskbasedBuf(pSup->pResultBuf); + blockDataDestroy(pSup->pScanBlock); } void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { @@ -3103,6 +3142,10 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(pChInfo); } } + colDataDestroy(&pInfo->twAggSup.timeWindowData); + blockDataDestroy(pInfo->pDelRes); + blockDataDestroy(pInfo->pWinBlock); + blockDataDestroy(pInfo->pUpdateRes); taosMemoryFreeClear(param); } @@ -3200,7 +3243,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; - // pInfo->pDelRes = createPullDataBlock(); + // pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false); // todo(liuyao) for delete pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; // todo(liuyao) for delete pInfo->pChildren = NULL; @@ -3559,7 +3602,7 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); TSKEY* endDatas = (TSKEY*)pEndTsCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { int32_t winIndex = 0; @@ -4255,7 +4298,6 @@ static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual, pSeDeleted); ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData)); - taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); deleteWindow(pAggSup->pCurWins, winIndex); } @@ -4280,8 +4322,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } else { return; } - + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) { if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { @@ -4296,7 +4339,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo, pSDataBlock->info.rows, i, &allEqual, pInfo->pSeDeleted); if (!allEqual) { - taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win); + appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, + &pSDataBlock->info.groupId); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); deleteWindow(pAggSup->pCurWins, winIndex); continue; @@ -4460,7 +4504,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; - // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete + // pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false); // todo(liuyao) for delete pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; // todo(liuyao) for delete pInfo->pChildren = NULL; @@ -4626,7 +4670,7 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { getTableScanInfo(pOperator, &iaInfo->order, &scanFlag); setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->order, scanFlag, true); doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); - doFilter(miaInfo->pCondition, pRes); + doFilter(miaInfo->pCondition, pRes, NULL); if (pRes->info.rows >= pOperator->resultInfo.capacity) { break; } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 567941c157..324a17320e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2242,7 +2242,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last_row", .type = FUNCTION_TYPE_LAST_ROW, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2253,7 +2253,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_cache_last_row", .type = FUNCTION_TYPE_CACHE_LAST_ROW, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2263,7 +2263,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "first", .type = FUNCTION_TYPE_FIRST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2277,7 +2277,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_first_partial", .type = FUNCTION_TYPE_FIRST_PARTIAL, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLastPartial, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2288,7 +2288,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_first_merge", .type = FUNCTION_TYPE_FIRST_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLastMerge, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2299,7 +2299,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last", .type = FUNCTION_TYPE_LAST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2313,7 +2313,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_last_partial", .type = FUNCTION_TYPE_LAST_PARTIAL, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLastPartial, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2324,7 +2324,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_last_merge", .type = FUNCTION_TYPE_LAST_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLastMerge, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 05f84df7f8..b6fe5b9998 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -476,16 +476,16 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); - char* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + SFirstLastRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); int32_t type = pDestCtx->input.pData[0]->info.type; int32_t bytes = pDestCtx->input.pData[0]->info.bytes; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); - char* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || *(TSKEY*)(pDBuf + bytes) > *(TSKEY*)(pSBuf + bytes))) { - memcpy(pDBuf, pSBuf, bytes); - *(TSKEY*)(pDBuf + bytes) = *(TSKEY*)(pSBuf + bytes); + if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || pDBuf->ts > pSBuf->ts)) { + memcpy(pDBuf->buf, pSBuf->buf, bytes); + pDBuf->ts = pSBuf->ts; pDResInfo->numOfRes = 1; } return TSDB_CODE_SUCCESS; @@ -2465,8 +2465,8 @@ bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResult } int32_t apercentileFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElems = 0; - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + int32_t numOfElems = 0; + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; @@ -2502,7 +2502,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { } qDebug("add %d elements into histogram, total:%d, numOfEntry:%d, %p", numOfElems, pInfo->pHisto->numOfElems, - pInfo->pHisto->numOfEntries, pInfo->pHisto); + pInfo->pHisto->numOfEntries, pInfo->pHisto); } SET_VAL(pResInfo, numOfElems, 1); @@ -2542,18 +2542,17 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("merge histo, total:%"PRId64", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, pHisto); + qDebug("merge histo, total:%" PRId64 ", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, pHisto); } else { pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("input histogram, elem:%"PRId64", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, - pInput->pHisto); + qDebug("input histogram, elem:%" PRId64 ", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, + pInput->pHisto); SHistogramInfo* pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("merge histo, total:%"PRId64", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, - pHisto); + qDebug("merge histo, total:%" PRId64 ", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, pHisto); tHistogramDestroy(&pRes); } } @@ -2580,7 +2579,8 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { } if (pInfo->algo != APERCT_ALGO_TDIGEST) { - qDebug("after merge, total:%d, numOfEntry:%d, %p", pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto); + qDebug("after merge, total:%d, numOfEntry:%d, %p", pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, + pInfo->pHisto); } SET_VAL(pResInfo, 1, 1); @@ -2602,7 +2602,8 @@ int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } else { buildHistogramInfo(pInfo); if (pInfo->pHisto->numOfElems > 0) { - qDebug("get the final res:%d, elements:%"PRId64", entry:%d", pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries); + qDebug("get the final res:%d, elements:%" PRId64 ", entry:%d", pInfo->pHisto->numOfElems, + pInfo->pHisto->numOfEntries); double ratio[] = {pInfo->percent}; double* res = tHistogramUniform(pInfo->pHisto, ratio, 1); @@ -2656,7 +2657,6 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SAPercentileInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - ASSERT(pDBuf->algo == pSBuf->algo); qDebug("start to combine apercentile, %p", pDBuf->pHisto); @@ -2994,16 +2994,16 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { // todo rewrite: int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); - char* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + SFirstLastRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); int32_t type = pDestCtx->input.pData[0]->info.type; int32_t bytes = pDestCtx->input.pData[0]->info.bytes; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); - char* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || *(TSKEY*)(pDBuf + bytes) < *(TSKEY*)(pSBuf + bytes))) { - memcpy(pDBuf, pSBuf, bytes); - *(TSKEY*)(pDBuf + bytes) = *(TSKEY*)(pSBuf + bytes); + if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || pDBuf->ts < pSBuf->ts)) { + memcpy(pDBuf->buf, pSBuf->buf, bytes); + pDBuf->ts = pSBuf->ts; pDResInfo->numOfRes = 1; } return TSDB_CODE_SUCCESS; @@ -3879,11 +3879,11 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) { if (pCtx->end.key != INT64_MIN) { pInfo->min = pCtx->end.key; } else { - pInfo->min = ptsList[0]; + pInfo->min = ptsList[start]; } } else { if (pCtx->start.key == INT64_MIN) { - pInfo->min = (pInfo->min > ptsList[0]) ? ptsList[0] : pInfo->min; + pInfo->min = (pInfo->min > ptsList[start]) ? ptsList[start] : pInfo->min; } else { pInfo->min = pCtx->start.key; } @@ -4665,10 +4665,8 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { SSumRes* pSumRes = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; - TSKEY* tsList = (int64_t*)pInput->pPTS->pData; SColumnInfoData* pInputCol = pInput->pData[0]; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; int32_t numOfElems = 0; @@ -4704,11 +4702,6 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { } } - // TODO: remove this after pTsOutput is handled - if (pTsOutput != NULL) { - colDataAppendInt64(pTsOutput, pos, &tsList[i]); - } - numOfElems++; } @@ -5205,8 +5198,8 @@ bool twaFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { STwaInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); pInfo->isNull = false; - pInfo->p.key = INT64_MIN; - pInfo->win = TSWINDOW_INITIALIZER; + pInfo->p.key = INT64_MIN; + pInfo->win = TSWINDOW_INITIALIZER; return true; } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 6ad552576c..dae8c99aba 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -875,7 +875,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf); void udfcUvHandleRsp(SClientUvConn *conn); void udfcUvHandleError(SClientUvConn *conn); void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); -void onUdfcPipetWrite(uv_write_t *write, int status); +void onUdfcPipeWrite(uv_write_t *write, int status); void onUdfcPipeConnect(uv_connect_t *connect, int status); int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask); int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask); @@ -1226,7 +1226,7 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode * } void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - SClientUvConn *conn = handle->data; + SClientUvConn *conn = handle->data; SClientConnBuf *connBuf = &conn->readBuf; int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); @@ -1244,6 +1244,9 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf buf->base = NULL; buf->len = 0; } + } else if (connBuf->total == -1 && connBuf->len < msgHeadSize) { + buf->base = connBuf->buf + connBuf->len; + buf->len = msgHeadSize - connBuf->len; } else { connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap; void *resultBuf = taosMemoryRealloc(connBuf->buf, connBuf->cap); @@ -1258,8 +1261,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf } } - fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); - + fnDebug("udfc uv alloc buffer: cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); } bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { @@ -1267,7 +1269,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { connBuf->total = *(int32_t *) (connBuf->buf); } if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { - fnTrace("udfc complete message is received, now handle it"); + fnDebug("udfc complete message is received, now handle it"); return true; } return false; @@ -1278,7 +1280,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) { int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum if (QUEUE_EMPTY(&conn->taskQueue)) { - fnError("udfc no task waiting for response on connection"); + fnError("udfc no task waiting on connection. response seqnum:%"PRId64, seqNum); return; } bool found = false; @@ -1287,6 +1289,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) { SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); while (h != &conn->taskQueue) { + fnDebug("udfc handle response iterate through queue. uvTask:%d-%p", task->seqNum, task); if (task->seqNum == seqNum) { if (found == false) { found = true; @@ -1315,6 +1318,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) { } void udfcUvHandleError(SClientUvConn *conn) { + fnDebug("handle error on conn: %p, pipe: %p", conn, conn->pipe); while (!QUEUE_EMPTY(&conn->taskQueue)) { QUEUE* h = QUEUE_HEAD(&conn->taskQueue); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); @@ -1328,7 +1332,7 @@ void udfcUvHandleError(SClientUvConn *conn) { } void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread); + fnDebug("udfc client %p, client read from pipe. nread: %zd", client, nread); if (nread == 0) return; SClientUvConn *conn = client->data; @@ -1338,31 +1342,25 @@ void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { if (isUdfcUvMsgComplete(connBuf)) { udfcUvHandleRsp(conn); } - } if (nread < 0) { - fnError("udfc client pipe %p read error: %zd, %s.", client, nread, uv_strerror(nread)); + fnError("udfc client pipe %p read error: %zd(%s).", client, nread, uv_strerror(nread)); if (nread == UV_EOF) { fnError("\tudfc client pipe %p closed", client); } udfcUvHandleError(conn); } - } -void onUdfcPipetWrite(uv_write_t *write, int status) { - SClientUvTaskNode *uvTask = write->data; - uv_pipe_t *pipe = uvTask->pipe; - fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len); - SClientUvConn *conn = pipe->data; - if (status == 0) { - QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); - } else { - fnError("udfc client %p write error.", pipe); +void onUdfcPipeWrite(uv_write_t *write, int status) { + SClientUvConn *conn = write->data; + if (status < 0) { + fnError("udfc client connection %p write failed. status: %d(%s)", conn, status, uv_strerror(status)); udfcUvHandleError(conn); + } else { + fnDebug("udfc client connection %p write succeed", conn); } taosMemoryFree(write); - taosMemoryFree(uvTask->reqBuf.base); } void onUdfcPipeConnect(uv_connect_t *connect, int status) { @@ -1419,7 +1417,7 @@ int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN } int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) { - fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask); + fnDebug("queue uv task to event loop, uvTask: %d-%p", uvTask->type, uvTask); SUdfcProxy *udfc = uvTask->udfc; uv_mutex_lock(&udfc->taskQueueMutex); QUEUE_INSERT_TAIL(&udfc->taskQueue, &uvTask->recvTaskQueue); @@ -1427,14 +1425,14 @@ int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) { uv_async_send(&udfc->loopTaskAync); uv_sem_wait(&uvTask->taskSem); - fnInfo("udfc uv task finished. task: %d, %p", uvTask->type, uvTask); + fnInfo("udfc uvTask finished. uvTask:%"PRId64"-%d-%p", uvTask->seqNum, uvTask->type, uvTask); uv_sem_destroy(&uvTask->taskSem); return 0; } int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { - fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask); + fnDebug("event loop start uv task. uvTask: %"PRId64"-%d-%p", uvTask->seqNum, uvTask->type, uvTask); int32_t code = 0; switch (uvTask->type) { @@ -1465,10 +1463,12 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { code = TSDB_CODE_UDF_PIPE_NO_PIPE; } else { uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); - write->data = uvTask; - int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipetWrite); + write->data = pipe->data; + QUEUE* connTaskQueue = &((SClientUvConn*)pipe->data)->taskQueue; + QUEUE_INSERT_TAIL(connTaskQueue, &uvTask->connTaskQueue); + int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipeWrite); if (err != 0) { - fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err)); + fnError("udfc event loop start req_rsp task uv_write failed. uvtask: %p, code: %s", uvTask, uv_strerror(err)); } code = err; } @@ -1618,6 +1618,7 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { SClientUvTaskNode *uvTask = NULL; udfcCreateUvTask(task, uvTaskType, &uvTask); + fnDebug("udfc client task: %p created uvTask: %p. pipe: %p", task, uvTask, task->session->udfUvPipe); udfcQueueUvTask(uvTask); udfcGetUdfTaskResultFromUvTask(task, uvTask); if (uvTaskType == UV_TASK_CONNECT) { @@ -1625,6 +1626,8 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { SClientUvConn *conn = uvTask->pipe->data; conn->session = task->session; } + taosMemoryFree(uvTask->reqBuf.base); + uvTask->reqBuf.base = NULL; taosMemoryFree(uvTask); uvTask = NULL; return task->errCode; @@ -1670,7 +1673,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, SSDataBlock* output, SUdfInterBuf *newState) { - fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle); + fnDebug("udfc call udf. callType: %d, funcHandle: %p", callType, handle); SUdfcUvSession *session = (SUdfcUvSession *) handle; if (session->udfUvPipe == NULL) { fnError("No pipe to udfd"); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 2402607251..24749729c0 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -671,6 +671,9 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } + } else if (ctx->inputTotal == -1 && ctx->inputLen < msgHeadSize) { + buf->base = ctx->inputBuf + ctx->inputLen; + buf->len = msgHeadSize - ctx->inputLen; } else { ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 27c90af3e7..ba56350b38 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -708,7 +708,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } sifFreeRes(ctx.pRes); - + SIF_RET(code); } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c92e6908f1..eec4780293 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2667,6 +2667,7 @@ static int32_t jsonToExprNode(const SJson* pJson, void* pObj) { } static const char* jkColumnTableId = "TableId"; +static const char* jkColumnTableType = "TableType"; static const char* jkColumnColId = "ColId"; static const char* jkColumnColType = "ColType"; static const char* jkColumnDbName = "DbName"; @@ -2683,6 +2684,9 @@ static int32_t columnNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkColumnTableId, pNode->tableId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnTableType, pNode->tableType); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkColumnColId, pNode->colId); } @@ -2718,6 +2722,9 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkColumnTableId, &pNode->tableId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkColumnTableType, &pNode->tableType); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index e198df4bc4..5a6bea6024 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1237,7 +1237,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType); pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType); pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType); - pSelect->hasTimeLineFunc = pSelect->hasLastRowFunc ? true : fmIsTimelineFunc(pFunc->funcId); + pSelect->hasTimeLineFunc = pSelect->hasTimeLineFunc ? true : fmIsTimelineFunc(pFunc->funcId); } } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 366f970710..6a30f32f2b 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -478,8 +478,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } pAgg->hasLastRow = pSelect->hasLastRowFunc; + pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc; pAgg->node.groupAction = GROUP_ACTION_SET; - pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; + pAgg->node.requireDataOrder = pAgg->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; int32_t code = TSDB_CODE_SUCCESS; @@ -786,6 +787,14 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return code; } +static bool isPrimaryKeySort(SNodeList* pOrderByList) { + SNode* pExpr = ((SOrderByExprNode*)nodesListGetNode(pOrderByList, 0))->pExpr; + if (QUERY_NODE_COLUMN != nodeType(pExpr)) { + return false; + } + return PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pExpr)->colId; +} + static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (NULL == pSelect->pOrderByList) { return TSDB_CODE_SUCCESS; @@ -799,7 +808,9 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pSort->groupSort = pSelect->groupSort; pSort->node.groupAction = pSort->groupSort ? GROUP_ACTION_KEEP : GROUP_ACTION_CLEAR; pSort->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; - pSort->node.resultDataOrder = pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL; + pSort->node.resultDataOrder = isPrimaryKeySort(pSelect->pOrderByList) + ? (pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) + : DATA_ORDER_LEVEL_NONE; int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { @@ -928,7 +939,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe int32_t code = TSDB_CODE_SUCCESS; // set grouyp keys, agg funcs and having conditions SNodeList* pGroupKeys = NULL; - SNode* pProjection = NULL; + SNode* pProjection = NULL; FOREACH(pProjection, pSelect->pProjectionList) { code = nodesListMakeStrictAppend(&pGroupKeys, createGroupingSetNode(pProjection)); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index a7ccb72792..8586234b7e 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -215,6 +215,14 @@ static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild)); } +static bool stbSplIsMultiTbScanChild(bool streamQuery, SLogicNode* pNode) { + if (1 != LIST_LENGTH(pNode->pChildren)) { + return false; + } + SNode* pChild = nodesListGetNode(pNode->pChildren, 0); + return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild)); +} + static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) { SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode; if (WINDOW_TYPE_INTERVAL == pWindow->winType) { @@ -247,7 +255,7 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { case QUERY_NODE_LOGIC_PLAN_JOIN: return !(((SJoinLogicNode*)pNode)->isSingleTableJoin); case QUERY_NODE_LOGIC_PLAN_PARTITION: - return stbSplHasMultiTbScan(streamQuery, pNode); + return stbSplIsMultiTbScanChild(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_AGG: return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: @@ -904,14 +912,6 @@ static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplit return code; } -static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - SScanLogicNode* pScan = (SScanLogicNode*)pInfo->pSplitNode; - if (NULL != pScan->pGroupTags) { - return stbSplSplitScanNodeWithPartTags(pCxt, pInfo); - } - return stbSplSplitScanNodeWithoutPartTags(pCxt, pInfo); -} - static SNode* stbSplFindPrimaryKeyFromScan(SScanLogicNode* pScan) { SNode* pCol = NULL; FOREACH(pCol, pScan->pScanCols) { @@ -922,11 +922,12 @@ static SNode* stbSplFindPrimaryKeyFromScan(SScanLogicNode* pScan) { return NULL; } -static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan) { +static int32_t stbSplSplitMergeScanNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan, + bool groupSort) { SNodeList* pMergeKeys = NULL; int32_t code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pScan), &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, false); + code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, groupSort); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pSubplan->pChildren, @@ -937,12 +938,24 @@ static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pS return code; } +static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + SScanLogicNode* pScan = (SScanLogicNode*)pInfo->pSplitNode; + if (SCAN_TYPE_TABLE_MERGE == pScan->scanType) { + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + return stbSplSplitMergeScanNode(pCxt, pInfo->pSubplan, pScan, true); + } + if (NULL != pScan->pGroupTags) { + return stbSplSplitScanNodeWithPartTags(pCxt, pInfo); + } + return stbSplSplitScanNodeWithoutPartTags(pCxt, pInfo); +} + static int32_t stbSplSplitJoinNodeImpl(SSplitContext* pCxt, SLogicSubplan* pSubplan, SJoinLogicNode* pJoin) { int32_t code = TSDB_CODE_SUCCESS; SNode* pChild = NULL; FOREACH(pChild, pJoin->node.pChildren) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild)) { - code = stbSplSplitScanNodeForJoin(pCxt, pSubplan, (SScanLogicNode*)pChild); + code = stbSplSplitMergeScanNode(pCxt, pSubplan, (SScanLogicNode*)pChild, false); } else if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild)) { code = stbSplSplitJoinNodeImpl(pCxt, pSubplan, (SJoinLogicNode*)pChild); } else { @@ -964,8 +977,28 @@ static int32_t stbSplSplitJoinNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) return code; } +static int32_t stbSplCreateMergeKeysForPartitionNode(SLogicNode* pPart, SNodeList** pMergeKeys) { + SNode* pPrimaryKey = + nodesCloneNode(stbSplFindPrimaryKeyFromScan((SScanLogicNode*)nodesListGetNode(pPart->pChildren, 0))); + if (NULL == pPrimaryKey) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t code = nodesListAppend(pPart->pTargets, pPrimaryKey); + if (TSDB_CODE_SUCCESS == code) { + code = stbSplCreateMergeKeysByPrimaryKey(pPrimaryKey, pMergeKeys); + } + return code; +} + static int32_t stbSplSplitPartitionNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, NULL, pInfo->pSplitNode, true); + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pMergeKeys = NULL; + if (pInfo->pSplitNode->requireDataOrder >= DATA_ORDER_LEVEL_IN_GROUP) { + code = stbSplCreateMergeKeysForPartitionNode(pInfo->pSplitNode, &pMergeKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pInfo->pSplitNode, true); + } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index d4bd7e5162..bfa6079cb1 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -124,7 +124,7 @@ int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* } static int32_t adjustScanDataRequirement(SScanLogicNode* pScan, EDataOrderLevel requirement) { - if (SCAN_TYPE_TABLE != pScan->scanType || SCAN_TYPE_TABLE_MERGE != pScan->scanType) { + if (SCAN_TYPE_TABLE != pScan->scanType && SCAN_TYPE_TABLE_MERGE != pScan->scanType) { return TSDB_CODE_SUCCESS; } // The lowest sort level of scan output data is DATA_ORDER_LEVEL_IN_BLOCK @@ -161,7 +161,9 @@ static int32_t adjustAggDataRequirement(SAggLogicNode* pAgg, EDataOrderLevel req return TSDB_CODE_PLAN_INTERNAL_ERROR; } pAgg->node.resultDataOrder = requirement; - pAgg->node.requireDataOrder = requirement; + if (pAgg->hasTimeLineFunc) { + pAgg->node.requireDataOrder = requirement < DATA_ORDER_LEVEL_IN_GROUP ? DATA_ORDER_LEVEL_IN_GROUP : requirement; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 2fe731a553..8f9cd94c19 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -139,6 +139,10 @@ TEST_F(PlanBasicTest, timeLineFunc) { run("SELECT CSUM(c1) FROM t1"); run("SELECT CSUM(c1) FROM st1"); + + run("SELECT TWA(c1) FROM t1"); + + run("SELECT TWA(c1) FROM st1"); } TEST_F(PlanBasicTest, multiResFunc) { diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp index f1dd9403dd..48ab1e1ac2 100644 --- a/source/libs/planner/test/planPartByTest.cpp +++ b/source/libs/planner/test/planPartByTest.cpp @@ -58,7 +58,21 @@ TEST_F(PlanPartitionByTest, withInterval) { TEST_F(PlanPartitionByTest, withGroupBy) { useDb("root", "test"); - run("select count(*) from t1 partition by c1 group by c2"); + run("SELECT COUNT(*) FROM t1 PARTITION BY c1 GROUP BY c2"); run("SELECT TBNAME, c1 FROM st1 PARTITION BY TBNAME GROUP BY c1"); } + +TEST_F(PlanPartitionByTest, withTimeLineFunc) { + useDb("root", "test"); + + run("SELECT TWA(c1) FROM st1 PARTITION BY c1"); + + run("SELECT MAVG(c1, 2) FROM st1 PARTITION BY c1"); +} + +TEST_F(PlanPartitionByTest, withSlimit) { + useDb("root", "test"); + + run("SELECT CSUM(c1) FROM st1 PARTITION BY TBNAME SLIMIT 1"); +} diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt index 87f4bb9c64..776abd93e8 100644 --- a/source/libs/scalar/CMakeLists.txt +++ b/source/libs/scalar/CMakeLists.txt @@ -13,4 +13,4 @@ target_link_libraries(scalar if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 41ca72dc7b..29773b95c3 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3671,6 +3671,22 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { SNode *t = node->pLeft; node->pLeft = node->pRight; node->pRight = t; + switch (node->opType) { + case OP_TYPE_GREATER_THAN: + node->opType = OP_TYPE_LOWER_THAN; + break; + case OP_TYPE_LOWER_THAN: + node->opType = OP_TYPE_GREATER_THAN; + break; + case OP_TYPE_GREATER_EQUAL: + node->opType = OP_TYPE_LOWER_EQUAL; + break; + case OP_TYPE_LOWER_EQUAL: + node->opType = OP_TYPE_GREATER_EQUAL; + break; + default: + break; + } } if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 050f77bb21..a70afe25d6 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -5,6 +5,7 @@ #include "tdatablock.h" #include "tjson.h" #include "ttime.h" +#include "cJSON.h" #include "vnode.h" typedef float (*_float_fn)(float); @@ -1722,15 +1723,10 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { ASSERT(inputNum == 1); - SMetaReader mr = {0}; - metaReaderInit(&mr, pInput->param, 0); - uint64_t uid = *(uint64_t *)colDataGetData(pInput->columnData, 0); - metaGetTableEntryByUid(&mr, uid); char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(str, mr.me.name); - metaReaderClear(&mr); + metaGetTableNameByUid(pInput->param, uid, str); for(int32_t i = 0; i < pInput->numOfRows; ++i) { colDataAppend(pOutput->columnData, pOutput->numOfRows + i, str, false); diff --git a/source/libs/scalar/test/CMakeLists.txt b/source/libs/scalar/test/CMakeLists.txt index caaf86264c..32f5e098c5 100644 --- a/source/libs/scalar/test/CMakeLists.txt +++ b/source/libs/scalar/test/CMakeLists.txt @@ -1,4 +1,4 @@ enable_testing() -add_subdirectory(filter) +#add_subdirectory(filter) add_subdirectory(scalar) diff --git a/source/libs/scalar/test/filter/CMakeLists.txt b/source/libs/scalar/test/filter/CMakeLists.txt index a95a1655f8..94af1eb6f0 100644 --- a/source/libs/scalar/test/filter/CMakeLists.txt +++ b/source/libs/scalar/test/filter/CMakeLists.txt @@ -9,7 +9,7 @@ IF(NOT TD_DARWIN) ADD_EXECUTABLE(filterTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( filterTest - PUBLIC os util common gtest qcom function nodes scalar + PUBLIC os util common gtest qcom function nodes scalar parser catalog transport ) TARGET_INCLUDE_DIRECTORIES( @@ -17,4 +17,4 @@ IF(NOT TD_DARWIN) PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/" PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc" ) -ENDIF() \ No newline at end of file +ENDIF() diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 4c4d03fb37..bb7745dbd9 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -44,7 +44,6 @@ #include "scalar.h" #include "stub.h" #include "taos.h" -#include "tdatablock.h" #include "tdef.h" #include "tlog.h" #include "tvariant.h" diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index b7b635e28f..f2a5ba0ab5 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -207,6 +207,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { } taosArrayDestroy(pInfo->pTsSBFs); + taosHashCleanup(pInfo->pMap); taosMemoryFree(pInfo); } diff --git a/source/libs/sync/test/sh/a.sh b/source/libs/sync/test/sh/a.sh index b6d2bdeabf..44cd2edbec 100644 --- a/source/libs/sync/test/sh/a.sh +++ b/source/libs/sync/test/sh/a.sh @@ -22,7 +22,7 @@ done echo "" echo "generate vgId ..." -cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | awk -F, '{print $1}' | sort | uniq > ${logpath}/log.vgIds.tmp +cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | sort | uniq > ${logpath}/log.vgIds.tmp echo "all vgIds:" > ${logpath}/log.vgIds cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | awk -F, '{print $1}' | sort | uniq >> ${logpath}/log.vgIds for dnode in `ls ${logpath} | grep dnode | grep -v log`;do diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 3fa6344009..9fd4e483c2 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -396,6 +396,7 @@ typedef struct SDelayQueue { int transDQCreate(uv_loop_t* loop, SDelayQueue** queue); void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)); SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); +void transDQCancel(SDelayQueue* queue, SDelayTask* task); bool transEpSetIsEqual(SEpSet* a, SEpSet* b); /* diff --git a/source/libs/transport/src/.transSvr.c.swo b/source/libs/transport/src/.transSvr.c.swo new file mode 100644 index 0000000000..c9486c5086 Binary files /dev/null and b/source/libs/transport/src/.transSvr.c.swo differ diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9de8c273d9..da59dc605f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -27,7 +27,6 @@ typedef struct SCliConn { SConnBuffer readBuf; STransQueue cliMsgs; queue q; - uint64_t expireTime; STransCtx ctx; bool broken; // link broken or not @@ -37,6 +36,7 @@ typedef struct SCliConn { char* ip; uint32_t port; + SDelayTask* task; // debug and log info struct sockaddr_in addr; struct sockaddr_in localAddr; @@ -65,6 +65,7 @@ typedef struct SCliThrd { queue msg; TdThreadMutex msgMtx; SDelayQueue* delayQueue; + SDelayQueue* timeoutQueue; uint64_t nextTimeout; // next timeout void* pTransInst; // @@ -92,9 +93,10 @@ static void* createConnPool(int size); static void* destroyConnPool(void* pool); static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port); static void addConnToPool(void* pool, SCliConn* conn); +static void doCloseIdleConn(void* param); // register timer in each thread to clear expire conn -static void cliTimeoutCb(uv_timer_t* handle); +// static void cliTimeoutCb(uv_timer_t* handle); // alloc buf for recv static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); // callback after read nbytes from socket @@ -184,7 +186,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { pThrd = (SCliThrd*)(exh)->pThrd; \ } \ } while (0) -#define CONN_PERSIST_TIME(para) (para * 1000 * 10) +#define CONN_PERSIST_TIME(para) ((para) == 0 ? 3 * 1000 : (para)) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) #define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ @@ -194,6 +196,10 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ transClearBuffer(&conn->readBuf); \ transFreeMsg(transContFromHead((char*)head)); \ + if (transQueueSize(&conn->cliMsgs) > 0 && ahandle == 0) { \ + SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, 0); \ + if (cliMsg->type == Release) return; \ + } \ tDebug("%s conn %p receive release request, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \ if (T_REF_VAL_GET(conn) > 1) { \ transUnrefCliHandle(conn); \ @@ -384,10 +390,6 @@ void cliHandleResp(SCliConn* conn) { } uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb); - // start thread's timer of conn pool if not active - if (!uv_is_active((uv_handle_t*)&pThrd->timer) && pTransInst->idleTime > 0) { - // uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); - } } void cliHandleExcept(SCliConn* pConn) { @@ -441,30 +443,30 @@ void cliHandleExcept(SCliConn* pConn) { transUnrefCliHandle(pConn); } -void cliTimeoutCb(uv_timer_t* handle) { - SCliThrd* pThrd = handle->data; - STrans* pTransInst = pThrd->pTransInst; - int64_t currentTime = pThrd->nextTimeout; - tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label); - - SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); - while (p != NULL) { - while (!QUEUE_IS_EMPTY(&p->conn)) { - queue* h = QUEUE_HEAD(&p->conn); - SCliConn* c = QUEUE_DATA(h, SCliConn, q); - if (c->expireTime < currentTime) { - QUEUE_REMOVE(h); - transUnrefCliHandle(c); - } else { - break; - } - } - p = taosHashIterate((SHashObj*)pThrd->pool, p); - } - - pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); - uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pTransInst->idleTime) / 2, 0); -} +// void cliTimeoutCb(uv_timer_t* handle) { +// SCliThrd* pThrd = handle->data; +// STrans* pTransInst = pThrd->pTransInst; +// int64_t currentTime = pThrd->nextTimeout; +// tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label); +// +// SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); +// while (p != NULL) { +// while (!QUEUE_IS_EMPTY(&p->conn)) { +// queue* h = QUEUE_HEAD(&p->conn); +// SCliConn* c = QUEUE_DATA(h, SCliConn, q); +// if (c->expireTime < currentTime) { +// QUEUE_REMOVE(h); +// transUnrefCliHandle(c); +// } else { +// break; +// } +// } +// p = taosHashIterate((SHashObj*)pThrd->pool, p); +// } +// +// pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); +// uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pTransInst->idleTime) / 2, 0); +// } void* createConnPool(int size) { // thread local, no lock @@ -506,6 +508,10 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { QUEUE_REMOVE(&conn->q); QUEUE_INIT(&conn->q); assert(h == &conn->q); + + transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); + conn->task = NULL; + return conn; } static int32_t allocConnRef(SCliConn* conn, bool update) { @@ -537,6 +543,7 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { transReleaseExHandle(transGetRefMgt(), handle); return 0; } + static void addConnToPool(void* pool, SCliConn* conn) { if (conn->status == ConnInPool) { return; @@ -547,7 +554,6 @@ static void addConnToPool(void* pool, SCliConn* conn) { allocConnRef(conn, true); STrans* pTransInst = thrd->pTransInst; - conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); cliReleaseUnfinishedMsg(conn); transQueueClear(&conn->cliMsgs); transCtxCleanup(&conn->ctx); @@ -562,7 +568,13 @@ static void addConnToPool(void* pool, SCliConn* conn) { assert(plist != NULL); QUEUE_INIT(&conn->q); QUEUE_PUSH(&plist->conn, &conn->q); + assert(!QUEUE_IS_EMPTY(&plist->conn)); + + STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); + arg->param1 = conn; + arg->param2 = thrd; + conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, CONN_PERSIST_TIME(pTransInst->idleTime)); } static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; @@ -631,6 +643,8 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { transRemoveExHandle(transGetRefMgt(), conn->refId); conn->refId = -1; + if (conn->task != NULL) transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); + if (clear) { if (!uv_is_closing((uv_handle_t*)conn->stream)) { uv_read_stop(conn->stream); @@ -997,6 +1011,8 @@ static SCliThrd* createThrdObj() { pThrd->pool = createConnPool(4); transDQCreate(pThrd->loop, &pThrd->delayQueue); + transDQCreate(pThrd->loop, &pThrd->timeoutQueue); + pThrd->quit = false; return pThrd; } @@ -1012,6 +1028,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { transAsyncPoolDestroy(pThrd->asyncPool); transDQDestroy(pThrd->delayQueue, destroyCmsg); + transDQDestroy(pThrd->timeoutQueue, NULL); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } @@ -1058,6 +1075,10 @@ static void doCloseIdleConn(void* param) { STaskArg* arg = param; SCliConn* conn = arg->param1; SCliThrd* pThrd = arg->param2; + tTrace("%s conn %p idle, close it", CONN_GET_INST_LABEL(conn), conn); + conn->task = NULL; + cliDestroyConn(conn, true); + taosMemoryFree(arg); } static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { @@ -1248,11 +1269,17 @@ int transReleaseCliHandle(void* handle) { if (pThrd == NULL) { return -1; } + STransMsg tmsg = {.info.handle = handle}; - SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); + TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64()); + + SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; cmsg->type = Release; + STraceId* trace = &tmsg.info.traceId; + tGDebug("send release request at thread:%08" PRId64 "", pThrd->pid); + if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) { return -1; } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 34849df2b2..6ac75a75e1 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -480,7 +480,7 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) { SDelayTask* task = container_of(minNode, SDelayTask, node); STaskArg* arg = task->arg; - freeFunc(arg->param1); + if (freeFunc) freeFunc(arg->param1); taosMemoryFree(arg); taosMemoryFree(task); @@ -491,9 +491,16 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) { void transDQCancel(SDelayQueue* queue, SDelayTask* task) { uv_timer_stop(queue->timer); - if (heapSize(queue->heap) <= 0) return; + if (heapSize(queue->heap) <= 0) { + taosMemoryFree(task->arg); + taosMemoryFree(task); + return; + } heapRemove(queue->heap, &task->node); + taosMemoryFree(task->arg); + taosMemoryFree(task); + if (heapSize(queue->heap) != 0) { HeapNode* minNode = heapMin(queue->heap); if (minNode != NULL) return; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3fb947bdba..ac14e22a51 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -149,32 +149,35 @@ static void* transAcceptThread(void* arg); static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static bool addHandleToAcceptloop(void* arg); -#define CONN_SHOULD_RELEASE(conn, head) \ - do { \ - if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - conn->status = ConnRelease; \ - transClearBuffer(&conn->readBuf); \ - transFreeMsg(transContFromHead((char*)head)); \ - tTrace("conn %p received release request", conn); \ - \ - STransMsg tmsg = {.code = 0, .info.handle = (void*)conn, .info.ahandle = NULL}; \ - SSvrMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSvrMsg)); \ - srvMsg->msg = tmsg; \ - srvMsg->type = Release; \ - srvMsg->pConn = conn; \ - reallocConnRef(conn); \ - if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ - return; \ - } \ - if (conn->regArg.init) { \ - tTrace("conn %p release, notify server app", conn); \ - STrans* pTransInst = conn->pTransInst; \ - (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); \ - memset(&conn->regArg, 0, sizeof(conn->regArg)); \ - } \ - uvStartSendRespInternal(srvMsg); \ - return; \ - } \ +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + reallocConnRef(conn); \ + tTrace("conn %p received release request", conn); \ + \ + STraceId traceId = head->traceId; \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + \ + STransMsg tmsg = { \ + .code = 0, .info.handle = (void*)conn, .info.traceId = traceId, .info.ahandle = (void*)0x9527}; \ + SSvrMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSvrMsg)); \ + srvMsg->msg = tmsg; \ + srvMsg->type = Release; \ + srvMsg->pConn = conn; \ + if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ + return; \ + } \ + if (conn->regArg.init) { \ + tTrace("conn %p release, notify server app", conn); \ + STrans* pTransInst = conn->pTransInst; \ + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); \ + memset(&conn->regArg, 0, sizeof(conn->regArg)); \ + } \ + uvStartSendRespInternal(srvMsg); \ + return; \ + } \ } while (0) #define SRV_RELEASE_UV(loop) \ @@ -394,11 +397,11 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { if (pConn->status == ConnNormal) { pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType); + if (smsg->type == Release) pHead->msgType = 0; } else { if (smsg->type == Release) { pHead->msgType = 0; pConn->status = ConnNormal; - destroyConnRegArg(pConn); transUnrefSrvHandle(pConn); } else { diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 38613a77e6..e8f1f06ef1 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -58,7 +58,7 @@ static const int32_t TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) -#define SIMPLE8B_MAX_INT64 ((uint64_t)2305843009213693951LL) +#define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL) #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) #define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode @@ -101,8 +101,8 @@ int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; char bit_to_selector[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 12, 13, 13, 13, 13, 13, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; // get the byte limit. int32_t word_length = 0; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index db1207f057..11ae31919a 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -626,6 +626,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { int32_t cfgLoadFromEnvVar(SConfig *pConfig) { char *line = NULL, *name, *value, *value2, *value3; int32_t olen, vlen, vlen2, vlen3; + int32_t code = 0; ssize_t _bytes = 0; TdCmdPtr pCmd = taosOpenCmd("set"); if (pCmd == NULL) { @@ -658,9 +659,12 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { if (vlen3 != 0) value3[vlen3] = 0; } - cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR); if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { - cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR); + code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + } else { + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } @@ -674,6 +678,7 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { char buf[1024], *name, *value, *value2, *value3; int32_t olen, vlen, vlen2, vlen3; + int32_t code = 0; int32_t index = 0; if (envCmd == NULL) return 0; while (envCmd[index]!=NULL) { @@ -700,9 +705,12 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { if (vlen3 != 0) value3[vlen3] = 0; } - cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD); if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { - cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD); + code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + } else { + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } @@ -713,6 +721,7 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { char *line = NULL, *name, *value, *value2, *value3; int32_t olen, vlen, vlen2, vlen3; + int32_t code = 0; ssize_t _bytes = 0; const char *filepath = ".env"; @@ -761,9 +770,12 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { if (vlen3 != 0) value3[vlen3] = 0; } - cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE); if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { - cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE); + code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + } else { + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } @@ -819,11 +831,12 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { if (vlen3 != 0) value3[vlen3] = 0; } - code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + } else { + code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } @@ -839,9 +852,75 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } } +// int32_t cfgLoadFromCfgText(SConfig *pConfig, const char *configText) { +// char *line = NULL, *name, *value, *value2, *value3; +// int32_t olen, vlen, vlen2, vlen3; +// ssize_t _bytes = 0; +// int32_t code = 0; + +// TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); +// if (pFile == NULL) { +// // success when the file does not exist +// if (errno == ENOENT) { +// terrno = TAOS_SYSTEM_ERROR(errno); +// uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, terrstr()); +// return 0; +// } else { +// uError("failed to load from cfg file %s since %s", filepath, terrstr()); +// return -1; +// } +// } + +// while (!taosEOFFile(pFile)) { +// name = value = value2 = value3 = NULL; +// olen = vlen = vlen2 = vlen3 = 0; + +// _bytes = taosGetLineFile(pFile, &line); +// if (_bytes <= 0) { +// break; +// } + +// if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; + +// paGetToken(line, &name, &olen); +// if (olen == 0) continue; +// name[olen] = 0; + +// paGetToken(name + olen + 1, &value, &vlen); +// if (vlen == 0) continue; +// value[vlen] = 0; + +// paGetToken(value + vlen + 1, &value2, &vlen2); +// if (vlen2 != 0) { +// value2[vlen2] = 0; +// paGetToken(value2 + vlen2 + 1, &value3, &vlen3); +// if (vlen3 != 0) value3[vlen3] = 0; +// } + +// code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); +// if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; +// if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { +// code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); +// if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; +// } +// } + +// taosCloseFile(&pFile); +// if (line != NULL) taosMemoryFreeClear(line); + +// if (code == 0 || (code != 0 && terrno == TSDB_CODE_CFG_NOT_FOUND)) { +// uInfo("load from cfg file %s success", filepath); +// return 0; +// } else { +// uError("failed to load from cfg file %s since %s", filepath, terrstr()); +// return -1; +// } +// } + int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { char *cfgLineBuf = NULL, *name, *value, *value2, *value3; int32_t olen, vlen, vlen2, vlen3; + int32_t code = 0; if (url == NULL || strlen(url) == 0) { uInfo("fail to load apoll url"); return 0; @@ -916,9 +995,12 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { paGetToken(value2 + vlen2 + 1, &value3, &vlen3); if (vlen3 != 0) value3[vlen3] = 0; } - cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL); if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { - cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL); + code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + } else { + code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 184b1a9edf..927cb65f2f 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -336,6 +336,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST, "Table does not exists TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION, "Invalid table action") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS, "Table column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_COL_NOT_EXISTS, "Table column not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED, "Table column is subscribed") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") diff --git a/tests/pytest/topic/topicQuery.py b/tests/pytest/topic/topicQuery.py deleted file mode 100644 index 1ee3c3a427..0000000000 --- a/tests/pytest/topic/topicQuery.py +++ /dev/null @@ -1,91 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -from util.log import tdLog -from util.cases import tdCases -from util.sql import tdSql - -class TDTestCase: - def init(self, conn, logSql): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) - - self.ts = 1538548685000 - - def run(self): - tdSql.prepare() - - # test case for https://jira.taosdata.com:18080/browse/TD-3679 - print("==============step1") - tdSql.execute( - "create topic tq_test partitions 10") - tdSql.execute( - "insert into tq_test.p1(off, ts, content) values(0, %d, 'aaaa')" % self.ts) - tdSql.execute( - "insert into tq_test.p1(off, ts, content) values(1, %d, 'aaaa')" % (self.ts + 1)) - tdSql.execute( - "insert into tq_test.p1(off, ts, content) values(2, %d, 'aaaa')" % (self.ts + 2)) - tdSql.execute( - "insert into tq_test.p1(off, ts, content) values(3, %d, 'aaaa')" % (self.ts + 3)) - - print("==============step2") - - tdSql.query("select * from tq_test.p1") - tdSql.checkRows(4) - - tdSql.query("select * from tq_test.p1 where ts >= %d" % self.ts) - tdSql.checkRows(4) - - tdSql.query("select * from tq_test.p1 where ts > %d" % self.ts) - tdSql.checkRows(3) - - tdSql.query("select * from tq_test.p1 where ts = %d" % self.ts) - tdSql.checkRows(1) - - - tdSql.execute("use db") - tdSql.execute("create table test(ts timestamp, start timestamp, value int)") - tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts, self.ts)) - tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts + 1, self.ts + 1)) - tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts + 2, self.ts + 2)) - tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts + 3, self.ts + 3)) - - tdSql.query("select * from test") - tdSql.checkRows(4) - - tdSql.query("select * from test where ts >= %d" % self.ts) - tdSql.checkRows(4) - - tdSql.query("select * from test where ts > %d" % self.ts) - tdSql.checkRows(3) - - tdSql.query("select * from test where ts = %d" % self.ts) - tdSql.checkRows(1) - - tdSql.query("select * from test where start >= %d" % self.ts) - tdSql.checkRows(4) - - tdSql.query("select * from test where start > %d" % self.ts) - tdSql.checkRows(3) - - tdSql.query("select * from test where start = %d" % self.ts) - tdSql.checkRows(1) - - - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 7a5b70d6ca..9b72312028 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -457,14 +457,14 @@ class TDCom: def newcon(self,host='localhost',port=6030,user='root',password='taosdata'): con=taos.connect(host=host, user=user, password=password, port=port) - print(con) + # print(con) return con def newcur(self,host='localhost',port=6030,user='root',password='taosdata'): cfgPath = self.getClientCfgPath() con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port) cur=con.cursor() - print(cur) + # print(cur) return cur def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata'): diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 052bc62f59..98bc138f7e 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -69,6 +69,7 @@ ./test.sh -f tsim/insert/basic.sim ./test.sh -f tsim/insert/basic0.sim ./test.sh -f tsim/insert/basic1.sim +./test.sh -f tsim/insert/basic2.sim ./test.sh -f tsim/insert/commit-merge0.sim ./test.sh -f tsim/insert/insert_drop.sim ./test.sh -f tsim/insert/insert_select.sim @@ -88,17 +89,17 @@ ./test.sh -f tsim/parser/alter_column.sim ./test.sh -f tsim/parser/alter_stable.sim ./test.sh -f tsim/parser/alter.sim -# ./test.sh -f tsim/parser/alter1.sim +# TD-17661 ./test.sh -f tsim/parser/alter1.sim ./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim ./test.sh -f tsim/parser/auto_create_tb.sim ./test.sh -f tsim/parser/between_and.sim ./test.sh -f tsim/parser/binary_escapeCharacter.sim -# ./test.sh -f tsim/parser/col_arithmetic_operation.sim -# ./test.sh -f tsim/parser/columnValue.sim +# TD-17738 ./test.sh -f tsim/parser/col_arithmetic_operation.sim +# TD-17661 ./test.sh -f tsim/parser/columnValue.sim ./test.sh -f tsim/parser/commit.sim # TD-17661 ./test.sh -f tsim/parser/condition.sim ./test.sh -f tsim/parser/constCol.sim -./test.sh -f tsim/parser/create_db.sim +#./test.sh -f tsim/parser/create_db.sim ./test.sh -f tsim/parser/create_mt.sim # TD-17653 ./test.sh -f tsim/parser/create_tb_with_tag_name.sim ./test.sh -f tsim/parser/create_tb.sim @@ -109,61 +110,58 @@ ./test.sh -f tsim/parser/fill.sim ./test.sh -f tsim/parser/first_last.sim ./test.sh -f tsim/parser/fourArithmetic-basic.sim -# TD-17659 TD-17658 ./test.sh -f tsim/parser/function.sim +# TD-17659 ./test.sh -f tsim/parser/function.sim ./test.sh -f tsim/parser/groupby-basic.sim -# ./test.sh -f tsim/parser/groupby.sim +./test.sh -f tsim/parser/groupby.sim # TD-17622 ./test.sh -f tsim/parser/having_child.sim -# ./test.sh -f tsim/parser/having.sim +# TD-17622 ./test.sh -f tsim/parser/having.sim ./test.sh -f tsim/parser/import_commit1.sim ./test.sh -f tsim/parser/import_commit2.sim ./test.sh -f tsim/parser/import_commit3.sim -# TD-17663 ./test.sh -f tsim/parser/import_file.sim +./test.sh -f tsim/parser/import_file.sim ./test.sh -f tsim/parser/import.sim ./test.sh -f tsim/parser/insert_multiTbl.sim ./test.sh -f tsim/parser/insert_tb.sim -# ./test.sh -f tsim/parser/interp.sim +# TD-17038 ./test.sh -f tsim/parser/interp.sim ./test.sh -f tsim/parser/join_manyblocks.sim -# ./test.sh -f tsim/parser/join_multitables.sim +# TD-17713 ./test.sh -f tsim/parser/join_multitables.sim # TD-17713 ./test.sh -f tsim/parser/join_multivnode.sim # TD-17707 ./test.sh -f tsim/parser/join.sim ./test.sh -f tsim/parser/last_cache.sim ./test.sh -f tsim/parser/last_groupby.sim -# TD-17675 ./test.sh -f tsim/parser/lastrow.sim +# TD-17722 ./test.sh -f tsim/parser/lastrow.sim ./test.sh -f tsim/parser/like.sim -# ./test.sh -f tsim/parser/limit.sim -# ./test.sh -f tsim/parser/limit1.sim -# ./test.sh -f tsim/parser/limit2.sim +# TD-17464 ./test.sh -f tsim/parser/limit.sim +# TD-17464 ./test.sh -f tsim/parser/limit1.sim +# TD-17623 ./test.sh -f tsim/parser/limit2.sim ./test.sh -f tsim/parser/mixed_blocks.sim ./test.sh -f tsim/parser/nchar.sim # TD-17703 ./test.sh -f tsim/parser/nestquery.sim -# ./test.sh -f tsim/parser/null_char.sim +# TD-17685 ./test.sh -f tsim/parser/null_char.sim ./test.sh -f tsim/parser/precision_ns.sim ./test.sh -f tsim/parser/projection_limit_offset.sim ./test.sh -f tsim/parser/regex.sim ./test.sh -f tsim/parser/select_across_vnodes.sim ./test.sh -f tsim/parser/select_distinct_tag.sim ./test.sh -f tsim/parser/select_from_cache_disk.sim -# ./test.sh -f tsim/parser/select_with_tags.sim +# TD-17659 ./test.sh -f tsim/parser/select_with_tags.sim ./test.sh -f tsim/parser/selectResNum.sim # TD-17685 ./test.sh -f tsim/parser/set_tag_vals.sim ./test.sh -f tsim/parser/single_row_in_tb.sim # TD-17684 ./test.sh -f tsim/parser/sliding.sim -# ./test.sh -f tsim/parser/slimit_alter_tags.sim -# ./test.sh -f tsim/parser/slimit.sim -# ./test.sh -f tsim/parser/slimit1.sim +# TD-17722 ./test.sh -f tsim/parser/slimit_alter_tags.sim +# TD-17722 ./test.sh -f tsim/parser/slimit.sim +# TD-17722 ./test.sh -f tsim/parser/slimit1.sim ./test.sh -f tsim/parser/stableOp.sim -# ./test.sh -f tsim/parser/tags_dynamically_specifiy.sim -# ./test.sh -f tsim/parser/tags_filter.sim +# TD-17661 ./test.sh -f tsim/parser/tags_dynamically_specifiy.sim +# TD-17661 ./test.sh -f tsim/parser/tags_filter.sim ./test.sh -f tsim/parser/tbnameIn.sim ./test.sh -f tsim/parser/timestamp.sim ./test.sh -f tsim/parser/top_groupby.sim ./test.sh -f tsim/parser/topbot.sim -# ./test.sh -f tsim/parser/udf_dll_stable.sim -# ./test.sh -f tsim/parser/udf_dll.sim -# ./test.sh -f tsim/parser/udf.sim ./test.sh -f tsim/parser/union.sim # TD-17704 ./test.sh -f tsim/parser/union_sysinfo.sim -# ./test.sh -f tsim/parser/where.sim +# TD-17661 ./test.sh -f tsim/parser/where.sim # ---- query ./test.sh -f tsim/query/interval.sim @@ -225,7 +223,7 @@ # ---- stream ./test.sh -f tsim/stream/basic0.sim -./test.sh -f tsim/stream/basic1.sim +#./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/drop_stream.sim ./test.sh -f tsim/stream/distributeInterval0.sim @@ -324,7 +322,7 @@ ./test.sh -f tsim/vnode/stable_replica3_vnode3.sim # --- sync -# ./test.sh -f tsim/sync/3Replica1VgElect.sim +./test.sh -f tsim/sync/3Replica1VgElect.sim ./test.sh -f tsim/sync/3Replica5VgElect.sim ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim @@ -422,18 +420,18 @@ ./test.sh -f tsim/tag/bool_binary.sim ./test.sh -f tsim/tag/bool_int.sim ./test.sh -f tsim/tag/bool.sim -# ./test.sh -f tsim/tag/change.sim -# ./test.sh -f tsim/tag/column.sim -# ./test.sh -f tsim/tag/commit.sim -# ./test.sh -f tsim/tag/create.sim -# /test.sh -f tsim/tag/delete.sim -# ./test.sh -f tsim/tag/double.sim -# ./test.sh -f tsim/tag/filter.sim +# TD-17661 ./test.sh -f tsim/tag/change.sim +./test.sh -f tsim/tag/column.sim +./test.sh -f tsim/tag/commit.sim +# TD-17661 ./test.sh -f tsim/tag/create.sim +# TD-17661 ./test.sh -f tsim/tag/delete.sim +# TD-17661 ./test.sh -f tsim/tag/double.sim +# TD-17661 ./test.sh -f tsim/tag/filter.sim # TD-17407 ./test.sh -f tsim/tag/float.sim ./test.sh -f tsim/tag/int_binary.sim ./test.sh -f tsim/tag/int_float.sim ./test.sh -f tsim/tag/int.sim -# ./test.sh -f tsim/tag/set.sim +# TD-17661 ./test.sh -f tsim/tag/set.sim ./test.sh -f tsim/tag/smallint.sim ./test.sh -f tsim/tag/tinyint.sim diff --git a/tests/script/jenkins/clusterCase.txt b/tests/script/jenkins/clusterCase.txt new file mode 100644 index 0000000000..b6b20e9517 --- /dev/null +++ b/tests/script/jenkins/clusterCase.txt @@ -0,0 +1,29 @@ + +#======================b1-start=============== + +# ---- mnode +./test.sh -f tsim/mnode/basic1.sim +./test.sh -f tsim/mnode/basic2.sim +./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic4.sim +./test.sh -f tsim/mnode/basic5.sim + +# --- vnode +# unsupport ./test.sh -f tsim/vnode/replica3_basic.sim +# unsupport ./test.sh -f tsim/vnode/replica3_repeat.sim +# unsupport ./test.sh -f tsim/vnode/replica3_vgroup.sim +# unsupport ./test.sh -f tsim/vnode/replica3_many.sim +# unsupport ./test.sh -f tsim/vnode/replica3_import.sim +# unsupport ./test.sh -f tsim/vnode/stable_balance_replica1.sim +# unsupport ./test.sh -f tsim/vnode/stable_dnode2_stop.sim +./test.sh -f tsim/vnode/stable_dnode2.sim +./test.sh -f tsim/vnode/stable_dnode3.sim +./test.sh -f tsim/vnode/stable_replica3_dnode6.sim +./test.sh -f tsim/vnode/stable_replica3_vnode3.sim + +# --- sync +# jira ./test.sh -f tsim/sync/3Replica1VgElect.sim +./test.sh -f tsim/sync/3Replica5VgElect.sim +./test.sh -f tsim/sync/oneReplica1VgElect.sim +./test.sh -f tsim/sync/oneReplica5VgElect.sim + diff --git a/tests/script/tsim/dnode/offline_reason.sim b/tests/script/tsim/dnode/offline_reason.sim index 3c6fff8b59..3608dd4a3a 100644 --- a/tests/script/tsim/dnode/offline_reason.sim +++ b/tests/script/tsim/dnode/offline_reason.sim @@ -40,7 +40,7 @@ if $data(2)[4] != ready then endi print ========== step3 -system sh/exec.sh -n dnode2 -s stop +system sh/exec.sh -n dnode2 -s stop $x = 0 step3: @@ -64,9 +64,10 @@ if $rows != 1 then endi print ========== step5 -system sh/exec.sh -n dnode2 -s start sql create dnode $hostname port 7200 +system sh/exec.sh -n dnode2 -s start +return $x = 0 step5: $x = $x + 1 diff --git a/tests/script/tsim/insert/basic2.sim b/tests/script/tsim/insert/basic2.sim new file mode 100644 index 0000000000..0bd64b1d02 --- /dev/null +++ b/tests/script/tsim/insert/basic2.sim @@ -0,0 +1,322 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database +sql create database d0 keep 365000d,365000d,365000d +sql use d0 + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int unsigned, c2 double, c3 binary(10), c4 nchar(10), c5 double) tags (city binary(20),district binary(20)); + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags("BeiJing", "ChaoYang") +sql create table ct2 using stb tags("BeiJing", "HaiDian") + +sql show tables +if $rows != 2 then + return -1 +endi + +print =============== step3-1 insert records into ct1 +sql insert into ct1 values('2022-05-03 16:59:00.010', 10, 20, 'n','n',30); +sql insert into ct1 values('2022-05-03 16:59:00.011', 'N', 'n', 'N',"N",30); +sql insert into ct1 values('2022-05-03 16:59:00.012', 'Nu', 'nul', 'Nul','NUL',30); +sql insert into ct1 values('2022-05-03 16:59:00.013', NULL, 'null', 'Null',null,30); +sql insert into ct1 values('2022-05-03 16:59:00.014', NULL, 'NuLL', 'Null',NULL,30); + +sql_error insert into ct1 values('2022-05-03 16:59:00.015', NULL, 20, 'Null',NUL,30); +sql_error insert into ct1 values('2022-05-03 16:59:00.015', NULL, 20, 'Null',NU,30); +sql_error insert into ct1 values('2022-05-03 16:59:00.015', NULL, 20, 'Null',Nu,30); +sql_error insert into ct1 values('2022-05-03 16:59:00.015', NULL, 20, 'Null',N,30); +sql_error insert into ct1 values('2022-05-03 16:59:00.015', N, 20, 'Null',NULL,30); +sql_error insert into ct1 values('2022-05-03 16:59:00.015', Nu, 20, 'Null',NULL,30); +sql_error insert into ct1 values('2022-05-03 16:59:00.015', Nul, 20, 'Null',NULL,30); + +print =============== step3-1 query records of ct1 from memory +sql select * from ct1; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 + +if $rows != 5 then + print rows $rows != 5 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data02 != 20.000000000 then + print data02 $data02 != 20.000000000 + return -1 +endi + +if $data03 != n then + print data03 $data03 != n + return -1 +endi + +if $data04 != n then + print data04 $data04 != n + return -1 +endi + +if $data05 != 30.000000000 then + print data05 $data05 != 30.000000000 + return -1 +endi + +if $data11 != NULL then + print data11 $data11 != NULL + return -1 +endi + +if $data12 != NULL then + print data12 $data12 != NULL + return -1 +endi + +if $data13 != N then + print data13 $data13 != N + return -1 +endi + +if $data14 != N then + print data14 $data14 != N + return -1 +endi + +if $data15 != 30.000000000 then + print data15 $data15 != 30.000000000 + return -1 +endi + +if $data21 != NULL then + print data21 $data21 != NULL + return -1 +endi + +if $data22 != NULL then + print data22 $data22 != NULL + return -1 +endi + +if $data23 != Nul then + print data23 $data23 != Nul + return -1 +endi + +if $data24 != NUL then + print data24 $data24 != NUL + return -1 +endi + +if $data25 != 30.000000000 then + print data25 $data25 != 30.000000000 + return -1 +endi + +if $data31 != NULL then + print data31 $data31 != NULL + return -1 +endi + +if $data32 != NULL then + print data32 $data32 != NULL + return -1 +endi + +if $data33 != Null then + print data33 $data33 != Null + return -1 +endi + +if $data34 != NULL then + print data34 $data34 != NULL + return -1 +endi + +if $data35 != 30.000000000 then + print data35 $data35 != 30.000000000 + return -1 +endi + +if $data41 != NULL then + print data41 $data41 != NULL + return -1 +endi + +if $data42 != NULL then + print data42 $data42 != NULL + return -1 +endi + +if $data43 != Null then + print data43 $data43 != Null + return -1 +endi + +if $data44 != NULL then + print data44 $data44 != NULL + return -1 +endi + +if $data45 != 30.000000000 then + print data45 $data45 != 30.000000000 + return -1 +endi + +#==================== reboot to trigger commit data to file +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +print =============== step3-2 query records of ct1 from file +sql select * from ct1; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 + +if $rows != 5 then + print rows $rows != 5 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data02 != 20.000000000 then + print data02 $data02 != 20.000000000 + return -1 +endi + +if $data03 != n then + print data03 $data03 != n + return -1 +endi + +if $data04 != n then + print data04 $data04 != n + return -1 +endi + +if $data05 != 30.000000000 then + print data05 $data05 != 30.000000000 + return -1 +endi + +if $data11 != NULL then + print data11 $data11 != NULL + return -1 +endi + +if $data12 != NULL then + print data12 $data12 != NULL + return -1 +endi + +if $data13 != N then + print data13 $data13 != N + return -1 +endi + +if $data14 != N then + print data14 $data14 != N + return -1 +endi + +if $data15 != 30.000000000 then + print data15 $data15 != 30.000000000 + return -1 +endi + +if $data21 != NULL then + print data21 $data21 != NULL + return -1 +endi + +if $data22 != NULL then + print data22 $data22 != NULL + return -1 +endi + +if $data23 != Nul then + print data23 $data23 != Nul + return -1 +endi + +if $data24 != NUL then + print data24 $data24 != NUL + return -1 +endi + +if $data25 != 30.000000000 then + print data25 $data25 != 30.000000000 + return -1 +endi + +if $data31 != NULL then + print data31 $data31 != NULL + return -1 +endi + +if $data32 != NULL then + print data32 $data32 != NULL + return -1 +endi + +if $data33 != Null then + print data33 $data33 != Null + return -1 +endi + +if $data34 != NULL then + print data34 $data34 != NULL + return -1 +endi + +if $data35 != 30.000000000 then + print data35 $data35 != 30.000000000 + return -1 +endi + +if $data41 != NULL then + print data41 $data41 != NULL + return -1 +endi + +if $data42 != NULL then + print data42 $data42 != NULL + return -1 +endi + +if $data43 != Null then + print data43 $data43 != Null + return -1 +endi + +if $data44 != NULL then + print data44 $data44 != NULL + return -1 +endi + +if $data45 != 30.000000000 then + print data45 $data45 != 30.000000000 + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/insert/commit-merge0.sim b/tests/script/tsim/insert/commit-merge0.sim index 5fe7cc57b3..66486c4c31 100644 --- a/tests/script/tsim/insert/commit-merge0.sim +++ b/tests/script/tsim/insert/commit-merge0.sim @@ -242,3 +242,5 @@ else $reboot_cnt = $reboot_cnt + 1 goto reboot_and_check endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/insert/update0.sim b/tests/script/tsim/insert/update0.sim index c6843acb9d..c4bd29615b 100644 --- a/tests/script/tsim/insert/update0.sim +++ b/tests/script/tsim/insert/update0.sim @@ -226,4 +226,6 @@ endi if $data41 != NULL then print data41 $data41 != NULL return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/insert/update1_sort_merge.sim b/tests/script/tsim/insert/update1_sort_merge.sim index 79d72b43a0..13462520ea 100644 --- a/tests/script/tsim/insert/update1_sort_merge.sim +++ b/tests/script/tsim/insert/update1_sort_merge.sim @@ -816,4 +816,6 @@ endi if $data44 != n8 then print data44 $data44 != n8 return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/col_arithmetic_query.sim b/tests/script/tsim/parser/col_arithmetic_query.sim index 8ee891660b..10840b2296 100644 --- a/tests/script/tsim/parser/col_arithmetic_query.sim +++ b/tests/script/tsim/parser/col_arithmetic_query.sim @@ -36,28 +36,22 @@ sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc; if $rows != 10000 then return -1 endi - if $data00 != 0.000000000 then return -1 endi - #if $data01 != -nan then # print expect -nan, actual: $data01 # return -1 #endi - if $data10 != 0.666666667 then return -1 endi - if $data11 != 1.000000000 then return -1 endi - if $data90 != 6.000000000 then return -1 endi - if $data91 != 1.000000000 then return -1 endi @@ -66,65 +60,49 @@ sql select (c1 * 2) % 7.9, c1*1, c1*1*1, c1*c1, c1*c1*c1 from $tb order by ts de if $rows != 10000 then return -1 endi - if $data00 != 2.200000000 then print expect 2.200000000, actual:$data00 return -1 endi - if $data01 != 9.000000000 then return -1 endi - if $data02 != 9.000000000 then return -1 endi - if $data03 != 81.000000000 then return -1 endi - if $data04 != 729.000000000 then return -1 endi - - if $data10 != 0.200000000 then return -1 endi - if $data11 != 8.000000000 then return -1 endi - if $data12 != 8.000000000 then return -1 endi - if $data13 != 64.000000000 then return -1 endi - if $data14 != 512.000000000 then return -1 endi - if $data90 != 0.000000000 then return -1 endi - if $data91 != 0.000000000 then return -1 endi - if $data92 != 0.000000000 then return -1 endi - if $data93 != 0.000000000 then return -1 endi - if $data94 != 0.000000000 then return -1 endi @@ -134,20 +112,16 @@ sql select c1 * c2 /4 from $tb where ts < 1537166000000 and ts > 1537156000000 if $rows != 17 then return -1 endi - if $data00 != 12.250000000 then return -1 endi - if $data10 != 16.000000000 then return -1 endi - if $data20 != 20.250000000 then print expect 20.250000000, actual:$data21 return -1 endi - if $data30 != 0.000000000 then return -1 endi @@ -180,47 +154,36 @@ sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb if $rows != 10000 then return -1 endi - if $data00 != 0.000000000 then return -1 endi - #if $data01 != -nan then # return -1 #endi - if $data02 != 0.000000000 then return -1 endi - if $data03 != NULL then return -1 endi - if $data04 != 0.000000000 then return -1 endi - if $data05 != 0.000000000 then return -1 endi - if $data90 != -0.900000000 then return -1 endi - if $data91 != 1.000000000 then return -1 endi - if $data92 != 81.000000000 then return -1 endi - if $data93 != 0.000000000 then return -1 endi - if $data94 != 18.000000000 then return -1 endi @@ -237,10 +200,8 @@ sql select c8+c7, c9+c9+c8+c7/c6 from $tb # arithmetic expression in join [d.7]================================================== - # arithmetic expression in union [d.8]================================================= - # arithmetic expression in group by [d.9]============================================== # in group by tag, not support for normal table sql_error select c5*99 from $tb group by t1 @@ -248,17 +209,14 @@ sql_error select c5*99 from $tb group by t1 # in group by column sql_error select c6-(c6+c3)*12 from $tb group by c3; - # limit offset [d.10]================================================================== sql select c6 * c1 + 12 from $tb limit 12 offset 99; if $rows != 12 then return -1 endi - if $data00 != 93.000000000 then return -1 endi - if $data90 != 76.000000000 then return -1 endi @@ -267,7 +225,6 @@ sql select c4 / 99.123 from $tb limit 10 offset 9999; if $rows != 1 then return -1 endi - if $data00 != 0.090796283 then return -1 endi @@ -283,27 +240,21 @@ sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb if $rows != 10000 then return -1 endi - if $data00 != 0 then return -1 endi - if $data01 != 0.000000000 then return -1 endi - if $data02 != 12.987654568 then return -1 endi - if $data03 != 1 then return -1 endi - if $data04 != 1.100000000 then return -1 endi - if $data10 != 1 then return -1 endi @@ -313,27 +264,21 @@ sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2 if $rows != 2000 then return -1 endi - if $data00 != 0 then return -1 endi - if $data01 != 0.000000000 then return -1 endi - if $data02 != 12.987654568 then return -1 endi - if $data03 != 1 then return -1 endi - if $data10 != 1 then return -1 endi - if $data20 != 0 then return -1 endi @@ -377,7 +322,6 @@ sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc; if $rows != 1 then return -1 endi - if $data00 != 0.000000000 then return -1 endi @@ -386,7 +330,6 @@ sql select first(c1) * (2/99) from $stb order by ts desc; if $rows != 1 then return -1 endi - if $data00 != 0.000000000 then return -1 endi @@ -395,15 +338,12 @@ sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb if $rows != 1 then return -1 endi - if $data00 != 1.800000000 then return -1 endi - if $data01 != 100000.000000000 then return -1 endi - if $data02 != 200000.000000000 then return -1 endi @@ -412,16 +352,13 @@ sql select spread( c1 )/44, spread(c1), 0.204545455 * 44 from $stb if $rows != 1 then return -1 endi - if $data00 != 0.204545455 then print expect 0.204545455, actual: $data00 return -1 endi - if $data01 != 9.000000000 then return -1 endi - if $data02 != 9.000000020 then return -1 endi @@ -431,27 +368,21 @@ sql select min(c1) * max(c2) /4, sum(c1) * apercentile(c2, 20), apercentile(c4, if $rows != 1 then return -1 endi - if $data00 != 0.000000000 then return -1 endi - if $data01 != 225000.000000000 then return -1 endi - if $data02 != 8.077777778 then return -1 endi - if $data03 != NULL then return -1 endi - if $data04 != 0.444444444 then return -1 endi - if $data05 != 450000.000000000 then return -1 endi @@ -487,35 +418,29 @@ sql_error select top(c1, 99) - bottom(c1, 99) from $stb sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $stb # error case, ts/bool/binary/nchar not support arithmetic expression -sql select first(c7)*12 from $stb -sql select last(c8)/55 from $stb -sql_error select last_row(c9) + last_row(c8) from $stb +sql select first(c7)*12 from $stb +sql select last(c8)/55 from $stb +sql select last_row(c9) + last_row(c8) from $stb # arithmetic expression in join [d.7]=============================================================== - # arithmetic expression in union [d.8]=============================================================== - # arithmetic expression in group by [d.9]=============================================================== # in group by tag -sql select avg(c4)*99 from $stb group by t1 +sql select avg(c4)*99, t1 from $stb group by t1 order by t1 if $rows != 10 then return -1 endi - if $data00 != 445.500000000 then return -1 endi - if $data01 != 0 then return -1 endi - if $data90 != 445.500000000 then return -1 endi - if $data91 != 9 then return -1 endi @@ -550,22 +475,19 @@ endi # return -1 #endi # -sql_error select first(c6) - last(c6) *12 / count(*) from $stb group by c3; +sql select first(c6) - last(c6) *12 / count(*) from $stb group by c3; -sql select first(c6) - last(c6) *12 / count(*) from $stb group by c5; -if $rows != 10 then +sql select first(c6) - last(c6) *12 / count(*) from $stb group by c5 order by c5; +if $rows != 11 then return -1 endi - -if $data00 != 0.000000000 then +if $data10 != 0.000000000 then return -1 endi - -if $data10 != 0.997600000 then +if $data20 != 0.997600000 then return -1 endi - -if $data90 != 8.978400000 then +if $data90 != 7.980800000 then return -1 endi @@ -574,7 +496,6 @@ sql select first(c6) - sum(c6) + 12 from $stb limit 12 offset 0; if $rows != 1 then return -1 endi - if $data00 != -449988.000000000 then return -1 endi @@ -604,10 +525,8 @@ sql_error select first(c1) from $stb fill(value, 20); # constant column. [d.13]=============================================================== - # column value filter [d.14]=============================================================== - # tag filter. [d.15]=============================================================== sql select sum(c2)+99 from $stb where t1=12; @@ -633,7 +552,6 @@ sql select avg(c2)*count(c2), sum(c3)-first(c3), last(c4)+9 from $stb interval(1 if $rows != 10000 then return -1 endi - if $data00 != @18-09-17 09:00:00.000@ then return -1 endi @@ -645,11 +563,9 @@ sql_error select first(c7)- last(c1) from $tb interval(2y) # first/last query [d.19]=============================================================== - # multiple retrieve [d.20]=============================================================== sql select c2-c2 from $tb - sql select first(c1)-last(c1), spread(c2), max(c3) - min(c3), avg(c4)*count(c4) from $tb diff --git a/tests/script/tsim/parser/columnValue_bigint.sim b/tests/script/tsim/parser/columnValue_bigint.sim index 0d89d9e61c..8841418ed3 100644 --- a/tests/script/tsim/parser/columnValue_bigint.sim +++ b/tests/script/tsim/parser/columnValue_bigint.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/tsim/parser/columnValue_bool.sim b/tests/script/tsim/parser/columnValue_bool.sim index 3a7dcab265..3e8c408e13 100644 --- a/tests/script/tsim/parser/columnValue_bool.sim +++ b/tests/script/tsim/parser/columnValue_bool.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/tsim/parser/columnValue_double.sim b/tests/script/tsim/parser/columnValue_double.sim index e5ba89158b..c7ba7b0048 100644 --- a/tests/script/tsim/parser/columnValue_double.sim +++ b/tests/script/tsim/parser/columnValue_double.sim @@ -1,5 +1,3 @@ -#### -sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/tsim/parser/columnValue_float.sim b/tests/script/tsim/parser/columnValue_float.sim index 1832f7f847..8fca0d4671 100644 --- a/tests/script/tsim/parser/columnValue_float.sim +++ b/tests/script/tsim/parser/columnValue_float.sim @@ -1,5 +1,3 @@ -#### -sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 5536293ed2..66be28ef89 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/tsim/parser/columnValue_smallint.sim b/tests/script/tsim/parser/columnValue_smallint.sim index 98b83fd0e1..6608b6cea4 100644 --- a/tests/script/tsim/parser/columnValue_smallint.sim +++ b/tests/script/tsim/parser/columnValue_smallint.sim @@ -1,5 +1,3 @@ -sleep 100 -sql connect sql create database if not exists db sql use db diff --git a/tests/script/tsim/parser/columnValue_tinyint.sim b/tests/script/tsim/parser/columnValue_tinyint.sim index 7b0cad23df..67c0f998ca 100644 --- a/tests/script/tsim/parser/columnValue_tinyint.sim +++ b/tests/script/tsim/parser/columnValue_tinyint.sim @@ -1,5 +1,3 @@ -sleep 100 -sql connect sql create database if not exists db sql use db diff --git a/tests/script/tsim/parser/columnValue_unsign.sim b/tests/script/tsim/parser/columnValue_unsign.sim index 8e44ccb5fa..4b8baf10cd 100644 --- a/tests/script/tsim/parser/columnValue_unsign.sim +++ b/tests/script/tsim/parser/columnValue_unsign.sim @@ -1,5 +1,3 @@ -sleep 100 -sql connect sql create database if not exists db sql use db diff --git a/tests/script/tsim/parser/fill.sim b/tests/script/tsim/parser/fill.sim index 698314fa36..396bdd1e56 100644 --- a/tests/script/tsim/parser/fill.sim +++ b/tests/script/tsim/parser/fill.sim @@ -960,6 +960,7 @@ endi sql select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k) from tm0 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 4:2:15' interval(500a) fill(value, 99,91,90,89,88,87,86,85) ; if $rows != 21749 then + print expect 21749, actual: $rows return -1 endi diff --git a/tests/script/tsim/parser/first_last_query.sim b/tests/script/tsim/parser/first_last_query.sim index a001b929c3..adad554fb2 100644 --- a/tests/script/tsim/parser/first_last_query.sim +++ b/tests/script/tsim/parser/first_last_query.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = first_db diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 7dd66bedb0..7927715988 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -278,7 +278,6 @@ sql create stable td6086st(ts timestamp, d double) tags(t nchar(50)); sql create table td6086ct1 using td6086st tags("ct1"); sql create table td6086ct2 using td6086st tags("ct2"); -return sql SELECT LAST(d),t FROM td6086st WHERE tbname in ('td6086ct1', 'td6086ct2') and ts>="2019-07-30 00:00:00" and ts<="2021-08-31 00:00:00" partition BY tbname interval(1800s) fill(prev); print ==================> td-2624 diff --git a/tests/script/tsim/parser/gendata.sh b/tests/script/tsim/parser/gendata.sh index b2074147ca..eb3676729e 100755 --- a/tests/script/tsim/parser/gendata.sh +++ b/tests/script/tsim/parser/gendata.sh @@ -3,6 +3,6 @@ Cur_Dir=$(pwd) echo $Cur_Dir -echo "'2020-1-1 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql -echo "'2020-1-2 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql -echo "'2020-1-3 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql +echo "'2020-1-1 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> /tmp/data.sql +echo "'2020-1-2 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> /tmp/data.sql +echo "'2020-1-3 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> /tmp/data.sql diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim index 1edc99624a..7fe91f5100 100644 --- a/tests/script/tsim/parser/groupby-basic.sim +++ b/tests/script/tsim/parser/groupby-basic.sim @@ -3,25 +3,6 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - $dbPrefix = group_db $tbPrefix = group_tb $mtPrefix = group_mt @@ -80,8 +61,6 @@ while $i < $tbNum $tstart = 1640966400000 endw -sleep 100 - $i1 = 1 $i2 = 0 @@ -752,12 +731,7 @@ sql insert into tm1 values('2020-2-1 1:1:1', 2, 10); sql insert into tm1 values('2020-2-1 1:1:2', 2, 20); system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 100 system sh/exec.sh -n dnode1 -s start -sleep 100 - -sql connect -sleep 100 sql use group_db0; print =========================>TD-4894 diff --git a/tests/script/tsim/parser/import_file.sim b/tests/script/tsim/parser/import_file.sim index 5c778a5875..0250af16b3 100644 --- a/tests/script/tsim/parser/import_file.sim +++ b/tests/script/tsim/parser/import_file.sim @@ -7,7 +7,7 @@ sql drop database if exists indb sql create database if not exists indb sql use indb -$inFileName = '~/data.csv' +$inFileName = '/tmp/data.csv' $numOfRows = 10000 system tsim/parser/gendata.sh @@ -16,8 +16,8 @@ sql create table stbx (ts TIMESTAMP, collect_area NCHAR(12), device_id BINARY(16 sql create table tbx (ts TIMESTAMP, collect_area NCHAR(12), device_id BINARY(16), imsi BINARY(16), imei BINARY(16), mdn BINARY(10), net_type BINARY(4), mno NCHAR(4), province NCHAR(10), city NCHAR(16), alarm BINARY(2)) print ====== create tables success, starting insert data -sql insert into tbx file '~/data.sql' -sql import into tbx file '~/data.sql' +sql insert into tbx file '/tmp/data.sql' +sql import into tbx file '/tmp/data.sql' sql select count(*) from tbx if $rows != 1 then @@ -31,8 +31,8 @@ endi sql drop table tbx; -sql insert into tbx using stbx tags(1,'abc') file '~/data.sql'; -sql insert into tbx using stbx tags(1,'abc') file '~/data.sql'; +sql insert into tbx using stbx tags(1,'abc') file '/tmp/data.sql'; +sql insert into tbx using stbx tags(1,'abc') file '/tmp/data.sql'; sql select count(*) from tbx if $rows != 1 then @@ -44,7 +44,7 @@ if $data00 != 3 then endi sql drop table tbx; -sql insert into tbx using stbx(b) tags('abcf') file '~/data.sql'; +sql insert into tbx using stbx(b) tags('abcf') file '/tmp/data.sql'; sql select ts,a,b from tbx; if $rows != 3 then @@ -64,6 +64,6 @@ if $data02 != @abcf@ then return -1 endi -system rm -f ~/data.sql +system rm -f /tmp/data.sql system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/interp_test.sim b/tests/script/tsim/parser/interp_test.sim index 0bdf97a957..5c94b72b43 100644 --- a/tests/script/tsim/parser/interp_test.sim +++ b/tests/script/tsim/parser/interp_test.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = intp_db diff --git a/tests/script/tsim/parser/join.sim b/tests/script/tsim/parser/join.sim index c052e24856..269d4ca254 100644 --- a/tests/script/tsim/parser/join.sim +++ b/tests/script/tsim/parser/join.sim @@ -54,8 +54,6 @@ while $i < $tbNum $tstart = 100000 endw -sleep 100 - $tstart = 100000 $mt = $mtPrefix . 1 . $i sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12), t3 int) @@ -99,8 +97,6 @@ while $i < $tbNum $tstart = 100000 endw -sleep 100 - $i1 = 1 $i2 = 0 diff --git a/tests/script/tsim/parser/limit.sim b/tests/script/tsim/parser/limit.sim index f4a23697cd..bbee7a931d 100644 --- a/tests/script/tsim/parser/limit.sim +++ b/tests/script/tsim/parser/limit.sim @@ -62,11 +62,8 @@ run tsim/parser/limit_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -sql connect -sleep 100 run tsim/parser/limit_tb.sim run tsim/parser/limit_stb.sim diff --git a/tests/script/tsim/parser/limit1.sim b/tests/script/tsim/parser/limit1.sim index b6d0629c8f..6fbae4acd7 100644 --- a/tests/script/tsim/parser/limit1.sim +++ b/tests/script/tsim/parser/limit1.sim @@ -18,7 +18,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db cache 16 +sql create database $db print ====== create tables sql use $db sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) diff --git a/tests/script/tsim/parser/limit1_tb.sim b/tests/script/tsim/parser/limit1_tb.sim index 5a7c1bc201..82914f3011 100644 --- a/tests/script/tsim/parser/limit1_tb.sim +++ b/tests/script/tsim/parser/limit1_tb.sim @@ -370,7 +370,8 @@ sql select top(c1, 1) from $tb where ts >= $ts0 and ts <= $tsu limit 5 offset 1 if $rows != 0 then return -1 endi -sql select top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu limit 3 offset 1 + +sql select ts, top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts limit 3 offset 1 if $rows != 3 then return -1 endi @@ -392,6 +393,7 @@ endi if $data21 != 9 then return -1 endi + sql select top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu limit 3 offset 5 if $rows != 0 then return -1 @@ -401,7 +403,8 @@ sql select bottom(c1, 1) from $tb where ts >= $ts0 and ts <= $tsu limit 5 offset if $rows != 0 then return -1 endi -sql select bottom(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu limit 3 offset 1 + +sql select ts, bottom(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts limit 3 offset 1 if $rows != 3 then return -1 endi diff --git a/tests/script/tsim/parser/limit2.sim b/tests/script/tsim/parser/limit2.sim index ca308fa6e7..3951ab4e08 100644 --- a/tests/script/tsim/parser/limit2.sim +++ b/tests/script/tsim/parser/limit2.sim @@ -61,11 +61,10 @@ while $i < $halfNum endw print ====== tables created -#run tsim/parser/limit2_query.sim +run tsim/parser/limit2_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 100 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/tsim/parser/limit2_query.sim b/tests/script/tsim/parser/limit2_query.sim index 8a2da7988d..0200cf7d43 100644 --- a/tests/script/tsim/parser/limit2_query.sim +++ b/tests/script/tsim/parser/limit2_query.sim @@ -27,8 +27,8 @@ print select count(*) from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, sql select count(*), t1, t2, t3, t4, t5, t6 from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0 $val = $tbNum - 3 -print $rows $val -if $rows != $val then +print $rows +if $rows != 1 then return -1 endi if $data00 != $rowNum then @@ -51,7 +51,7 @@ if $data05 != 2 then return -1 endi -sql select count(*) from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 0 +sql select count(*), t3, t4 from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0 if $rows != 2 then return -1 endi @@ -70,15 +70,17 @@ endi if $data12 != 3 then return -1 endi + sql select count(*) from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1 -if $rows != 0 then +if $rows != 1 then return -1 endi + ## TBASE-348 sql_error select count(*) from $stb where t1 like 1 ##### aggregation on tb + where + fill + limit offset -sql select max(c1) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1, -2) limit 10 offset 1 +sql select _wstart, max(c1) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1, -2) limit 10 offset 1 if $rows != 10 then return -1 endi diff --git a/tests/script/tsim/parser/limit_tb.sim b/tests/script/tsim/parser/limit_tb.sim index d0d14c5bfc..f8a1e7ac6a 100644 --- a/tests/script/tsim/parser/limit_tb.sim +++ b/tests/script/tsim/parser/limit_tb.sim @@ -358,8 +358,8 @@ endi print ========> TD-6017 sql select * from (select ts, top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts desc limit 3 offset 1) -sql select top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts desc limit 3 offset 1 -print select top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts desc limit 3 offset 1 +sql select ts, top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts desc limit 3 offset 1 +print select ts, top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts desc limit 3 offset 1 print $data00 $data01 print $data10 $data11 print $data20 $data21 @@ -386,7 +386,7 @@ if $data21 != 6 then return -1 endi -sql select top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts asc limit 3 offset 1 +sql select ts, top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts asc limit 3 offset 1 if $rows != 3 then return -1 endi @@ -418,7 +418,7 @@ sql select bottom(c1, 1) from $tb where ts >= $ts0 and ts <= $tsu limit 5 offset if $rows != 0 then return -1 endi -sql select bottom(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu limit 3 offset 1 +sql select ts, bottom(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts limit 3 offset 1 if $rows != 3 then return -1 endi @@ -482,7 +482,7 @@ endi if $data41 != 4 then return -1 endi -sql select max(c1), max(c2), max(c3), max(c4), max(c5), max(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) limit 5 offset 1 +sql select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5), max(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) limit 5 offset 1 if $rows != 5 then return -1 endi @@ -518,7 +518,7 @@ if $data41 != 5 then endi ## TBASE-334 -sql select max(c1), max(c2), max(c3), max(c4), max(c5), max(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(30m) limit 2 offset 1 +sql select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5), max(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(30m) limit 2 offset 1 if $rows != 2 then return -1 endi @@ -634,7 +634,8 @@ sql select stddev(c1), stddev(c2), stddev(c3), stddev(c4), stddev(c5), stddev(c6 if $rows != 0 then return -1 endi -sql select stddev(c1), stddev(c2), stddev(c3), stddev(c4), stddev(c5), stddev(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(30m) limit 5 offset 1 + +sql select _wstart, stddev(c1), stddev(c2), stddev(c3), stddev(c4), stddev(c5), stddev(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(30m) limit 5 offset 1 if $rows != 3 then return -1 endi @@ -670,6 +671,7 @@ endi if $data31 != 3 then return -1 endi + sql select count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m) limit 5 offset 1 if $rows != 3 then return -1 @@ -707,7 +709,8 @@ sql select first(c1), first(c2), first(c3), first(c4), first(c5), first(c6) from if $rows != 0 then return -1 endi -sql select first(c1), first(c2), first(c3), first(c4), first(c5), first(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(30m) limit 3 offset 1 + +sql select _wstart, first(c1), first(c2), first(c3), first(c4), first(c5), first(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(30m) limit 3 offset 1 if $rows != 3 then return -1 endi @@ -721,7 +724,6 @@ if $data23 != 9.00000 then return -1 endi - sql select last(c1), last(c2), last(c3), last(c4), last(c5), last(c6) from $tb where ts >= $ts0 and ts <= $tsu limit 5 offset 1 if $rows != 0 then return -1 diff --git a/tests/script/tsim/parser/projection_limit_offset.sim b/tests/script/tsim/parser/projection_limit_offset.sim index 5a96af2b3e..669ff3a179 100644 --- a/tests/script/tsim/parser/projection_limit_offset.sim +++ b/tests/script/tsim/parser/projection_limit_offset.sim @@ -62,8 +62,6 @@ while $i < $half $tstart = 100000 endw -sleep 100 - $i1 = 1 $i2 = 0 diff --git a/tests/script/tsim/parser/single_row_in_tb.sim b/tests/script/tsim/parser/single_row_in_tb.sim index 59a0552809..1bd53ad24e 100644 --- a/tests/script/tsim/parser/single_row_in_tb.sim +++ b/tests/script/tsim/parser/single_row_in_tb.sim @@ -28,7 +28,6 @@ run tsim/parser/single_row_in_tb_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/tsim/parser/slimit1.sim b/tests/script/tsim/parser/slimit1.sim index bb12bc32f1..19e69ba541 100644 --- a/tests/script/tsim/parser/slimit1.sim +++ b/tests/script/tsim/parser/slimit1.sim @@ -52,11 +52,9 @@ run tsim/parser/slimit1_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 100 run tsim/parser/slimit1_query.sim diff --git a/tests/script/tsim/parser/slimit1_query.sim b/tests/script/tsim/parser/slimit1_query.sim index 8e3a61bee6..7c7d255fe0 100644 --- a/tests/script/tsim/parser/slimit1_query.sim +++ b/tests/script/tsim/parser/slimit1_query.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = slm_alt_tg_db diff --git a/tests/script/tsim/parser/slimit_alter_tags.sim b/tests/script/tsim/parser/slimit_alter_tags.sim index 53af0f3e84..e12a14e43a 100644 --- a/tests/script/tsim/parser/slimit_alter_tags.sim +++ b/tests/script/tsim/parser/slimit_alter_tags.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = slm_alt_tg_db @@ -93,7 +89,6 @@ if $data02 != tb0 then return -1 endi -sleep 500 sql reset query cache sql select count(*), first(ts) from stb group by tg_added order by tg_added asc slimit 5 soffset 3 if $rows != 5 then @@ -171,11 +166,8 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -sql connect -sleep 100 sql use $db ### repeat above queries diff --git a/tests/script/tsim/parser/slimit_query.sim b/tests/script/tsim/parser/slimit_query.sim index 0a793f0611..7ee7c6355b 100644 --- a/tests/script/tsim/parser/slimit_query.sim +++ b/tests/script/tsim/parser/slimit_query.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = slm_db diff --git a/tests/script/tsim/parser/union.sim b/tests/script/tsim/parser/union.sim index 95150616d1..8bafeff444 100644 --- a/tests/script/tsim/parser/union.sim +++ b/tests/script/tsim/parser/union.sim @@ -95,9 +95,6 @@ while $i < $tbNum $j = $j + 1 endw -print sleep 1sec. -sleep 100 - $i = 1 $tb = $tbPrefix . $i diff --git a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim index 868207c80b..e3b38d415c 100644 --- a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim +++ b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim @@ -179,4 +179,6 @@ if $data05 != 30.000000000 then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT + diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 4364b56d44..8ebadbfb50 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -367,7 +367,7 @@ if $data32 != 8 then endi #$loop_all = 0 -#looptest: +#=looptest: sql drop database IF EXISTS test2; sql drop stream IF EXISTS streams21; @@ -511,6 +511,6 @@ endi $loop_all = $loop_all + 1 print ============loop_all=$loop_all -#goto looptest +#=goto looptest system sh/stop_dnodes.sh \ No newline at end of file diff --git a/tests/script/tsim/tag/column.sim b/tests/script/tsim/tag/column.sim index c73999230f..f4e2e5b7fe 100644 --- a/tests/script/tsim/tag/column.sim +++ b/tests/script/tsim/tag/column.sim @@ -31,7 +31,7 @@ sql create table $tb using $mt tags( 0, '0' ) $i = 1 $tb = $tbPrefix . $i -sql create table $tb using $mt tags( 1, 1 ) +sql create table $tb using $mt tags( 1, '1' ) $i = 2 $tb = $tbPrefix . $i @@ -39,7 +39,7 @@ sql create table $tb using $mt tags( '2', '2' ) $i = 3 $tb = $tbPrefix . $i -sql create table $tb using $mt tags( '3', 3 ) +sql create table $tb using $mt tags( '3', '3' ) sql show tables if $rows != 4 then @@ -54,7 +54,7 @@ sql insert into $tb values(now, 0, '0') $i = 1 $tb = $tbPrefix . $i -sql insert into $tb values(now, 1, 1 ) +sql insert into $tb values(now, 1, '1' ) $i = 2 $tb = $tbPrefix . $i @@ -62,7 +62,7 @@ sql insert into $tb values(now, '2', '2') $i = 3 $tb = $tbPrefix . $i -sql insert into $tb values(now, '3', 3) +sql insert into $tb values(now, '3', '3') print =============== step4 sql select * from $mt where tgcol2 = '1' diff --git a/tests/script/tsim/tag/commit.sim b/tests/script/tsim/tag/commit.sim index 95ab2dbc7f..1a47fd838f 100644 --- a/tests/script/tsim/tag/commit.sim +++ b/tests/script/tsim/tag/commit.sim @@ -249,8 +249,8 @@ sql alter table $mt add tag tgcol6 binary(10) sql reset query cache sql alter table $tb set tag tgcol4=false -sql alter table $tb set tag tgcol5=5 -sql alter table $tb set tag tgcol6=6 +sql alter table $tb set tag tgcol5='5' +sql alter table $tb set tag tgcol6='6' sql reset query cache sql select * from $mt where tgcol5 = '5' @@ -321,7 +321,7 @@ if $data04 != 3 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol4 sql alter table $mt drop tag tgcol2 sql alter table $mt drop tag tgcol3 sql alter table $mt add tag tgcol5 bigint @@ -382,14 +382,14 @@ if $data04 != 3 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol4 sql alter table $mt drop tag tgcol2 sql alter table $mt drop tag tgcol3 sql alter table $mt add tag tgcol5 binary(17) sql alter table $mt add tag tgcol6 bool sql reset query cache sql alter table $tb set tag tgcol4=4 -sql alter table $tb set tag tgcol5=5 +sql alter table $tb set tag tgcol5='5' sql alter table $tb set tag tgcol6=1 sql reset query cache @@ -423,7 +423,7 @@ $i = 9 $mt = $mtPrefix . $i $tb = $tbPrefix . $i sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 double, tgcol2 binary(10), tgcol3 binary(10)) -sql create table $tb using $mt tags( 1, 2, '3' ) +sql create table $tb using $mt tags( 1, '2', '3' ) sql insert into $tb values(now, 1) sql select * from $mt where tgcol2 = '2' if $rows != 1 then @@ -442,7 +442,7 @@ if $data04 != 3 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol4 sql alter table $mt drop tag tgcol2 sql alter table $mt drop tag tgcol3 sql alter table $mt add tag tgcol5 bool @@ -506,7 +506,7 @@ if $data05 != 4 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol4 -x step103 +sql alter table $mt rename tag tgcol1 tgcol4 -x step103 return -1 step103: @@ -518,7 +518,7 @@ sql alter table $mt add tag tgcol4 binary(10) sql alter table $mt add tag tgcol5 bool sql reset query cache -sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol4='4' sql alter table $tb set tag tgcol5=false sql reset query cache @@ -580,7 +580,7 @@ if $data06 != 5 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol4 -x step114 +sql alter table $mt rename tag tgcol1 tgcol4 -x step114 return -1 step114: @@ -596,9 +596,9 @@ sql alter table $mt add tag tgcol7 bigint sql alter table $mt add tag tgcol8 smallint sql reset query cache -sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol4='4' sql alter table $tb set tag tgcol5=5 -sql alter table $tb set tag tgcol6=6 +sql alter table $tb set tag tgcol6='6' sql alter table $tb set tag tgcol7=7 sql alter table $tb set tag tgcol8=8 sql reset query cache @@ -685,11 +685,11 @@ sql alter table $mt add tag tgcol5 bigint sql reset query cache sql alter table $tb set tag tgcol1=false -sql alter table $tb set tag tgcol2=5 +sql alter table $tb set tag tgcol2='5' sql alter table $tb set tag tgcol3=4 -sql alter table $tb set tag tgcol4=3 +sql alter table $tb set tag tgcol4='3' sql alter table $tb set tag tgcol5=2 -sql alter table $tb set tag tgcol6=1 +sql alter table $tb set tag tgcol6='1' sql reset query cache sql select * from $mt where tgcol4 = '3' @@ -781,8 +781,8 @@ sql alter table $mt add tag tgcol4 int sql alter table $mt add tag tgcol6 bigint sql reset query cache -sql alter table $tb set tag tgcol1=7 -sql alter table $tb set tag tgcol2=8 +sql alter table $tb set tag tgcol1='7' +sql alter table $tb set tag tgcol2='8' sql alter table $tb set tag tgcol3=9 sql alter table $tb set tag tgcol4=10 sql alter table $tb set tag tgcol5=11 @@ -817,9 +817,7 @@ if $data07 != 12 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 3000 print =============== step1 $i = 0 diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index 91869cb012..bdb8c095e6 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -28,7 +28,7 @@ from util.dnodes import * class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) def csum_query_form(self, col="c1", alias="", table_expr="t1", condition=""): diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index a82c7bfe1a..ca270932b1 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -136,9 +136,14 @@ class TDTestCase: tdSql.query("SELECT _wstart,model,fleet,count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m) fill(value,0)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1 partition by model, fleet interval(1d) ;") + tdSql.query("SELECT _wstart,model,fleet,count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m) ) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1 partition by model, fleet interval(1d) ;") + + # 9. breakdown-frequency # NULL ---count(NULL)=0 expect count(NULL)= 100 tdSql.query("SELECT model,state_changed,count(state_changed) FROM (SELECT model,diff(broken_down) AS state_changed FROM (SELECT _wstart,model,cast(cast(floor(2*(sum(nzs)/count(nzs))) as bool) as int) AS broken_down FROM (SELECT ts,model, cast(cast(status as bool) as int) AS nzs FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' ) WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition BY model interval(10m)) partition BY model) where model is null partition BY model,state_changed ") + parRows=tdSql.queryRows + assert parRows != 0 , "query result is wrong" tdSql.query(" SELECT model,state_changed,count(state_changed) FROM (SELECT model,diff(broken_down) AS state_changed FROM (SELECT _wstart,model,cast(cast(floor(2*(sum(nzs)/count(nzs))) as bool) as int) AS broken_down FROM (SELECT ts,model, cast(cast(status as bool) as int) AS nzs FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' ) WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition BY model interval(10m)) partition BY model) where state_changed =1 partition BY model,state_changed ;") diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py index 7ce716c483..e4df9df4ea 100644 --- a/tests/system-test/6-cluster/5dnode2mnode.py +++ b/tests/system-test/6-cluster/5dnode2mnode.py @@ -127,52 +127,12 @@ class TDTestCase: tdLog.notice(" The election still succeeds when leader of both mnodes are killed ") except Exception: pass - tdSql.error("create user user1 pass '123';") + # tdSql.error("create user user1 pass '123';") tdLog.info("start leader") tdDnodes[0].starttaosd() if clusterComCheck.checkMnodeStatus(2) : print("both mnodes are ready") - # # fisrt drop follower of mnode - # BUG - # tdSql.execute("drop mnode on dnode 2") - # tdSql.query("show mnodes;") - # tdSql.checkRows(1) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,2,'leader') - # tdSql.checkData(0,3,'ready') - - # tdSql.execute("create mnode on dnode 2") - # count=0 - # while count < 10: - # time.sleep(1) - # tdSql.query("show mnodes;") - # tdSql.checkRows(2) - # if tdSql.queryResult[0][2]=='leader' : - # if tdSql.queryResult[1][2]=='follower': - # print("two mnodes is ready") - # break - # count+=1 - # else: - # print("two mnodes is not ready in 10s ") - - # tdSql.query("show mnodes;") - # tdSql.checkRows(2) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,2,'leader') - # tdSql.checkData(0,3,'ready') - # tdSql.checkData(1,1,'%s:6130'%self.host) - # tdSql.checkData(1,2,'follower') - # tdSql.checkData(2,3,'ready') - - - def getConnection(self, dnode): - host = dnode.cfgDict["fqdn"] - port = dnode.cfgDict["serverPort"] - config_dir = dnode.cfgDir - return taos.connect(host=host, port=int(port), config=config_dir) - - def run(self): self.five_dnode_two_mnode() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index 8971a51ef3..3c0e479030 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -93,7 +93,7 @@ class TDTestCase: def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db', - 'dbNumbers': 20, + 'dbNumbers': 8, 'dropFlag': 1, 'event': '', 'vgroups': 4, @@ -183,15 +183,22 @@ class TDTestCase: for tr in threads: tr.join() + tdLog.info("check dnode number:") clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkDbRows(allDbNumbers) - for i in range(restartNumbers): - clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + tdSql.query("show databases") + tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers)) + + # tdLog.info("check DB Rows:") + # clusterComCheck.checkDbRows(allDbNumbers) + # tdLog.info("check DB Status on by on") + # for i in range(restartNumbers): + # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=10,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py index 6db1a9fddd..f685ef2f1a 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py @@ -65,40 +65,15 @@ class TDTestCase: self._async_raise(thread.ident, SystemExit) - def insertData(self,countstart,countstop): - # fisrt add data : db\stable\childtable\general table - - for couti in range(countstart,countstop): - tdLog.debug("drop database if exists db%d" %couti) - tdSql.execute("drop database if exists db%d" %couti) - print("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("use db%d" %couti) - tdSql.execute( - '''create table stb1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - tags (t1 int) - ''' - ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') - - def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'db', + paraDict = {'dbName': 'db0_0', 'dropFlag': 1, 'event': '', 'vgroups': 4, 'replica': 1, 'stbName': 'stb', - 'stbNumbers': 100, + 'stbNumbers': 80, 'colPrefix': 'c', 'tagPrefix': 't', 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], @@ -188,7 +163,10 @@ class TDTestCase: tdSql.execute("use %s" %(paraDict["dbName"])) tdSql.query("show stables") - tdSql.checkRows(allStbNumbers) + tdLog.debug("we find %d stables but exepect to create %d stables "%(tdSql.queryRows,allStbNumbers)) + # # tdLog.info("check Stable Rows:") + # tdSql.checkRows(allStbNumbers) + def run(self): diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py index d9871bb35f..1788f24c3f 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -68,7 +68,7 @@ class TDTestCase: def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db', - 'dbNumbers': 10, + 'dbNumbers': 8, 'dropFlag': 1, 'event': '', 'vgroups': 2, diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py index f7d62857f9..3a10427664 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py @@ -98,7 +98,7 @@ class TDTestCase: 'vgroups': 4, 'replica': 1, 'stbName': 'stb', - 'stbNumbers': 100, + 'stbNumbers': 80, 'colPrefix': 'c', 'tagPrefix': 't', 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], @@ -142,7 +142,8 @@ class TDTestCase: threads=[] for i in range(restartNumbers): stableName= '%s%d'%(paraDict['stbName'],i) - threads.append(threading.Thread(target=clusterComCreate.create_stables, args=(tdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']))) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.create_stables, args=(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']))) for tr in threads: tr.start() @@ -190,6 +191,7 @@ class TDTestCase: tdSql.execute("use %s" %(paraDict["dbName"])) tdSql.query("show stables") tdLog.debug("we find %d stables but exepect to create %d stables "%(tdSql.queryRows,allStbNumbers)) + # # tdLog.info("check Stable Rows:") # tdSql.checkRows(allStbNumbers) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py index a9c8afd741..da32d1b4a8 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py @@ -159,19 +159,19 @@ class TDTestCase: for tr in threads: tr.join() clusterComCheck.checkDnodes(dnodeNumbers) - tdSql.query("show databases") - tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers)) + # tdSql.query("show databases") + # tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers)) - # # tdLog.info("check DB Rows:") - # clusterComCheck.checkDbRows(allDbNumbers) - # # tdLog.info("check DB Status on by on") + # tdLog.info("check DB Rows:") + clusterComCheck.checkDbRows(allDbNumbers) + # tdLog.info("check DB Status on by on") # for i in range(restartNumbers): # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=15,stopRole='vnode') + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=10,stopRole='vnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py index 128dc10b37..5c9e3587c4 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py @@ -98,7 +98,7 @@ class TDTestCase: 'vgroups': 4, 'replica': 1, 'stbName': 'stb', - 'stbNumbers': 100, + 'stbNumbers': 80, 'colPrefix': 'c', 'tagPrefix': 't', 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], @@ -192,6 +192,8 @@ class TDTestCase: tdSql.execute("use %s" %(paraDict["dbName"])) tdSql.query("show stables") + tdLog.debug("we find %d stables but exepect to create %d stables "%(tdSql.queryRows,allStbNumbers)) + # # tdLog.info("check Stable Rows:") tdSql.checkRows(allStbNumbers) diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index f932e5537e..46e7771079 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -111,7 +111,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(5,3,1) + self.fiveDnodeThreeMnode(dnodenumbers=5,mnodeNums=3,restartNumber=1) def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py b/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py new file mode 100644 index 0000000000..954e1ae003 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py @@ -0,0 +1,119 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +from test import tdDnodes +sys.path.append("./6-cluster") + +from clusterCommonCreate import * +from clusterCommonCheck import * +import time +import socket +import subprocess +from multiprocessing import Process + + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + dnodenumbers=int(dnodenumbers) + mnodeNums=int(mnodeNums) + dbNumbers = int(dnodenumbers * restartNumber) + + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + # print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodenumbers) + # restart all taosd + tdDnodes=cluster.dnodes + + tdDnodes[1].stoptaosd() + tdDnodes[2].stoptaosd() + + tdLog.info("check whether 2 mnode status is offline") + clusterComCheck.check3mnode2off() + # tdSql.error("create user user1 pass '123';") + + tdLog.info("start two follower") + tdDnodes[1].starttaosd() + tdDnodes[2].starttaosd() + + clusterComCheck.checkMnodeStatus(3) + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodenumbers=5,mnodeNums=3,restartNumber=1) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index 203d756353..294f7cf61c 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -63,6 +63,7 @@ class ClusterComCheck: count=0 while count < 5: tdSql.query("show databases;") + count+=1 if tdSql.checkRows(dbNumbers+2): tdLog.success("we find %d databases and expect %d in clusters! " %(tdSql.queryRows,dbNumbers+2)) return True @@ -204,7 +205,24 @@ class ClusterComCheck: tdLog.debug(tdSql.queryResult) tdLog.exit("stop mnodes on dnode %d failed in 10s ") - + def check3mnode2off(self,mnodeNums=3): + count=0 + while count < 10: + time.sleep(1) + tdSql.query("show mnodes;") + if tdSql.checkRows(mnodeNums) : + tdLog.success("cluster has %d mnodes" %self.mnodeNums ) + else: + tdLog.exit("mnode number is correct") + if tdSql.queryResult[0][2]=='leader' : + if tdSql.queryResult[1][2]=='offline' and tdSql.queryResult[1][3]== 'ready' : + if tdSql.queryResult[2][2]=='offline' and tdSql.queryResult[2][3]== 'ready' : + tdLog.success("stop mnodes of follower on dnode successfully in 10s") + return True + count+=1 + else: + tdLog.debug(tdSql.queryResult) + tdLog.exit("stop mnodes on dnode %d failed in 10s ") diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py new file mode 100644 index 0000000000..e6192ba313 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py @@ -0,0 +1,138 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def getConnection(self, dnode): + host = dnode.cfgDict["fqdn"] + port = dnode.cfgDict["serverPort"] + config_dir = dnode.cfgDir + return taos.connect(host=host, port=int(port), config=config_dir) + + + def run(self): + self.check_setup_cluster_status() + self.create_db_check_vgroups() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py new file mode 100644 index 0000000000..d5fef08945 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py @@ -0,0 +1,179 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.db_name ='testdb' + self.replica = 1 + self.vgroups = 2 + self.tb_nums = 10 + self.row_nums = 100 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_db_replica_1_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(tb_nums): + sub_tbname = "sub_tb_{}".format(i) + tdSql.execute("create table {} using stb1 tags({})".format(sub_tbname,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + 1000*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== create database {} and insert rows execute end =====".format(dbname)) + + def check_insert_status(self, dbname, tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.query("select count(*) from {}.{}".format(dbname,'stb1')) + tdSql.checkData(0 , 0 , tb_nums*row_nums) + tdSql.query("select distinct tbname from {}.{}".format(dbname,'stb1')) + tdSql.checkRows(tb_nums) + + def run(self): + self.check_setup_cluster_status() + self.create_db_check_vgroups() + self.create_db_replica_1_insertdatas(self.db_name , self.replica , self.vgroups , self.tb_nums , self.row_nums) + self.check_insert_status(self.db_name , self.tb_nums , self.row_nums) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py new file mode 100644 index 0000000000..00bd8a48d9 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py @@ -0,0 +1,179 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 2 + self.tb_nums = 10 + self.row_nums = 100 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_db_replica_3_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(tb_nums): + sub_tbname = "sub_tb_{}".format(i) + tdSql.execute("create table {} using stb1 tags({})".format(sub_tbname,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + 1000*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== create database {} and insert rows execute end =====".format(dbname)) + + def check_insert_status(self, dbname, tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.query("select count(*) from {}.{}".format(dbname,'stb1')) + tdSql.checkData(0 , 0 , tb_nums*row_nums) + tdSql.query("select distinct tbname from {}.{}".format(dbname,'stb1')) + tdSql.checkRows(tb_nums) + + def run(self): + self.check_setup_cluster_status() + self.create_db_check_vgroups() + self.create_db_replica_3_insertdatas(self.db_name , self.replica , self.vgroups , self.tb_nums , self.row_nums) + self.check_insert_status(self.db_name , self.tb_nums , self.row_nums) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py new file mode 100644 index 0000000000..22d3ba6dbc --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py @@ -0,0 +1,496 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import datetime +import inspect +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 5 + self.current_thread = None + self.max_restart_time = 10 + self.try_check_times = 10 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups; '".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 + def _get_stop_dnode_id(self,dbname): + tdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = tdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='follower': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin stop dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].stoptaosd() + + self.wait_stop_dnode_OK() + + # append rows of stablename when dnode stop + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin start dnode + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + start = time.time() + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + tdDnodes[self.stop_dnode_id-1].stoptaosd() + self.wait_stop_dnode_OK() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + self.current_thread.join() + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + self.sync_run_case() + # self.unsync_run_case() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py new file mode 100644 index 0000000000..28198b9529 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py @@ -0,0 +1,497 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import datetime +import inspect +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 5 + self.current_thread = None + self.max_restart_time = 10 + self.try_check_times = 10 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups; '".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 + + def _get_stop_dnode_id(self,dbname): + tdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = tdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='follower': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin stop dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].stoptaosd() + + self.wait_stop_dnode_OK() + + # append rows of stablename when dnode stop + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin start dnode + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + start = time.time() + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + tdDnodes[self.stop_dnode_id-1].stoptaosd() + self.wait_stop_dnode_OK() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + self.current_thread.join() + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + # self.sync_run_case() + self.unsync_run_case() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py new file mode 100644 index 0000000000..83faba4578 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py @@ -0,0 +1,521 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import datetime +import inspect +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 5 + self.current_thread = None + self.max_restart_time = 10 + self.try_check_times = 10 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups; '".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 + + def _get_stop_dnode_id(self,dbname): + tdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = tdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='follower': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin stop dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].forcestop() + + self.wait_stop_dnode_OK() + + # append rows of stablename when dnode stop + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin start dnode + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + start = time.time() + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + while not self.stop_dnode_id: + time.sleep(0.5) + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + + # force stop taosd by kill -9 + self.force_stop_dnode(self.stop_dnode_id) + self.wait_stop_dnode_OK() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + self.current_thread.join() + + def force_stop_dnode(self, dnode_id ): + + tdSql.query("show dnodes") + port = None + for dnode_info in tdSql.queryResult: + if dnode_id == dnode_info[0]: + port = dnode_info[1].split(":")[-1] + break + else: + continue + if port: + tdLog.info(" ==== dnode {} will be force stop by kill -9 ====".format(dnode_id)) + psCmd = '''netstat -anp|grep -w LISTEN|grep -w %s |grep -o "LISTEN.*"|awk '{print $2}'|cut -d/ -f1|head -n1''' %(port) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + ps_kill_taosd = ''' kill -9 {} '''.format(processID) + # print(ps_kill_taosd) + os.system(ps_kill_taosd) + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + # self.sync_run_case() + self.unsync_run_case() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py new file mode 100644 index 0000000000..c0aafa7978 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -0,0 +1,558 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 10 + self.current_thread = None + self.max_restart_time = 5 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups; '".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 + def _get_stop_dnode_id(self,dbname): + newTdSql=tdCom.newTdSql() + newTdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = newTdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='leader': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def get_leader_infos(self ,dbname): + + newTdSql=tdCom.newTdSql() + newTdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = newTdSql.queryResult + + leader_infos = set() + for vgroup_info in vgroup_infos: + leader_infos.add(vgroup_info[3:-4]) + + return leader_infos + + def check_revote_leader_success(self, dbname, before_leader_infos , after_leader_infos): + check_status = False + vote_act = set(set(after_leader_infos)-set(before_leader_infos)) + if not vote_act: + print("=======before_revote_leader_infos ======\n" , before_leader_infos) + print("=======after_revote_leader_infos ======\n" , after_leader_infos) + tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + else: + for vgroup_info in vote_act: + for ind , role in enumerate(vgroup_info): + if role==self.stop_dnode_id: + + if vgroup_info[ind+1] =="offline" and "leader" in vgroup_info: + tdLog.info(" === revote leader ok , leader is {} now ====".format(vgroup_info[list(vgroup_info).index("leader")-1])) + check_status = True + elif vgroup_info[ind+1] !="offline": + tdLog.info(" === dnode {} should be offline ".format(self.stop_dnode_id)) + else: + continue + break + return check_status + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # get leader info before stop + before_leader_infos = self.get_leader_infos(db_name) + + # begin stop dnode + + tdDnodes[self.stop_dnode_id-1].stoptaosd() + + self.wait_stop_dnode_OK() + + # vote leaders check + + # get leader info after stop + after_leader_infos = self.get_leader_infos(db_name) + + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + # append rows of stablename when dnode stop make sure revote leaders + + while not revote_status: + after_leader_infos = self.get_leader_infos(db_name) + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + + if revote_status: + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + else: + tdLog.info("===== leader of database {} is not ok , append rows fail =====".format(db_name)) + + # begin start dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + + tdDnodes[self.stop_dnode_id-1].stoptaosd() + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # # get leader info before stop + # before_leader_infos = self.get_leader_infos(db_name) + # self.wait_stop_dnode_OK() + + # check revote leader when restart servers + # # get leader info after stop + # after_leader_infos = self.get_leader_infos(db_name) + # revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + # # append rows of stablename when dnode stop make sure revote leaders + # while not revote_status: + # after_leader_infos = self.get_leader_infos(db_name) + # revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + tdDnodes[self.stop_dnode_id-1].starttaosd() + start = time.time() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + + self.current_thread.join() + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + self.sync_run_case() + # self.unsync_run_case() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py new file mode 100644 index 0000000000..d6edfda770 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py @@ -0,0 +1,580 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 10 + self.current_thread = None + self.max_restart_time = 5 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups; '".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 + + def _get_stop_dnode_id(self,dbname): + newTdSql=tdCom.newTdSql() + newTdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = newTdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='leader': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def get_leader_infos(self ,dbname): + + newTdSql=tdCom.newTdSql() + newTdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = newTdSql.queryResult + + leader_infos = set() + for vgroup_info in vgroup_infos: + leader_infos.add(vgroup_info[3:-4]) + + return leader_infos + + def check_revote_leader_success(self, dbname, before_leader_infos , after_leader_infos): + check_status = False + vote_act = set(set(after_leader_infos)-set(before_leader_infos)) + if not vote_act: + print("=======before_revote_leader_infos ======\n" , before_leader_infos) + print("=======after_revote_leader_infos ======\n" , after_leader_infos) + tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + else: + for vgroup_info in vote_act: + for ind , role in enumerate(vgroup_info): + if role==self.stop_dnode_id: + + if vgroup_info[ind+1] =="offline" and "leader" in vgroup_info: + tdLog.info(" === revote leader ok , leader is {} now ====".format(vgroup_info[list(vgroup_info).index("leader")-1])) + check_status = True + elif vgroup_info[ind+1] !="offline": + tdLog.info(" === dnode {} should be offline ".format(self.stop_dnode_id)) + else: + continue + break + return check_status + + def force_stop_dnode(self, dnode_id ): + + tdSql.query("show dnodes") + port = None + for dnode_info in tdSql.queryResult: + if dnode_id == dnode_info[0]: + port = dnode_info[1].split(":")[-1] + break + else: + continue + if port: + tdLog.info(" ==== dnode {} will be force stop by kill -9 ====".format(dnode_id)) + psCmd = '''netstat -anp|grep -w LISTEN|grep -w %s |grep -o "LISTEN.*"|awk '{print $2}'|cut -d/ -f1|head -n1''' %(port) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + ps_kill_taosd = ''' kill -9 {} '''.format(processID) + # print(ps_kill_taosd) + os.system(ps_kill_taosd) + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # get leader info before stop + before_leader_infos = self.get_leader_infos(db_name) + + # begin stop dnode + # force stop taosd by kill -9 + self.force_stop_dnode(self.stop_dnode_id) + + self.wait_stop_dnode_OK() + + # vote leaders check + + # get leader info after stop + after_leader_infos = self.get_leader_infos(db_name) + + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + # append rows of stablename when dnode stop make sure revote leaders + + while not revote_status: + after_leader_infos = self.get_leader_infos(db_name) + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + + if revote_status: + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + else: + tdLog.info("===== leader of database {} is not ok , append rows fail =====".format(db_name)) + + # begin start dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + # force stop taosd by kill -9 + # get leader info before stop + before_leader_infos = self.get_leader_infos(db_name) + self.force_stop_dnode(self.stop_dnode_id) + + self.wait_stop_dnode_OK() + + # check revote leader when restart servers + # get leader info after stop + after_leader_infos = self.get_leader_infos(db_name) + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + # append rows of stablename when dnode stop make sure revote leaders + while not revote_status: + after_leader_infos = self.get_leader_infos(db_name) + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + + tdDnodes[self.stop_dnode_id-1].starttaosd() + start = time.time() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + + self.current_thread.join() + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + # self.sync_run_case() + self.unsync_run_case() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py new file mode 100644 index 0000000000..5529a5e256 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py @@ -0,0 +1,206 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.db_name ='testdb' + self.replica = 1 + self.vgroups = 2 + self.tb_nums = 10 + self.row_nums = 100 + self.max_vote_time_cost = 10 # seconds + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def check_vgroups_init_done(self,dbname): + + status = True + + tdSql.query("show {}.vgroups".format(dbname)) + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + vgroup_status = [] + for ind , role in enumerate(vgroup_info[3:-4]): + + if ind%2==0: + continue + else: + vgroup_status.append(role) + if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=2: + status = False + return status + return status + + + def vote_leader_time_costs(self,dbname): + start = time.time() + status = self.check_vgroups_init_done(dbname) + while not status: + time.sleep(0.1) + status = self.check_vgroups_init_done(dbname) + + # tdLog.info("=== database {} show vgroups vote the leader is in progress ===".format(dbname)) + end = time.time() + cost_time = end - start + tdLog.info(" ==== database %s vote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) ) + # os.system("taos -s 'show {}.vgroups;'".format(dbname)) + if cost_time >= self.max_vote_time_cost: + tdLog.exit(" ==== database %s vote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) + + + return cost_time + + def test_init_vgroups_time_costs(self): + + tdLog.info(" ====start check time cost about vgroups vote leaders ==== ") + tdLog.info(" ==== current max time cost is set value : {} =======".format(self.max_vote_time_cost)) + + # create database replica 3 vgroups 1 + + db1 = 'db_1' + create_db_replica_3_vgroups_1 = "create database {} replica 3 vgroups 1".format(db1) + tdLog.info('=======database {} replica 3 vgroups 1 ======'.format(db1)) + tdSql.execute(create_db_replica_3_vgroups_1) + self.vote_leader_time_costs(db1) + + # create database replica 3 vgroups 10 + db2 = 'db_2' + create_db_replica_3_vgroups_10 = "create database {} replica 3 vgroups 10".format(db2) + tdLog.info('=======database {} replica 3 vgroups 10 ======'.format(db2)) + tdSql.execute(create_db_replica_3_vgroups_10) + self.vote_leader_time_costs(db2) + + # create database replica 3 vgroups 100 + db3 = 'db_3' + create_db_replica_3_vgroups_100 = "create database {} replica 3 vgroups 100".format(db3) + tdLog.info('=======database {} replica 3 vgroups 100 ======'.format(db3)) + tdSql.execute(create_db_replica_3_vgroups_100) + self.vote_leader_time_costs(db3) + + + + def run(self): + self.check_setup_cluster_status() + self.test_init_vgroups_time_costs() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py new file mode 100644 index 0000000000..3be36c067e --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py @@ -0,0 +1,364 @@ +# author : wenzhouwww +from errno import ESOCKTNOSUPPORT +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import random +import socket +import subprocess + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.db_name ='testdb' + self.replica = 1 + self.vgroups = 2 + self.tb_nums = 10 + self.row_nums = 100 + self.max_vote_time_cost = 10 # seconds + self.stop_dnode = None + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def _get_stop_dnode(self): + only_dnode_list = self.dnode_list.keys() - self.mnode_list.keys() + self.stop_dnode = random.sample(only_dnode_list , 1 )[0] + return self.stop_dnode + + + def check_vgroups_revote_leader(self,dbname): + + status = True + stop_dnode_id = self.dnode_list[self.stop_dnode][0] + + tdSql.query("show {}.vgroups".format(dbname)) + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + vgroup_status = [] + vgroups_leader_follower = vgroup_info[3:-4] + for ind , role in enumerate(vgroups_leader_follower): + + if ind%2==0: + if role == stop_dnode_id and vgroups_leader_follower[ind+1]=="offline": + tdLog.info("====== dnode {} has offline , endpoint is {}".format(stop_dnode_id , self.stop_dnode)) + elif role == stop_dnode_id : + tdLog.exit("====== dnode {} has not offline , endpoint is {}".format(stop_dnode_id , self.stop_dnode)) + else: + continue + else: + vgroup_status.append(role) + if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=1 or vgroup_status.count("offline")!=1: + status = False + return status + return status + + + def wait_stop_dnode_OK(self): + + def _get_status(): + + status = "" + tdSql.query("show dnodes") + dnode_infos = tdSql.queryResult + for dnode_info in dnode_infos: + endpoint = dnode_info[1] + dnode_status = dnode_info[4] + if endpoint == self.stop_dnode: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , endpoint is {}".format(self.stop_dnode)) + + def wait_start_dnode_OK(self): + + def _get_status(): + + status = "" + tdSql.query("show dnodes") + dnode_infos = tdSql.queryResult + for dnode_info in dnode_infos: + endpoint = dnode_info[1] + dnode_status = dnode_info[4] + if endpoint == self.stop_dnode: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , endpoint is {}".format(self.stop_dnode)) + + + + def random_stop_One_dnode(self): + self.stop_dnode = self._get_stop_dnode() + stop_dnode_id = self.dnode_list[self.stop_dnode][0] + tdLog.info(" ==== dnode {} will offline ,endpoints is {} ====".format(stop_dnode_id , self.stop_dnode)) + tdDnodes=cluster.dnodes + tdDnodes[stop_dnode_id-1].stoptaosd() + self.wait_stop_dnode_OK() + # os.system("taos -s 'show dnodes;'") + + def Restart_stop_dnode(self): + + tdDnodes=cluster.dnodes + stop_dnode_id = self.dnode_list[self.stop_dnode][0] + tdDnodes[stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + # os.system("taos -s 'show dnodes;'") + + def check_vgroups_init_done(self,dbname): + + status = True + + tdSql.query("show {}.vgroups".format(dbname)) + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + vgroup_status = [] + for ind , role in enumerate(vgroup_info[3:-4]): + + if ind%2==0: + continue + else: + vgroup_status.append(role) + if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=2: + status = False + return status + return status + + def vote_leader_time_costs(self,dbname): + start = time.time() + status = self.check_vgroups_init_done(dbname) + while not status: + time.sleep(0.1) + status = self.check_vgroups_init_done(dbname) + + # tdLog.info("=== database {} show vgroups vote the leader is in progress ===".format(dbname)) + end = time.time() + cost_time = end - start + tdLog.info(" ==== database %s vote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) ) + # os.system("taos -s 'show {}.vgroups;'".format(dbname)) + if cost_time >= self.max_vote_time_cost: + tdLog.exit(" ==== database %s vote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) + + return cost_time + + + def revote_leader_time_costs(self,dbname): + start = time.time() + + status = self.check_vgroups_revote_leader(dbname) + while not status: + time.sleep(0.1) + status = self.check_vgroups_revote_leader(dbname) + + # tdLog.info("=== database {} show vgroups vote the leader is in progress ===".format(dbname)) + end = time.time() + cost_time = end - start + tdLog.info(" ==== database %s revote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) ) + # os.system("taos -s 'show {}.vgroups;'".format(dbname)) + if cost_time >= self.max_vote_time_cost: + tdLog.exit(" ==== database %s revote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) + + + return cost_time + + def exec_revote_action(self,dbname): + + tdSql.query("show {}.vgroups".format(dbname)) + before_revote = tdSql.queryResult + + before_vgroups = set() + for vgroup_info in before_revote: + before_vgroups.add(vgroup_info[3:-4]) + + self.random_stop_One_dnode() + tdSql.query("show {}.vgroups".format(dbname)) + after_revote = tdSql.queryResult + + after_vgroups = set() + for vgroup_info in after_revote: + after_vgroups.add(vgroup_info[3:-4]) + + vote_act = set(set(after_vgroups)-set(before_vgroups)) + if not vote_act: + tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + else: + for vgroup_info in vote_act: + for ind , role in enumerate(vgroup_info): + if role==self.dnode_list[self.stop_dnode][0]: + + if vgroup_info[ind+1] =="offline" and "leader" in vgroup_info: + tdLog.info(" === revote leader ok , leader is {} now ====".format(list(vgroup_info).index("leader")-1)) + elif vgroup_info[ind+1] !="offline": + tdLog.exit(" === dnode {} should be offline ".format(self.stop_dnode)) + else: + continue + break + + + + self.revote_leader_time_costs(dbname) + self.Restart_stop_dnode() + def test_init_vgroups_time_costs(self): + + tdLog.info(" ====start check time cost about vgroups vote leaders ==== ") + tdLog.info(" ==== current max time cost is set value : {} =======".format(self.max_vote_time_cost)) + + # create database replica 3 vgroups 1 + + db1 = 'db_1' + create_db_replica_3_vgroups_1 = "create database {} replica 3 vgroups 1".format(db1) + tdLog.info('=======database {} replica 3 vgroups 1 ======'.format(db1)) + tdSql.execute(create_db_replica_3_vgroups_1) + self.vote_leader_time_costs(db1) + self.exec_revote_action(db1) + + # create database replica 3 vgroups 10 + db2 = 'db_2' + create_db_replica_3_vgroups_10 = "create database {} replica 3 vgroups 10".format(db2) + tdLog.info('=======database {} replica 3 vgroups 10 ======'.format(db2)) + tdSql.execute(create_db_replica_3_vgroups_10) + self.vote_leader_time_costs(db2) + self.exec_revote_action(db2) + + # create database replica 3 vgroups 100 + db3 = 'db_3' + create_db_replica_3_vgroups_100 = "create database {} replica 3 vgroups 100".format(db3) + tdLog.info('=======database {} replica 3 vgroups 100 ======'.format(db3)) + tdSql.execute(create_db_replica_3_vgroups_100) + self.vote_leader_time_costs(db3) + self.exec_revote_action(db3) + + + + def run(self): + self.check_setup_cluster_status() + self.test_init_vgroups_time_costs() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index a3462feb14..df8517e315 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -358,8 +358,8 @@ class TDTestCase: tdSql.error("alter table %s.%s modify column c2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s modify tag t2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t1=20"%(parameterDict['dbName'], ctbName)) - tdSql.error("alter table %s.%s set tag t2='20'"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t1=20"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t2='20'"%(parameterDict['dbName'], ctbName)) tdSql.error("alter table %s.%s rename column c1 c1new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s rename column c2 c2new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -374,13 +374,13 @@ class TDTestCase: tdSql.query("alter table %s.%s set tag t4='20'"%(parameterDict['dbName'], ctbName)) tdSql.query("alter table %s.%s set tag t5='20'"%(parameterDict['dbName'], ctbName)) - tdSql.query("alter table %s.%s rename column c3 c3new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s rename column c4 c4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c3 c3new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c4 c4new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename tag t3 t3new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename tag t4 t4new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s drop column c3new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s drop column c4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c3"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c4"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s drop tag t3new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s drop tag t4new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -508,10 +508,10 @@ class TDTestCase: tdSql.error("alter table %s.%s modify tag t2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s modify tag t4 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t1=20"%(parameterDict['dbName'], ctbName)) - tdSql.error("alter table %s.%s set tag t2='20'"%(parameterDict['dbName'], ctbName)) - tdSql.error("alter table %s.%s set tag t3=20"%(parameterDict['dbName'], ctbName)) - tdSql.error("alter table %s.%s set tag t4='20'"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t1=20"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t2='20'"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t3=20"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t4='20'"%(parameterDict['dbName'], ctbName)) tdSql.error("alter table %s.%s rename column c1 c1new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s rename column c2 c2new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -528,10 +528,10 @@ class TDTestCase: tdSql.query("alter table %s.%s set tag t5='50'"%(parameterDict['dbName'], ctbName)) - tdSql.query("alter table %s.%s rename column c5 c5new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c5 c5new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename tag t5 t5new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s drop column c5new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c5"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s drop tag t5new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add column c5 float"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -830,9 +830,9 @@ class TDTestCase: cfgPath = buildPath + "/../sim/psim/cfg" tdLog.info("cfgPath: %s" % cfgPath) - # self.tmqCase1(cfgPath, buildPath) - # self.tmqCase2(cfgPath, buildPath) - # self.tmqCase3(cfgPath, buildPath) + self.tmqCase1(cfgPath, buildPath) + self.tmqCase2(cfgPath, buildPath) + self.tmqCase3(cfgPath, buildPath) self.tmqCase4(cfgPath, buildPath) self.tmqCase5(cfgPath, buildPath) diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index ea100ae0d3..277fdf7afb 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -18,7 +18,7 @@ class TDTestCase: def __init__(self): self.snapshot = 0 self.vgroups = 4 - self.ctbNum = 1000 + self.ctbNum = 500 self.rowsPerTbl = 1000 def init(self, conn, logSql): @@ -38,9 +38,9 @@ class TDTestCase: 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 1000, + 'ctbNum': 500, 'rowsPerTbl': 1000, - 'batchNum': 400, + 'batchNum': 500, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 3, 'showMsg': 1, @@ -83,9 +83,9 @@ class TDTestCase: 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 1000, + 'ctbNum': 500, 'rowsPerTbl': 1000, - 'batchNum': 1000, + 'batchNum': 500, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 5, 'showMsg': 1, @@ -156,7 +156,7 @@ class TDTestCase: 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 1000, + 'ctbNum': 500, 'rowsPerTbl': 1000, 'batchNum': 1000, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index a56f79d20f..20cffed486 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -129,9 +129,12 @@ class TMQCom: def stopTmqSimProcess(self, processorName): psCmd = "ps -ef|grep -w %s|grep -v grep | awk '{print $2}'"%(processorName) processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + onlyKillOnceWindows = 0 while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) + if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): + killCmd = "kill -INT %s > /dev/null 2>&1" % processID + os.system(killCmd) + onlyKillOnceWindows = 1 time.sleep(0.2) processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") tdLog.debug("%s is stopped by kill -INT" % (processorName)) @@ -535,6 +538,32 @@ class TMQCom: column_value_str = column_value_str.rstrip()[:-1] insert_sql = f'insert into {dbname}.{tbname_prefix}{tblIdx+tbname_index_start_num} values ({column_value_str});' tsql.execute(insert_sql) + + def waitSubscriptionExit(self, tsql, topicName): + wait_cnt = 0 + while True: + exit_flag = 1 + tsql.query("show subscriptions") + rows = tsql.getRows() + for idx in range (rows): + if tsql.getData(idx, 0) != topicName: + continue + + if tsql.getData(idx, 3) == None: + continue + else: + time.sleep(0.5) + wait_cnt += 1 + exit_flag = 0 + break + + if exit_flag == 1: + break + + tsql.query("show subscriptions") + tdLog.info("show subscriptions:") + tdLog.info(tsql.queryResult) + tdLog.info("wait subscriptions exit for %d s"%wait_cnt) def close(self): self.cursor.close() diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 9699c4b32c..cec6985a4e 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -19,7 +19,7 @@ class TDTestCase: self.snapshot = 0 self.vgroups = 2 self.ctbNum = 100 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") @@ -39,8 +39,8 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 100, - 'rowsPerTbl': 10000, - 'batchNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 3, 'showMsg': 1, @@ -83,8 +83,8 @@ class TDTestCase: 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 100, - 'rowsPerTbl': 10000, + 'ctbNum': 1000, + 'rowsPerTbl': 1000, 'batchNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 5, @@ -117,13 +117,13 @@ class TDTestCase: tdSql.execute(sqlString) consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 topicList = topicFromStb1 ifcheckdata = 0 ifManualCommit = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:500,\ + auto.commit.interval.ms:3000,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) @@ -131,13 +131,13 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) # time.sleep(3) - tmqCom.getStartCommitNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("================= restart dnode ===========================") - tdDnodes.stop(1) - tdDnodes.start(1) - time.sleep(3) + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + # time.sleep(3) - tdLog.info("insert process end, and start to check consume result") + tdLog.info(" restart taosd end and wait to check consume result") expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) totalConsumeRows = 0 @@ -147,10 +147,46 @@ class TDTestCase: tdSql.query(queryString) totalRowsFromQury = tdSql.getRows() - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, totalRowsFromQury)) - if totalConsumeRows != totalRowsFromQury: + tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury)) + if not (totalConsumeRows == totalRowsFromQury): tdLog.exit("tmq consume rows error!") + + + + # tdLog.info("****************************************************************************") + # tmqCom.initConsumerTable() + # consumerId = 1 + # expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + # topicList = topicFromStb1 + # ifcheckdata = 0 + # ifManualCommit = 0 + # keyList = 'group.id:cgrp2,\ + # enable.auto.commit:true,\ + # auto.commit.interval.ms:3000,\ + # auto.offset.reset:earliest' + # tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + # tdLog.info("start consume processor") + # tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + # expectRows = 1 + # resultList = tmqCom.selectConsumeResult(expectRows) + # totalConsumeRows = 0 + # for i in range(expectRows): + # totalConsumeRows += resultList[i] + + # tdSql.query(queryString) + # totalRowsFromQury = tdSql.getRows() + + # tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury)) + # if not (totalConsumeRows == totalRowsFromQury): + # tdLog.exit("tmq consume rows error!") + + + # tdLog.info("****************************************************************************") + + tmqCom.waitSubscriptionExit(tdSql, topicFromStb1) tdSql.query("drop topic %s"%topicFromStb1) tdLog.printNoPrefix("======== test case 1 end ...... ") @@ -169,10 +205,10 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 100, - 'rowsPerTbl': 10000, - 'batchNum': 3000, + 'rowsPerTbl': 1000, + 'batchNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -201,7 +237,7 @@ class TDTestCase: tdSql.execute(sqlString) consumerId = 1 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + 100000 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 topicList = topicFromStb1 ifcheckdata = 0 ifManualCommit = 0 @@ -214,14 +250,16 @@ class TDTestCase: tdLog.info("start consume processor") tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tmqCom.getStartCommitNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("================= restart dnode ===========================") - tdDnodes.stop(1) - tdDnodes.start(1) - time.sleep(3) + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + # time.sleep(3) tdLog.info("create some new child table and insert data ") - tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],"ctb",paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + paraDict["batchNum"] = 100 + paraDict["ctbPrefix"] = 'newCtb' + # tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) tdLog.info("insert process end, and start to check consume result") expectRows = 1 @@ -237,14 +275,15 @@ class TDTestCase: if totalConsumeRows != totalRowsFromQuery: tdLog.exit("tmq consume rows error!") + tmqCom.waitSubscriptionExit(tdSql, topicFromStb1) tdSql.query("drop topic %s"%topicFromStb1) tdLog.printNoPrefix("======== test case 2 end ...... ") def run(self): - tdSql.prepare() + # tdSql.prepare() self.prepareTestEnv() - # self.tmqCase1() + self.tmqCase1() self.tmqCase2() def stop(self): diff --git a/tests/system-test/7-tmq/tmqDropNtb.py b/tests/system-test/7-tmq/tmqDropNtb.py index 9200200588..e1f5794ce2 100644 --- a/tests/system-test/7-tmq/tmqDropNtb.py +++ b/tests/system-test/7-tmq/tmqDropNtb.py @@ -25,18 +25,6 @@ class TDTestCase: tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) - def waitSubscriptionExit(self, max_wait_count=20): - wait_cnt = 0 - while (wait_cnt < max_wait_count): - tdSql.query("show subscriptions") - if tdSql.getRows() == 0: - break - else: - time.sleep(1) - wait_cnt += 1 - - tdLog.info("wait subscriptions exit for %d s"%wait_cnt) - # drop some ntbs def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") @@ -115,7 +103,7 @@ class TDTestCase: tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.info("wait subscriptions exit ....") - self.waitSubscriptionExit() + tmqCom.waitSubscriptionExit(tdSql, topicFromDb) tdSql.query("drop topic %s"%topicFromDb) tdLog.info("success dorp topic: %s"%topicFromDb) @@ -208,7 +196,7 @@ class TDTestCase: tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.info("wait subscriptions exit ....") - self.waitSubscriptionExit() + tmqCom.waitSubscriptionExit(tdSql, topicFromDb) tdSql.query("drop topic %s"%topicFromDb) tdLog.info("success dorp topic: %s"%topicFromDb) diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index d9e675ddc6..f86d1295f4 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -24,19 +24,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) - - def waitSubscriptionExit(self, max_wait_count=20): - wait_cnt = 0 - while (wait_cnt < max_wait_count): - tdSql.query("show subscriptions") - if tdSql.getRows() == 0: - break - else: - time.sleep(2) - wait_cnt += 1 - - tdLog.info("wait subscriptions exit for %d s"%wait_cnt) - + def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -63,7 +51,7 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl - tmqCom.initConsumerTable() + # tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) @@ -110,6 +98,8 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl + tmqCom.initConsumerTable() + # again create one new stb1 paraDict["stbName"] = 'stb1' paraDict['ctbPrefix'] = 'ctb1n_' @@ -169,7 +159,7 @@ class TDTestCase: tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.info("wait subscriptions exit ....") - self.waitSubscriptionExit() + tmqCom.waitSubscriptionExit(tdSql, topicFromDb) tdSql.query("drop topic %s"%topicFromDb) tdLog.info("success dorp topic: %s"%topicFromDb) @@ -203,6 +193,8 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl + tmqCom.initConsumerTable() + # again create one new stb1 paraDict["stbName"] = 'stb2' paraDict['ctbPrefix'] = 'ctb2n_' @@ -221,9 +213,9 @@ class TDTestCase: tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) if self.snapshot == 0: - consumerId = 0 + consumerId = 2 elif self.snapshot == 1: - consumerId = 1 + consumerId = 3 expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) topicList = topicFromDb @@ -258,7 +250,7 @@ class TDTestCase: tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.info("wait subscriptions exit ....") - self.waitSubscriptionExit() + tmqCom.waitSubscriptionExit(tdSql, topicFromDb) tdSql.query("drop topic %s"%topicFromDb) tdLog.info("success dorp topic: %s"%topicFromDb) diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py new file mode 100644 index 0000000000..4f21beffc4 --- /dev/null +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -0,0 +1,190 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.vgroups = 4 + self.ctbNum = 100 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # update to half tables + paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + consumerId = 0 + + if self.snapshot == 0: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/2 + 1)) + elif self.snapshot == 1: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (2)) + + topicList = topicFromStb1 + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + # update to half data + paraDict["startTs"] = paraDict["startTs"] + int(self.rowsPerTbl / 2) + tmqCom.asyncInsertDataByInterlace(paraDict) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + tdLog.info("act consume rows: %d, expect consume rows: %d, act insert rows: %d"%(totalConsumeRows, expectrowcnt, totalRowsInserted)) + if self.snapshot == 0: + if totalConsumeRows != expectrowcnt: + tdLog.exit("tmq consume rows error!") + elif self.snapshot == 1: + if not (totalConsumeRows < expectrowcnt and totalConsumeRows >= totalRowsInserted): + tdLog.exit("tmq consume rows error!") + + # tmqCom.checkFileContent(consumerId, queryString) + + tdSql.query("drop topic %s"%topicFromStb1) + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") + self.ctbNum = 1 + self.snapshot = 0 + self.prepareTestEnv() + self.tmqCase1() + + tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + self.prepareTestEnv() + self.snapshot = 1 + self.tmqCase1() + + tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") + self.ctbNum = 100 + self.snapshot = 0 + self.prepareTestEnv() + self.tmqCase1() + tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + self.prepareTestEnv() + self.snapshot = 1 + self.tmqCase1() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/TD-17699.py b/tests/system-test/99-TDcase/TD-17699.py similarity index 87% rename from tests/system-test/7-tmq/TD-17699.py rename to tests/system-test/99-TDcase/TD-17699.py index 87b11f6f83..cf5e7797ec 100644 --- a/tests/system-test/7-tmq/TD-17699.py +++ b/tests/system-test/99-TDcase/TD-17699.py @@ -31,7 +31,8 @@ class TDTestCase: 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 20, 'showMsg': 1, - 'showRow': 1} + 'showRow': 1, + 'snapshot': 1} cdbName = 'cdb' # some parameter to consumer processor @@ -88,13 +89,15 @@ class TDTestCase: tmqCom.insertConsumerInfo(self.consumerId, self.expectrowcnt,topicList,keyList,self.ifcheckdata,self.ifManualCommit) tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(self.pollDelay,self.paraDict["dbName"],self.showMsg, self.showRow,self.cdbName) - + # tmqCom.startTmqSimProcess(self.pollDelay,self.paraDict["dbName"],self.showMsg, self.showRow,self.cdbName) + tmqCom.startTmqSimProcess(pollDelay=self.pollDelay,dbName=self.paraDict["dbName"],showMsg=self.showMsg, showRow=self.showRow,snapshot=self.paraDict['snapshot']) + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("drop one stable") self.paraDict["stbName"] = 'stb1' - tdSql.execute("drop table %s.%s" %(self.paraDict['dbName'], self.paraDict['stbName'])) - tmqCom.drop_ctable(tdSql, dbname=self.paraDict['dbName'], count=self.paraDict["ctbNum"], default_ctbname_prefix=self.paraDict["ctbPrefix"]) + tdSql.execute("drop table %s.%s" %(self.paraDict['dbName'], self.paraDict['stbName'])) + dropTblNum = int(self.paraDict["ctbNum"] / 4) + tmqCom.drop_ctable(tdSql, dbname=self.paraDict['dbName'], count=dropTblNum, default_ctbname_prefix=self.paraDict["ctbPrefix"]) # pThread2.join() @@ -106,7 +109,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if not (totalConsumeRows >= self.expectrowcnt/2 and totalConsumeRows <= self.expectrowcnt): + if not (totalConsumeRows >= self.expectrowcnt*3/8 and totalConsumeRows <= self.expectrowcnt): tdLog.info("act consume rows: %d, expect consume rows: between %d and %d"%(totalConsumeRows, self.expectrowcnt/2, self.expectrowcnt)) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/clusterCase.sh b/tests/system-test/clusterCase.sh new file mode 100755 index 0000000000..cfc44f7f95 --- /dev/null +++ b/tests/system-test/clusterCase.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e +set -x + +python3 ./test.py -f 6-cluster/5dnode1mnode.py +#python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 +#python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 +# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 +# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 +# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 5 -M 3 +# python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 5 -M 3 +# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py +# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +# python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 +# BUG Redict python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 +# python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 5 -M 3 + python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 + + diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 838b9da904..e4401ae0f1 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -94,7 +94,7 @@ python3 ./test.py -f 2-query/distribute_agg_min.py -R -# python3 ./test.py -f 1-insert/update_data.py +python3 ./test.py -f 1-insert/update_data.py python3 ./test.py -f 1-insert/delete_data.py @@ -167,22 +167,26 @@ python3 ./test.py -f 2-query/max_partition.py python3 ./test.py -f 2-query/last_row.py python3 ./test.py -f 6-cluster/5dnode1mnode.py -#python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 -#python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 -# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 -# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 + +# python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 5 -M 3 -# python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 5 -M 3 + +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 5 -M 3 +# python3 ./test.py -f 6-cluster/5dnode3mnodeRestartMnodeInsertData.py -N 5 -M 3 +# python3 ./test.py -f 6-cluster/5dnode3mnodeRestartVnodeInsertData.py -N 5 -M 3 + +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py # python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 -# BUG Redict python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 -# python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 5 -M 3 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 - python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py @@ -199,7 +203,7 @@ python3 ./test.py -f 7-tmq/subscribeStb3.py python3 ./test.py -f 7-tmq/subscribeStb4.py python3 ./test.py -f 7-tmq/db.py python3 ./test.py -f 7-tmq/tmqError.py -python3 ./test.py -f 7-tmq/schema.py +#python3 ./test.py -f 7-tmq/schema.py python3 ./test.py -f 7-tmq/stbFilter.py python3 ./test.py -f 7-tmq/tmqCheckData.py python3 ./test.py -f 7-tmq/tmqCheckData1.py @@ -219,17 +223,20 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -#python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py +python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py #python3 ./test.py -f 7-tmq/tmqDnodeRestart.py python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py python3 ./test.py -f 7-tmq/tmqDropStb.py +python3 ./test.py -f 7-tmq/tmqDropStbCtb.py +python3 ./test.py -f 7-tmq/tmqDropNtb.py python3 ./test.py -f 7-tmq/tmqUdf.py -# python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py -# python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py +python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py # python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 505c290f2a..5db97a0f0f 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -28,6 +28,7 @@ target_link_libraries( sdbDump PUBLIC dnode PUBLIC mnode + PUBLIC stream PUBLIC sdb PUBLIC os ) @@ -37,4 +38,4 @@ target_include_directories( PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc" PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/sdb/inc" PRIVATE "${TD_SOURCE_DIR}/source/dnode/mgmt/node_mgmt/inc" -) \ No newline at end of file +) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 190b4224a4..6a18263d50 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -352,6 +352,29 @@ void ltrim(char* str) { // return str; } +int queryDB(TAOS* taos, char* command) { + int retryCnt = 10; + int code; + TAOS_RES* pRes; + + while (retryCnt--) { + pRes = taos_query(taos, command); + code = taos_errno(pRes); + if (code != 0) { + taosSsleep(1); + taos_free_result(pRes); + continue; + } + taos_free_result(pRes); + return 0; + } + + pError("failed to reason:%s, sql: %s", tstrerror(code), command); + taos_free_result(pRes); + return -1; +} + + void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { int32_t i; for (i = 0; i < pInfo->numOfVgroups; i++) { @@ -374,30 +397,49 @@ void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { } } +TAOS* createNewTaosConnect() { + TAOS* taos = NULL; + int32_t retryCnt = 10; + + while (retryCnt--) { + TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (NULL != taos) { + return taos; + } + taosSsleep(1); + } + + taosFprintfFile(g_fp, "taos_connect() fail\n"); + return NULL; +} + + int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { char sqlStr[1100] = {0}; if (strlen(buf) > 1024) { taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); taosCloseFile(&g_fp); - exit(-1); + return -1; } TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - assert(pConn != NULL); + if (pConn == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not save consume result to main script\n"); + return -1; + } sprintf(sqlStr, "insert into %s.content_%d values (%" PRId64 ", \'%s\')", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->ts++, buf); - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - pError("error in insert consume result, reason:%s\n", taos_errstr(pRes)); - taosFprintfFile(g_fp, "error in insert consume result, reason:%s\n", taos_errstr(pRes)); + int retCode = queryDB(pConn, sqlStr); + if (retCode != 0) { + taosFprintfFile(g_fp, "error in save consume content\n"); taosCloseFile(&g_fp); - taos_free_result(pRes); + taos_close(pConn); exit(-1); } - taos_free_result(pRes); + taos_close(pConn); return 0; } @@ -591,15 +633,12 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t code = tmq_get_raw_meta(msg, &raw); if(code == TSDB_CODE_SUCCESS){ - TAOS_RES* pRes = taos_query(pInfo->taos, "use metadb"); - if (taos_errno(pRes) != 0) { - pError("error when use metadb, reason:%s\n", taos_errstr(pRes)); - taosFprintfFile(g_fp, "error when use metadb, reason:%s\n", taos_errstr(pRes)); + int retCode = queryDB(pInfo->taos, "use metadb"); + if (retCode != 0) { + taosFprintfFile(g_fp, "error when use metadb\n"); taosCloseFile(&g_fp); - taos_free_result(pRes); exit(-1); } - taos_free_result(pRes); taosFprintfFile(g_fp, "raw:%p\n", &raw); taos_write_raw_meta(pInfo->taos, raw); @@ -618,19 +657,6 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn return totalRows; } - -int queryDB(TAOS* taos, char* command) { - TAOS_RES* pRes = taos_query(taos, command); - int code = taos_errno(pRes); - if (code != 0) { - pError("failed to reason:%s, sql: %s", tstrerror(code), command); - taos_free_result(pRes); - return -1; - } - taos_free_result(pRes); - return 0; -} - static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {} int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { @@ -720,15 +746,12 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { char tmpString[128]; taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId, sqlStr); - TAOS_RES* pRes = taos_query(pInfo->taos, sqlStr); - if (taos_errno(pRes) != 0) { - pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - exit(-1); + int retCode = queryDB(pInfo->taos, sqlStr); + if (retCode != 0) { + taosFprintfFile(g_fp, "consume id %d error in save consume result\n", pInfo->consumerId); + return -1; } - taos_free_result(pRes); - return 0; } @@ -823,18 +846,18 @@ void loop_consume(SThreadInfo* pInfo) { void* consumeThreadFunc(void* param) { SThreadInfo* pInfo = (SThreadInfo*)param; - pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + pInfo->taos = createNewTaosConnect(); if (pInfo->taos == NULL) { taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); - ASSERT(0); return NULL; } build_consumer(pInfo); build_topic_list(pInfo); if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { - taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); - assert(0); + taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -842,7 +865,8 @@ void* consumeThreadFunc(void* param) { if (err != 0) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); - assert(0); + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -926,17 +950,20 @@ void parseConsumeInfo() { int32_t getConsumeInfo() { char sqlStr[1024] = {0}; - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - assert(pConn != NULL); + TAOS* pConn = createNewTaosConnect(); + if (pConn == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not get consume info for start consumer\n"); + return -1; + } sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); - TAOS_RES* pRes = taos_query(pConn, sqlStr); + TAOS_RES *pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { - pError("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); - taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); + taosFprintfFile(g_fp, "error in get consumeinfo for %s\n", taos_errstr(pRes)); taosCloseFile(&g_fp); taos_free_result(pRes); - exit(-1); + taos_close(pConn); + return -1; } TAOS_ROW row = NULL; @@ -981,6 +1008,7 @@ int32_t getConsumeInfo() { taos_free_result(pRes); parseConsumeInfo(); + taos_close(pConn); return 0; } @@ -1123,7 +1151,6 @@ void* ombConsumeThreadFunc(void* param) { if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); - assert(0); return NULL; } @@ -1131,7 +1158,6 @@ void* ombConsumeThreadFunc(void* param) { if (err != 0) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); - assert(0); return NULL; } @@ -1181,9 +1207,9 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type) { void* ombProduceThreadFunc(void* param) { SThreadInfo* pInfo = (SThreadInfo*)param; - pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + pInfo->taos = createNewTaosConnect(); if (pInfo->taos == NULL) { - printf("taos_connect() fail\n"); + taosFprintfFile(g_fp, "taos_connect() fail, can not start producers!\n"); return NULL; } @@ -1200,6 +1226,8 @@ void* ombProduceThreadFunc(void* param) { char* sqlBuf = taosMemoryMalloc(MAX_SQL_LEN); if (NULL == sqlBuf) { printf("malloc fail for sqlBuf\n"); + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -1232,6 +1260,8 @@ void* ombProduceThreadFunc(void* param) { int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); if (affectedRows < 0) { + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -1266,6 +1296,8 @@ void* ombProduceThreadFunc(void* param) { } printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -1301,10 +1333,9 @@ void startOmbConsume() { taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); if (0 != g_stConfInfo.producers) { - TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + TAOS* taos = createNewTaosConnect(); if (taos == NULL) { - taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); - ASSERT(0); + taosFprintfFile(g_fp, "taos_connect() fail, can not create db, stbl, ctbl, topic!\n"); return ; } @@ -1357,9 +1388,11 @@ void startOmbConsume() { taosFprintfFile(g_fp, "==== close tmqlog ====\n"); taosCloseFile(&g_fp); + taos_close(taos); return; } + taos_close(taos); } // pthread_create one thread to consume @@ -1418,7 +1451,11 @@ int main(int32_t argc, char* argv[]) { return 0; } - getConsumeInfo(); + int32_t retCode = getConsumeInfo(); + if (0 != retCode) { + return -1; + } + saveConfigToLogFile(); tmqSetSignalHandle(); diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index e6cd47b91d..58095940f3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,46 +1,44 @@ IF (TD_WEBSOCKET) MESSAGE("${Green} use libtaos-ws${ColourReset}") - IF (TD_LINUX) - IF (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" OR "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" IS_NEWER_THAN "${CMAKE_SOURCE_DIR}/.git/modules/tools/taosws-rs/FETCH_HEAD") - include(ExternalProject) - ExternalProject_Add(taosws-rs - PREFIX "taosws-rs" - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs - BUILD_ALWAYS off - DEPENDS taos - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" - PATCH_COMMAND - COMMAND git clean -f -d - BUILD_COMMAND - COMMAND cargo build --release -p taos-ws-sys - COMMAND ./taos-ws-sys/ci/package.sh - INSTALL_COMMAND - COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib - COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include - COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include - ) - ELSE() - include(ExternalProject) - ExternalProject_Add(taosws-rs - PREFIX "taosws-rs" - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs - BUILD_ALWAYS on - DEPENDS taos - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" - PATCH_COMMAND - COMMAND git clean -f -d - BUILD_COMMAND - COMMAND cargo build --release -p taos-ws-sys - COMMAND ./taos-ws-sys/ci/package.sh - INSTALL_COMMAND - COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib - COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include - COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include - ) - ENDIF () - ENDIF() + IF (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" OR "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" IS_NEWER_THAN "${CMAKE_SOURCE_DIR}/.git/modules/tools/taosws-rs/FETCH_HEAD") + include(ExternalProject) + ExternalProject_Add(taosws-rs + PREFIX "taosws-rs" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND cargo build --release -p taos-ws-sys + COMMAND ./taos-ws-sys/ci/package.sh + INSTALL_COMMAND + COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include + COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include + ) + ELSE() + include(ExternalProject) + ExternalProject_Add(taosws-rs + PREFIX "taosws-rs" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs + BUILD_ALWAYS on + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND cargo build --release -p taos-ws-sys + COMMAND ./taos-ws-sys/ci/package.sh + INSTALL_COMMAND + COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include + COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include + ) + ENDIF () ENDIF () IF (TD_TAOS_TOOLS) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 825866e163..358377f804 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -26,6 +26,10 @@ #include "ttypes.h" #include "tutil.h" +#ifdef WEBSOCKET +#include "taosws.h" +#endif + #define SHELL_MAX_HISTORY_SIZE 1000 #define SHELL_MAX_COMMAND_SIZE 1048586 #define SHELL_HISTORY_FILE ".taos_history" @@ -67,6 +71,12 @@ typedef struct { int32_t pktNum; int32_t displayWidth; int32_t abort; +#ifdef WEBSOCKET + bool restful; + bool cloud; + char* dsn; + int32_t timeout; +#endif } SShellArgs; typedef struct { @@ -85,6 +95,10 @@ typedef struct { TAOS* conn; TdThread pid; tsem_t cancelSem; +#ifdef WEBSOCKET + WS_TAOS* ws_conn; + bool stop_query; +#endif } SShellObj; // shellArguments.c @@ -95,7 +109,10 @@ int32_t shellReadCommand(char* command); // shellEngine.c int32_t shellExecute(); - +int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision); +void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields); +void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision); +void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int32_t precision); // shellUtil.c int32_t shellCheckIntSize(); void shellPrintVersion(); @@ -109,6 +126,14 @@ void shellExit(); // shellNettest.c void shellTestNetWork(); +#ifdef WEBSOCKET +void shellCheckConnectMode(); +// shellWebsocket.c +int shell_conn_ws_server(bool first); +int32_t shell_run_websocket(); +void shellRunSingleCommandWebsocketImp(char *command); +#endif + // shellMain.c extern SShellObj shell; diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 466aa52390..88ef46c5d6 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -43,6 +43,12 @@ #define SHELL_VERSION "Print program version." #define SHELL_EMAIL "" +#ifdef WEBSOCKET +#define SHELL_DSN "The dsn to use when connecting to cloud server." +#define SHELL_REST "Use restful mode when connecting." +#define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 10." +#endif + static int32_t shellParseSingleOpt(int32_t key, char *arg); void shellPrintHelp() { @@ -65,6 +71,11 @@ void shellPrintHelp() { printf("%s%s%s%s\r\n", indent, "-s,", indent, SHELL_CMD); printf("%s%s%s%s\r\n", indent, "-t,", indent, SHELL_STARTUP); printf("%s%s%s%s\r\n", indent, "-u,", indent, SHELL_USER); +#ifdef WEBSOCKET + printf("%s%s%s%s\r\n", indent, "-E,", indent, SHELL_DSN); + printf("%s%s%s%s\r\n", indent, "-R,", indent, SHELL_REST); + printf("%s%s%s%s\r\n", indent, "-T,", indent, SHELL_TIMEOUT); +#endif printf("%s%s%s%s\r\n", indent, "-w,", indent, SHELL_WIDTH); printf("%s%s%s%s\r\n", indent, "-V,", indent, SHELL_VERSION); printf("\r\n\r\nReport bugs to %s.\r\n", SHELL_EMAIL); @@ -95,6 +106,11 @@ static struct argp_option shellOptions[] = { {"display-width", 'w', "WIDTH", 0, SHELL_WIDTH}, {"netrole", 'n', "NETROLE", 0, SHELL_NET_ROLE}, {"pktlen", 'l', "PKTLEN", 0, SHELL_PKG_LEN}, +#ifdef WEBSOCKET + {"dsn", 'E', "DSN", 0, SHELL_DSN}, + {"restful", 'R', 0, 0, SHELL_REST}, + {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, +#endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {0}, }; @@ -120,9 +136,15 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { switch (key) { case 'h': pArgs->host = arg; +#ifdef WEBSOCKET + pArgs->cloud = false; +#endif break; case 'P': pArgs->port = atoi(arg); +#ifdef WEBSOCKET + pArgs->cloud = false; +#endif if (pArgs->port == 0) pArgs->port = -1; break; case 'u': @@ -137,6 +159,9 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { pArgs->is_gen_auth = true; break; case 'c': +#ifdef WEBSOCKET + pArgs->cloud = false; +#endif pArgs->cfgdir = arg; break; case 'C': @@ -172,6 +197,18 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { case 'N': pArgs->pktNum = atoi(arg); break; +#ifdef WEBSOCKET + case 'R': + pArgs->restful = true; + break; + case 'E': + pArgs->dsn = arg; + pArgs->cloud = true; + break; + case 'T': + pArgs->timeout = atoi(arg); + break; +#endif case 'V': pArgs->is_version = true; break; @@ -208,7 +245,11 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { } if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' || - key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N') { + key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' +#ifdef WEBSOCKET + || key[1] == 'E' || key[1] == 'T' +#endif + ) { if (i + 1 >= argc) { fprintf(stderr, "option %s requires an argument\r\n", key); return -1; @@ -221,7 +262,11 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { shellParseSingleOpt(key[1], val); i++; } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' || key[1] == 'r' || key[1] == 'k' || - key[1] == 't' || key[1] == 'V' || key[1] == '?' || key[1] == 1) { + key[1] == 't' || key[1] == 'V' || key[1] == '?' || key[1] == 1 +#ifdef WEBSOCKET + ||key[1] == 'R' +#endif + ) { shellParseSingleOpt(key[1], NULL); } else { fprintf(stderr, "invalid option %s\r\n", key); diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index ce3718e001..1af6ee4c06 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -524,10 +524,8 @@ int32_t shellReadCommand(char *command) { c = taosGetConsoleChar(); switch (c) { case 'A': // Up arrow - if (hist_counter != pHistory->hstart) { - hist_counter = (hist_counter + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; - shellResetCommand(&cmd, (pHistory->hist[hist_counter] == NULL) ? "" : pHistory->hist[hist_counter]); - } + hist_counter = (hist_counter + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; + shellResetCommand(&cmd, (pHistory->hist[hist_counter] == NULL) ? "" : pHistory->hist[hist_counter]); break; case 'B': // Down arrow if (hist_counter != pHistory->hend) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 56bc1ed6cc..a2310ea9c9 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -25,14 +25,9 @@ static int32_t shellRunSingleCommand(char *command); static int32_t shellRunCommand(char *command); static void shellRunSingleCommandImp(char *command); static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision); -static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, - int32_t precision); static int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); static void shellPrintNChar(const char *str, int32_t length, int32_t width); -static void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision); static int32_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql); -static int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision); -static void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields); static int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql); static int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql); static void shellReadHistory(); @@ -94,8 +89,15 @@ int32_t shellRunSingleCommand(char *command) { shellSourceFile(c_ptr); return 0; } - - shellRunSingleCommandImp(command); +#ifdef WEBSOCKET + if (shell.args.restful || shell.args.cloud) { + shellRunSingleCommandWebsocketImp(command); + } else { +#endif + shellRunSingleCommandImp(command); +#ifdef WEBSOCKET + } +#endif return 0; } @@ -937,7 +939,16 @@ void *shellCancelHandler(void *arg) { taosMsleep(10); continue; } - taos_kill_query(shell.conn); + +#ifdef WEBSOCKET + if (shell.args.restful || shell.args.cloud) { + shell.stop_query = true; + } else { +#endif + taos_kill_query(shell.conn); +#ifdef WEBSOCKET + } +#endif #ifdef WINDOWS printf("\n%s", shell.info.promptHeader); #endif @@ -981,16 +992,26 @@ int32_t shellExecute() { fflush(stdout); SShellArgs *pArgs = &shell.args; - if (shell.args.auth == NULL) { - shell.conn = taos_connect(pArgs->host, pArgs->user, pArgs->password, pArgs->database, pArgs->port); +#ifdef WEBSOCKET + if (shell.args.restful || shell.args.cloud) { + if (shell_conn_ws_server(1)) { + return -1; + } } else { - shell.conn = taos_connect_auth(pArgs->host, pArgs->user, pArgs->auth, pArgs->database, pArgs->port); - } +#endif + if (shell.args.auth == NULL) { + shell.conn = taos_connect(pArgs->host, pArgs->user, pArgs->password, pArgs->database, pArgs->port); + } else { + shell.conn = taos_connect_auth(pArgs->host, pArgs->user, pArgs->auth, pArgs->database, pArgs->port); + } - if (shell.conn == NULL) { - fflush(stdout); - return -1; + if (shell.conn == NULL) { + fflush(stdout); + return -1; + } +#ifdef WEBSOCKET } +#endif shellReadHistory(); @@ -1005,8 +1026,16 @@ int32_t shellExecute() { if (pArgs->file[0] != 0) { shellSourceFile(pArgs->file); } +#ifdef WEBSOCKET + if (shell.args.restful || shell.args.cloud) { + ws_close(shell.ws_conn); + } else { +#endif + taos_close(shell.conn); +#ifdef WEBSOCKET + } +#endif - taos_close(shell.conn); shellWriteHistory(); shellCleanupHistory(); return 0; @@ -1026,10 +1055,15 @@ int32_t shellExecute() { taosSetSignal(SIGINT, shellQueryInterruptHandler); - shellGetGrantInfo(); - +#ifdef WEBSOCKET + if (!shell.args.restful && !shell.args.cloud) { +#endif + shellGetGrantInfo(); +#ifdef WEBSOCKET + } +#endif while (1) { - taosThreadCreate(&shell.pid, NULL, shellThreadLoop, shell.conn); + taosThreadCreate(&shell.pid, NULL, shellThreadLoop, NULL); taosThreadJoin(shell.pid, NULL); taosThreadClear(&shell.pid); } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 6672cee367..703533f8a9 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -19,6 +19,11 @@ SShellObj shell = {0}; int main(int argc, char *argv[]) { +#ifdef WEBSOCKET + shell.args.timeout = 10; + shell.args.cloud = true; +#endif + if (shellCheckIntSize() != 0) { return -1; } @@ -41,7 +46,9 @@ int main(int argc, char *argv[]) { shellPrintHelp(); return 0; } - +#ifdef WEBSOCKET + shellCheckConnectMode(); +#endif taos_init(); if (shell.args.is_dump_config) { diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index e96e3d3619..e5e61e0b24 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -121,6 +121,36 @@ void shellCheckServerStatus() { } } while (1); } +#ifdef WEBSOCKET +void shellCheckConnectMode() { + if (shell.args.dsn) { + shell.args.cloud = true; + shell.args.restful = false; + return; + } + if (shell.args.cloud) { + shell.args.dsn = getenv("TDENGINE_CLOUD_DSN"); + if (shell.args.dsn) { + shell.args.cloud = true; + shell.args.restful = false; + return; + } + if (shell.args.restful) { + if (!shell.args.host) { + shell.args.host = "localhost"; + } + if (!shell.args.port) { + shell.args.port = 6041; + } + shell.args.dsn = taosMemoryCalloc(1, 1024); + snprintf(shell.args.dsn, 1024, "ws://%s:%d/rest/ws", + shell.args.host, shell.args.port); + } + shell.args.cloud = false; + return; + } +} +#endif void shellExit() { if (shell.conn != NULL) { @@ -129,4 +159,4 @@ void shellExit() { } taos_cleanup(); exit(EXIT_FAILURE); -} \ No newline at end of file +} diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c new file mode 100644 index 0000000000..fee2325c34 --- /dev/null +++ b/tools/shell/src/shellWebsocket.c @@ -0,0 +1,260 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifdef WEBSOCKET +#include "taosws.h" +#include "shellInt.h" + +int shell_conn_ws_server(bool first) { + shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); + if (!shell.ws_conn) { + fprintf(stderr, "failed to connect %s, reason: %s\n", + shell.args.dsn, ws_errstr(NULL)); + return -1; + } + if (first && shell.args.restful) { + fprintf(stdout, "successfully connect to %s\n\n", + shell.args.dsn); + } else if (first && shell.args.cloud) { + fprintf(stdout, "successfully connect to cloud service\n"); + } + return 0; +} + +static int horizontalPrintWebsocket(WS_RES* wres) { + const void* data = NULL; + int rows; + ws_fetch_block(wres, &data, &rows); + if (!rows) { + return 0; + } + int num_fields = ws_field_count(wres); + TAOS_FIELD* fields = (TAOS_FIELD*)ws_fetch_fields(wres); + int precision = ws_result_precision(wres); + + int width[TSDB_MAX_COLUMNS]; + for (int col = 0; col < num_fields; col++) { + width[col] = shellCalcColWidth(fields + col, precision); + } + + shellPrintHeader(fields, width, num_fields); + + int numOfRows = 0; + do { + uint8_t ty; + uint32_t len; + for (int i = 0; i < rows; i++) { + for (int j = 0; j < num_fields; j++) { + putchar(' '); + const void *value = ws_get_value_in_block(wres, i, j, &ty, &len); + shellPrintField((const char*)value, fields+j, width[j], len, precision); + putchar(' '); + putchar('|'); + } + putchar('\r'); + putchar('\n'); + } + numOfRows += rows; + ws_fetch_block(wres, &data, &rows); + } while (rows && !shell.stop_query); + return numOfRows; +} + +static int verticalPrintWebsocket(WS_RES* wres) { + int rows = 0; + const void* data = NULL; + ws_fetch_block(wres, &data, &rows); + if (!rows) { + return 0; + } + int num_fields = ws_field_count(wres); + TAOS_FIELD* fields = (TAOS_FIELD*)ws_fetch_fields(wres); + int precision = ws_result_precision(wres); + + int maxColNameLen = 0; + for (int col = 0; col < num_fields; col++) { + int len = (int)strlen(fields[col].name); + if (len > maxColNameLen) { + maxColNameLen = len; + } + } + int numOfRows = 0; + do { + uint8_t ty; + uint32_t len; + for (int i = 0; i < rows; i++) { + printf("*************************** %d.row ***************************\n", + numOfRows + 1); + for (int j = 0; j < num_fields; j++) { + TAOS_FIELD* field = fields + j; + int padding = (int)(maxColNameLen - strlen(field->name)); + printf("%*.s%s: ", padding, " ", field->name); + const void *value = ws_get_value_in_block(wres, i, j, &ty, &len); + shellPrintField((const char*)value, field, 0, len, precision); + putchar('\n'); + } + numOfRows++; + } + ws_fetch_block(wres, &data, &rows); + } while (rows && !shell.stop_query); + return numOfRows; +} + +static int dumpWebsocketToFile(const char* fname, WS_RES* wres) { + char fullname[PATH_MAX] = {0}; + if (taosExpandDir(fname, fullname, PATH_MAX) != 0) { + tstrncpy(fullname, fname, PATH_MAX); + } + + TdFilePtr pFile = taosOpenFile(fullname, + TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (pFile == NULL) { + fprintf(stderr, "failed to open file: %s\r\n", fullname); + return -1; + } + int rows = 0; + const void* data = NULL; + ws_fetch_block(wres, &data, &rows); + if (!rows) { + taosCloseFile(&pFile); + return 0; + } + int numOfRows = 0; + TAOS_FIELD* fields = (TAOS_FIELD*)ws_fetch_fields(wres); + int num_fields = ws_field_count(wres); + int precision = ws_result_precision(wres); + for (int col = 0; col < num_fields; col++) { + if (col > 0) { + taosFprintfFile(pFile, ","); + } + taosFprintfFile(pFile, "%s", fields[col].name); + } + taosFprintfFile(pFile, "\r\n"); + do { + uint8_t ty; + uint32_t len; + numOfRows += rows; + for (int i = 0; i < rows; i++) { + for (int j = 0; j < num_fields; j++) { + if (j > 0) { + taosFprintfFile(pFile, ","); + } + const void *value = ws_get_value_in_block(wres, i, j, &ty, &len); + shellDumpFieldToFile(pFile, (const char*)value, fields + j, len, precision); + } + taosFprintfFile(pFile, "\r\n"); + } + ws_fetch_block(wres, &data, &rows); + } while (rows && !shell.stop_query); + taosCloseFile(&pFile); + return numOfRows; +} + +static int shellDumpWebsocket(WS_RES *wres, char *fname, int *error_no, bool vertical) { + int numOfRows = 0; + if (fname != NULL) { + numOfRows = dumpWebsocketToFile(fname, wres); + } else if (vertical) { + numOfRows = verticalPrintWebsocket(wres); + } else { + numOfRows = horizontalPrintWebsocket(wres); + } + *error_no = ws_errno(wres); + return numOfRows; +} + +void shellRunSingleCommandWebsocketImp(char *command) { + int64_t st, et; + char *sptr = NULL; + char *cptr = NULL; + char *fname = NULL; + bool printMode = false; + + if ((sptr = strstr(command, ">>")) != NULL) { + cptr = strstr(command, ";"); + if (cptr != NULL) { + *cptr = '\0'; + } + + fname = sptr + 2; + while (*fname == ' ') fname++; + *sptr = '\0'; + } + + if ((sptr = strstr(command, "\\G")) != NULL) { + cptr = strstr(command, ";"); + if (cptr != NULL) { + *cptr = '\0'; + } + + *sptr = '\0'; + printMode = true; // When output to a file, the switch does not work. + } + + if (!shell.ws_conn && shell_conn_ws_server(0)) { + return; + } + + shell.stop_query = false; + st = taosGetTimestampUs(); + + WS_RES* res = ws_query_timeout(shell.ws_conn, command, shell.args.timeout); + int code = ws_errno(res); + if (code != 0) { + et = taosGetTimestampUs(); + fprintf(stderr, "\nDB: error: %s (%.6fs)\n", ws_errstr(res), (et - st)/1E6); + if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { + fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); + } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { + fprintf(stderr, "TDengine server is down, will try to reconnect\n"); + shell.ws_conn = NULL; + } + ws_free_result(res); + return; + } + + if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { + fprintf(stdout, "Database changed.\r\n\r\n"); + fflush(stdout); + ws_free_result(res); + return; + } + + int numOfRows = 0; + if (ws_is_update_query(res)) { + numOfRows = ws_affected_rows(res); + et = taosGetTimestampUs(); + printf("Query Ok, %d of %d row(s) in database (%.6fs)\n", numOfRows, numOfRows, + (et - st)/1E6); + } else { + int error_no = 0; + numOfRows = shellDumpWebsocket(res, fname, &error_no, printMode); + if (numOfRows < 0) { + ws_free_result(res); + return; + } + et = taosGetTimestampUs(); + if (error_no == 0 && !shell.stop_query) { + printf("Query OK, %d row(s) in set (%.6fs)\n", numOfRows, + (et - st)/1E6); + } else { + printf("Query interrupted, %d row(s) in set (%.6fs)\n", numOfRows, + (et - st)/1E6); + } + } + printf("\n"); + ws_free_result(res); +} +#endif diff --git a/tools/taos-tools b/tools/taos-tools index 69b558ccbf..0b8a3373bb 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 69b558ccbfe54a4407fe23eeae2e67c540f59e55 +Subproject commit 0b8a3373bb7548f8106d13e7d3b0a988d3c4d48a diff --git a/tools/taosadapter b/tools/taosadapter index d8f19ede56..df8678f070 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit d8f19ede56f1f489c5d2ac8f963cced01e68ecef +Subproject commit df8678f070e3f707faf59baebec90065f6e1268b diff --git a/tools/taosws-rs b/tools/taosws-rs index c5fded266d..9de599dc52 160000 --- a/tools/taosws-rs +++ b/tools/taosws-rs @@ -1 +1 @@ -Subproject commit c5fded266d3b10508e38bf3285bb7ecf798bc343 +Subproject commit 9de599dc5293e9c90bc00bc4a03f8b91ba756bc3