diff --git a/.travis.yml b/.travis.yml index 2a0aa6372b..4d7a809e29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ branches: matrix: - os: linux - dist: bionic + dist: focal language: c git: @@ -28,8 +28,6 @@ matrix: - build-essential - cmake - net-tools - - python-pip - - python-setuptools - python3-pip - python3-setuptools - valgrind @@ -54,13 +52,19 @@ matrix: cd ${TRAVIS_BUILD_DIR}/debug make install > /dev/null || travis_terminate $? - pip install numpy - pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ pip3 install numpy pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ cd ${TRAVIS_BUILD_DIR}/tests ./test-all.sh smoke || travis_terminate $? + sleep 1 + + cd ${TRAVIS_BUILD_DIR}/tests/pytest + pkill -TERM -x taosd + fuser -k -n tcp 6030 + sleep 1 + ./crash_gen.sh -a -p -t 4 -s 25|| travis_terminate $? + sleep 1 cd ${TRAVIS_BUILD_DIR}/tests/pytest ./valgrind-test.sh 2>&1 > mem-error-out.log @@ -160,7 +164,7 @@ matrix: script: - cmake .. > /dev/null - - make > /dev/null + - make - os: linux dist: bionic diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index f687d7f244..4992692109 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -87,6 +87,16 @@ typedef struct SVgroupTableInfo { SArray* itemList; //SArray } SVgroupTableInfo; +static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) { + assert(pCmd != NULL && subClauseIndex >= 0 && subClauseIndex < TSDB_MAX_UNION_CLAUSE); + + if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) { + return NULL; + } + + return pCmd->pQueryInfo[subClauseIndex]; +} + int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); void tscDestroyDataBlock(STableDataBlocks* pDataBlock); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index da24e3691a..41464aa660 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -477,7 +477,13 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscDebug("%p redo parse sql string to build submit block", pSql); pCmd->parseFinished = false; - if ((code = tsParseSql(pSql, true)) == TSDB_CODE_SUCCESS) { + code = tsParseSql(pSql, true); + + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + return; + } + + if (code == TSDB_CODE_SUCCESS) { /* * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks, * and send the required submit block according to index value in supporter to server. diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 0917a3b19c..909338aa4a 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -340,13 +340,12 @@ bool stableQueryFunctChanged(int32_t funcId) { */ void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; } -void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable) { +void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) { assert(pResInfo->interResultBuf == NULL); pResInfo->bufLen = size; pResInfo->superTableQ = superTable; - - pResInfo->interResultBuf = calloc(1, (size_t)size); + pResInfo->interResultBuf = buf; } // set the query flag to denote that query is completed diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 4400ca2c25..95098bbab1 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1310,6 +1310,11 @@ int tsParseSql(SSqlObj *pSql, bool initial) { tscDebug("%p resume to parse sql: %s", pSql, pCmd->curSql); } + ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (TSDB_CODE_SUCCESS != ret) { + return ret; + } + if (tscIsInsertData(pSql->sqlstr)) { /* * Set the fp before parse the sql string, in case of getTableMeta failed, in which @@ -1326,11 +1331,6 @@ int tsParseSql(SSqlObj *pSql, bool initial) { ret = tsParseInsertSql(pSql); } else { - ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (TSDB_CODE_SUCCESS != ret) { - return ret; - } - SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr); ret = tscToSQLCmd(pSql, &SQLInfo); SQLInfoDestroy(&SQLInfo); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ed9822eea9..54c9cdea65 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1464,16 +1464,6 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { return pQueryInfo->pTableMetaInfo[tableIndex]; } -SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) { - assert(pCmd != NULL && subClauseIndex >= 0 && subClauseIndex < TSDB_MAX_UNION_CLAUSE); - - if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) { - return NULL; - } - - return pCmd->pQueryInfo[subClauseIndex]; -} - int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo) { int32_t ret = TSDB_CODE_SUCCESS; @@ -2097,7 +2087,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { } void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { - SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, columnIndex);//tscFieldInfoGetSupp(pFieldInfo, columnIndex); + SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, columnIndex); assert(pInfo->pSqlExpr != NULL); int32_t type = pInfo->pSqlExpr->resType; @@ -2112,7 +2102,7 @@ void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t column if (isNull(pData, type)) { pRes->tsrow[columnIndex] = NULL; } else { - pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE; + pRes->tsrow[columnIndex] = ((tstr*)pData)->data; } if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index d6a84b9cfe..e4ceb7b760 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -124,7 +124,7 @@ int32_t tsMnodeEqualVnodeNum = 4; int32_t tsEnableHttpModule = 1; int32_t tsRestRowLimit = 10240; uint16_t tsHttpPort = 6020; // only tcp, range tcp[6020] -int32_t tsHttpCacheSessions = 100; +int32_t tsHttpCacheSessions = 1000; int32_t tsHttpSessionExpire = 36000; int32_t tsHttpMaxThreads = 2; int32_t tsHttpEnableCompress = 0; diff --git a/src/connector/jdbc/buildTDengine.sh b/src/connector/jdbc/buildTDengine.sh deleted file mode 100755 index cf98215c85..0000000000 --- a/src/connector/jdbc/buildTDengine.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -ulimit -c unlimited - -function buildTDengine { - cd /root/TDengine - - git remote update - REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` - LOCAL_COMMIT=`git rev-parse --short @` - - echo " LOCAL: $LOCAL_COMMIT" - echo "REMOTE: $REMOTE_COMMIT" - if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then - echo "repo up-to-date" - else - echo "repo need to pull" - git pull - - LOCAL_COMMIT=`git rev-parse --short @` - cd /root/TDengine/debug - rm -rf /root/TDengine/debug/* - cmake .. - make > /dev/null - make install - fi -} - -function restartTaosd { - systemctl stop taosd - pkill -KILL -x taosd - sleep 10 - - logDir=`grep 'logDir' /etc/taos/taos.cfg|awk 'END{print $2}'` - dataDir=`grep 'dataDir' /etc/taos/taos.cfg|awk '{print $2}'` - - rm -rf $logDir/* - rm -rf $dataDir/* - - taosd 2>&1 > /dev/null & - sleep 10 -} - -buildTDengine -restartTaosd diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java new file mode 100644 index 0000000000..48290d3e62 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java @@ -0,0 +1,236 @@ +package com.taosdata.jdbc.utils; + +import java.io.File; +import java.util.*; +import java.util.concurrent.TimeUnit; + +public class TDNode { + + private int index; + private int running; + private int deployed; + private boolean testCluster; + private String path; + private String cfgDir; + private String dataDir; + private String logDir; + private String cfgPath; + + public TDNode(int index) { + this.index = index; + running = 0; + deployed = 0; + testCluster = false; + } + + public void setPath(String path) { + this.path = path; + } + + public void setTestCluster(boolean testCluster) { + this.testCluster = testCluster; + } + + public void searchTaosd(File dir, ArrayList taosdPath) { + File[] fileList = dir.listFiles(); + + if(fileList == null || fileList.length == 0) { + return; + } + + for(File file : fileList) { + if(file.isFile()) { + if(file.getName().equals("taosd")) { + taosdPath.add(file.getAbsolutePath()); + } + } else { + searchTaosd(file, taosdPath); + } + } + } + + public void start() { + String selfPath = System.getProperty("user.dir"); + String binPath = ""; + String projDir = selfPath + "/../../../../"; + + try { + ArrayList taosdPath = new ArrayList<>(); + + File dir = new File(projDir); + String realProjDir = dir.getCanonicalPath(); + dir = new File(realProjDir); + System.out.println("project Dir: " + projDir); + searchTaosd(dir, taosdPath); + + if(taosdPath.size() == 0) { + System.out.println("The project path doens't exist"); + return; + } else { + for(String p : taosdPath) { + if(!p.contains("packing")) { + binPath = p; + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + if(binPath.isEmpty()) { + System.out.println("taosd not found"); + return; + } else { + System.out.println("taosd found in " + binPath); + } + + if(this.deployed == 0) { + System.out.println("dnode" + index + "is not deployed"); + return; + } + + String cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & "; + System.out.println("start taosd cmd: " + cmd); + + try{ + Runtime.getRuntime().exec(cmd); + TimeUnit.SECONDS.sleep(5); + } catch (Exception e) { + e.printStackTrace(); + } + + this.running = 1; + } + + public void stop() { + String toBeKilled = "taosd"; + + if (this.running != 0) { + String killCmd = "pkill -kill -x " + toBeKilled; + String[] killCmds = {"sh", "-c", killCmd}; + try { + Runtime.getRuntime().exec(killCmds).waitFor(); + + for(int port = 6030; port < 6041; port ++) { + String fuserCmd = "fuser -k -n tcp " + port; + Runtime.getRuntime().exec(fuserCmd).waitFor(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + this.running = 0; + System.out.println("dnode:" + this.index + " is stopped by pkill"); + } + } + + public void startIP() { + try{ + String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " up"; + Runtime.getRuntime().exec(cmd).waitFor(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void stopIP() { + try{ + String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " down"; + Runtime.getRuntime().exec(cmd).waitFor(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void setCfgConfig(String option, String value) { + try{ + String cmd = "echo " + option + " " + value + " >> " + this.cfgPath; + String[] cmdLine = {"sh", "-c", cmd}; + Process ps = Runtime.getRuntime().exec(cmdLine); + ps.waitFor(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public String getDnodeRootDir() { + String dnodeRootDir = this.path + "/sim/psim/dnode" + this.index; + return dnodeRootDir; + } + + public String getDnodesRootDir() { + String dnodesRootDir = this.path + "/sim/psim" + this.index; + return dnodesRootDir; + } + + public void deploy() { + this.logDir = this.path + "/sim/dnode" + this.index + "/log"; + this.dataDir = this.path + "/sim/dnode" + this.index + "/data"; + this.cfgDir = this.path + "/sim/dnode" + this.index + "/cfg"; + this.cfgPath = this.path + "/sim/dnode" + this.index + "/cfg/taos.cfg"; + + try { + String cmd = "rm -rf " + this.logDir; + Runtime.getRuntime().exec(cmd).waitFor(); + + cmd = "rm -rf " + this.cfgDir; + Runtime.getRuntime().exec(cmd).waitFor(); + + cmd = "rm -rf " + this.dataDir; + Runtime.getRuntime().exec(cmd).waitFor(); + + cmd = "mkdir -p " + this.logDir; + Runtime.getRuntime().exec(cmd).waitFor(); + + cmd = "mkdir -p " + this.cfgDir; + Runtime.getRuntime().exec(cmd).waitFor(); + + cmd = "mkdir -p " + this.dataDir; + Runtime.getRuntime().exec(cmd).waitFor(); + + cmd = "touch " + this.cfgPath; + Runtime.getRuntime().exec(cmd).waitFor(); + } catch (Exception e) { + e.printStackTrace(); + } + + if(this.testCluster) { + startIP(); + setCfgConfig("masterIp", "192.168.0.1"); + setCfgConfig("secondIp", "192.168.0.2"); + setCfgConfig("publicIp", "192.168.0." + this.index); + setCfgConfig("internalIp", "192.168.0." + this.index); + setCfgConfig("privateIp", "192.168.0." + this.index); + } + setCfgConfig("dataDir", this.dataDir); + setCfgConfig("logDir", this.logDir); + setCfgConfig("numOfLogLines", "1000000/00"); + setCfgConfig("mnodeEqualVnodeNum", "0"); + setCfgConfig("walLevel", "1"); + setCfgConfig("statusInterval", "1"); + setCfgConfig("numOfTotalVnodes", "64"); + setCfgConfig("numOfMnodes", "3"); + setCfgConfig("numOfThreadsPerCore", "2.0"); + setCfgConfig("monitor", "0"); + setCfgConfig("maxVnodeConnections", "30000"); + setCfgConfig("maxMgmtConnections", "30000"); + setCfgConfig("maxMeterConnections", "30000"); + setCfgConfig("maxShellConns", "30000"); + setCfgConfig("locale", "en_US.UTF-8"); + setCfgConfig("charset", "UTF-8"); + setCfgConfig("asyncLog", "0"); + setCfgConfig("anyIp", "0"); + setCfgConfig("dDebugFlag", "135"); + setCfgConfig("mDebugFlag", "135"); + setCfgConfig("sdbDebugFlag", "135"); + setCfgConfig("rpcDebugFlag", "135"); + setCfgConfig("tmrDebugFlag", "131"); + setCfgConfig("cDebugFlag", "135"); + setCfgConfig("httpDebugFlag", "135"); + setCfgConfig("monitorDebugFlag", "135"); + setCfgConfig("udebugFlag", "135"); + setCfgConfig("jnidebugFlag", "135"); + setCfgConfig("qdebugFlag", "135"); + this.deployed = 1; + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java new file mode 100644 index 0000000000..ea15ae9863 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java @@ -0,0 +1,94 @@ +package com.taosdata.jdbc.utils; + +import java.io.File; +import java.util.*; + +public class TDNodes { + private ArrayList tdNodes; + private boolean testCluster; + + public TDNodes () { + tdNodes = new ArrayList<>(); + for(int i = 1; i < 11; i ++) { + tdNodes.add(new TDNode(i)); + } + } + + public void setPath(String path) { + try { + String killCmd = "pkill -kill -x taosd"; + String[] killCmds = {"sh", "-c", killCmd}; + Runtime.getRuntime().exec(killCmds).waitFor(); + + String binPath = System.getProperty("user.dir"); + binPath += "/../../../debug"; + System.out.println("binPath: " + binPath); + + File file = new File(path); + binPath = file.getCanonicalPath(); + System.out.println("binPath real path: " + binPath); + + if(path.isEmpty()){ + file = new File(path + "/../../"); + path = file.getCanonicalPath(); + } + + for(int i = 0; i < tdNodes.size(); i++) { + tdNodes.get(i).setPath(path); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void setTestCluster(boolean testCluster) { + this.testCluster = testCluster; + } + + public void check(int index) { + if(index < 1 || index > 10) { + System.out.println("index: " + index + " should on a scale of [1, 10]"); + return; + } + } + + public void deploy(int index) { + try { + File file = new File(System.getProperty("user.dir") + "/../../../"); + String projectRealPath = file.getCanonicalPath(); + check(index); + tdNodes.get(index - 1).setTestCluster(this.testCluster); + tdNodes.get(index - 1).setPath(projectRealPath); + tdNodes.get(index - 1).deploy(); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("deploy Test Exception"); + } + } + + public void cfg(int index, String option, String value) { + check(index); + tdNodes.get(index - 1).setCfgConfig(option, value); + } + + public void start(int index) { + check(index); + tdNodes.get(index - 1).start(); + } + + public void stop(int index) { + check(index); + tdNodes.get(index - 1).stop(); + } + + public void startIP(int index) { + check(index); + tdNodes.get(index - 1).startIP(); + } + + public void stopIP(int index) { + check(index); + tdNodes.get(index - 1).stopIP(); + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java index fd9ab49c49..5f105fb782 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java @@ -1,27 +1,38 @@ package com.taosdata.jdbc; -import java.io.BufferedReader; -import java.io.InputStreamReader; +import java.io.File; +import com.taosdata.jdbc.utils.TDNodes; +import org.junit.AfterClass; import org.junit.BeforeClass; public class BaseTest { + + private static boolean testCluster = false; + private static String deployPath = System.getProperty("user.dir"); + private static TDNodes tdNodes = new TDNodes(); + @BeforeClass public static void setupEnv() { try{ - String path = System.getProperty("user.dir"); - String bashPath = path + "/buildTDengine.sh"; + File file = new File(deployPath + "/../../../"); + String rootPath = file.getCanonicalPath(); + + tdNodes.setPath(rootPath); + tdNodes.setTestCluster(testCluster); - Process ps = Runtime.getRuntime().exec(bashPath); - ps.waitFor(); + tdNodes.deploy(1); + tdNodes.start(1); - BufferedReader br = new BufferedReader(new InputStreamReader(ps.getInputStream())); - while(br.readLine() != null) { - System.out.println(br.readLine()); - } } catch (Exception e) { e.printStackTrace(); + System.out.println("Base Test Exception"); } } + + @AfterClass + public static void cleanUpEnv() { + tdNodes.stop(1); + } } \ No newline at end of file diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h index 826f4ff1c1..092c06d84b 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/dnode/inc/dnodeMgmt.h @@ -22,6 +22,8 @@ extern "C" { int32_t dnodeInitMgmt(); void dnodeCleanupMgmt(); +int32_t dnodeInitMgmtTimer(); +void dnodeCleanupMgmtTimer(); void dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg); void* dnodeGetVnode(int32_t vgId); diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 8d1ae0a50e..987a189959 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -57,6 +57,7 @@ static const SDnodeComponent tsDnodeComponents[] = { {"server", dnodeInitServer, dnodeCleanupServer}, {"mgmt", dnodeInitMgmt, dnodeCleanupMgmt}, {"modules", dnodeInitModules, dnodeCleanupModules}, + {"mgmt-tmr",dnodeInitMgmtTimer, dnodeCleanupMgmtTimer}, {"shell", dnodeInitShell, dnodeCleanupShell} }; diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index e8da87bd5f..d0c42c4412 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -147,6 +147,12 @@ int32_t dnodeInitMgmt() { return -1; } + dInfo("dnode mgmt is initialized"); + + return TSDB_CODE_SUCCESS; +} + +int32_t dnodeInitMgmtTimer() { tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM"); if (tsDnodeTmr == NULL) { dError("failed to init dnode timer"); @@ -155,13 +161,11 @@ int32_t dnodeInitMgmt() { } taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer); - - dInfo("dnode mgmt is initialized"); - + dInfo("dnode mgmt timer is initialized"); return TSDB_CODE_SUCCESS; } -void dnodeCleanupMgmt() { +void dnodeCleanupMgmtTimer() { if (tsStatusTimer != NULL) { taosTmrStopA(&tsStatusTimer); tsStatusTimer = NULL; @@ -171,7 +175,10 @@ void dnodeCleanupMgmt() { taosTmrCleanUp(tsDnodeTmr); tsDnodeTmr = NULL; } +} +void dnodeCleanupMgmt() { + dnodeCleanupMgmtTimer(); dnodeCloseVnodes(); if (tsMgmtQset) taosQsetThreadResume(tsMgmtQset); diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 01f0cf25c0..971bd0a110 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -20,6 +20,7 @@ #include "tglobal.h" #include "dnodeInt.h" #include "dnodeMain.h" +#include "tfile.h" static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static sem_t exitSem; @@ -67,6 +68,18 @@ int32_t main(int32_t argc, char *argv[]) { taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true); } } +#endif +#ifdef TAOS_RANDOM_FILE_FAIL + else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { + if ( (i+1) < argc ) { + int factor = atoi(argv[i+1]); + printf("The factor of random failure is %d\n", factor); + taosSetRandomFileFailFactor(factor); + } else { + printf("Please specify a number for random failure factor!"); + exit(EXIT_FAILURE); + } + } #endif } diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index db7c35fe2d..042e356442 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -41,7 +41,7 @@ typedef struct { void (*cleanup)(); } SMnodeComponent; -void *tsMnodeTmr; +void *tsMnodeTmr = NULL; static bool tsMgmtIsRunning = false; static const SMnodeComponent tsMnodeComponents[] = { diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index c4a2d12e3d..cdcb7357f1 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -471,8 +471,8 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { atomic_add_fetch_32(&pTable->autoIndex, 1); } - sdbDebug("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 " ver:%" PRIu64 ", msg:%p", pTable->tableName, - sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion(), pOper->pMsg); + sdbDebug("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 ", msg:%p", pTable->tableName, + sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, pOper->pMsg); (*pTable->insertFp)(pOper); return TSDB_CODE_SUCCESS; @@ -490,8 +490,8 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { taosHashRemove(pTable->iHandle, key, keySize); atomic_sub_fetch_32(&pTable->numOfRows, 1); - sdbDebug("table:%s, delete record:%s from hash, numOfRows:%" PRId64 " ver:%" PRIu64 ", msg:%p", pTable->tableName, - sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion(), pOper->pMsg); + sdbDebug("table:%s, delete record:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, + sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; *updateEnd = 1; @@ -501,8 +501,8 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { } static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { - sdbDebug("table:%s, update record:%s in hash, numOfRows:%" PRId64 " ver:%" PRIu64 ", msg:%p", pTable->tableName, - sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion(), pOper->pMsg); + sdbDebug("table:%s, update record:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, + sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); (*pTable->updateFp)(pOper); return TSDB_CODE_SUCCESS; @@ -967,7 +967,11 @@ static void *sdbWorkerFp(void *param) { } int32_t code = sdbWrite(pOper, pHead, type); - if (pOper && code <= 0) pOper->retCode = code; + if (code > 0) code = 0; + if (pOper) + pOper->retCode = code; + else + pHead->len = code; // hackway } walFsync(tsSdbObj.wal); @@ -982,7 +986,8 @@ static void *sdbWorkerFp(void *param) { sdbDecRef(pOper->table, pOper->pObj); sdbConfirmForward(NULL, pOper, pOper->retCode); } else if (type == TAOS_QTYPE_FWD) { - syncConfirmForward(tsSdbObj.sync, pHead->version, TSDB_CODE_SUCCESS); + pHead = (SWalHead *)item; + syncConfirmForward(tsSdbObj.sync, pHead->version, pHead->len); taosFreeQitem(item); } else { taosFreeQitem(item); diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 1b958e72e6..a56ad34a25 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -310,7 +310,8 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); - return TSDB_CODE_MND_DB_IN_DROPPING; + code = TSDB_CODE_MND_DB_IN_DROPPING; + goto connect_over; } mnodeDecDbRef(pDb); } @@ -355,7 +356,7 @@ static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { int32_t code = TSDB_CODE_SUCCESS; if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pUseDbMsg->db); if (pMsg->pDb == NULL) { - code = TSDB_CODE_MND_INVALID_DB; + return TSDB_CODE_MND_INVALID_DB; } if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 7f60001b24..0715f182b1 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -470,7 +470,6 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { pVgroup->vnodeGid[i].dnodeId); } - mnodeIncVgroupRef(pVgroup); pMsg->expected = pVgroup->numOfVnodes; mnodeSendCreateVgroupMsg(pVgroup, pMsg); @@ -492,6 +491,9 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) { return TSDB_CODE_MND_NO_ENOUGH_DNODES; } + pMsg->pVgroup = pVgroup; + mnodeIncVgroupRef(pVgroup); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, @@ -501,8 +503,6 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) { .cb = mnodeCreateVgroupCb }; - pMsg->pVgroup = pVgroup; - int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { pMsg->pVgroup = NULL; @@ -925,19 +925,20 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) { mDebug("dnode:%s, vgId:%d, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId); return TSDB_CODE_MND_VGROUP_NOT_EXIST; } - mnodeDecDnodeRef(pDnode); SVgObj *pVgroup = mnodeGetVgroup(pCfg->vgId); if (pVgroup == NULL) { mDebug("dnode:%s, vgId:%d, no vgroup info", taosIpStr(pCfg->dnodeId), pCfg->vgId); + mnodeDecDnodeRef(pDnode); return TSDB_CODE_MND_VGROUP_NOT_EXIST; } - mnodeDecVgroupRef(pVgroup); mDebug("vgId:%d, send create vnode msg to dnode %s for vnode cfg msg", pVgroup->vgId, pDnode->dnodeEp); SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); mnodeSendCreateVnodeMsg(pVgroup, &ipSet, NULL); + mnodeDecDnodeRef(pDnode); + mnodeDecVgroupRef(pVgroup); return TSDB_CODE_SUCCESS; } diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 5d94e8456e..8ca1c2ff11 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -206,7 +206,7 @@ typedef struct HttpThread { pthread_mutex_t threadMutex; bool stop; int pollFd; - int numOfFds; + int numOfContexts; int threadId; char label[HTTP_LABEL_SIZE]; bool (*processData)(HttpContext *pContext); diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index cdaee53c38..98fba9cb3b 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -44,7 +44,7 @@ static void httpDestroyContext(void *data) { HttpThread *pThread = pContext->pThread; httpRemoveContextFromEpoll(pContext); httpReleaseSession(pContext); - atomic_sub_fetch_32(&pThread->numOfFds, 1); + atomic_sub_fetch_32(&pThread->numOfContexts, 1); pContext->pThread = 0; pContext->state = HTTP_CONTEXT_STATE_CLOSED; @@ -171,38 +171,39 @@ bool httpInitContext(HttpContext *pContext) { void httpCloseContextByApp(HttpContext *pContext) { pContext->parsed = false; - bool keepAlive = true; + if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) { keepAlive = false; } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) { keepAlive = false; - } else {} + } else { + } if (keepAlive) { if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) { - httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse connect", - pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd, + pContext->ipstr); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) { httpRemoveContextFromEpoll(pContext); - httpDebug("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect", - pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect", pContext, pContext->fd, + pContext->ipstr); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) { - httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse connect", - pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd, + pContext->ipstr); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { httpRemoveContextFromEpoll(pContext); - httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect", - pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect", pContext, pContext->fd, + pContext->ipstr); } else { httpRemoveContextFromEpoll(pContext); - httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect", - pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); + httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd, + pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); } } else { httpRemoveContextFromEpoll(pContext); - httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close connect", - pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); + httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close context", pContext, pContext->fd, + pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); } httpReleaseContext(pContext); @@ -214,7 +215,7 @@ void httpCloseContextByServer(HttpContext *pContext) { } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) { httpDebug("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) { - httpDebug("context:%p, fd:%d, ip:%s, epoll finished, close context", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, ip:%s, epoll finished, close connect", pContext, pContext->fd, pContext->ipstr); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { httpDebug("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr); } else { diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index a89ea7d8f1..2c94f61950 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -313,9 +313,9 @@ bool httpParseRequest(HttpContext* pContext) { return true; } - httpTraceL("context:%p, fd:%d, ip:%s, thread:%s, numOfFds:%d, read size:%d, raw data:\n%s", pContext, pContext->fd, - pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfFds, pContext->parser.bufsize, - pContext->parser.buffer); + httpTraceL("context:%p, fd:%d, ip:%s, thread:%s, numOfContexts:%d, read size:%d, raw data:\n%s", pContext, + pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfContexts, + pContext->parser.bufsize, pContext->parser.buffer); if (!httpGetHttpMethod(pContext)) { return false; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index d7d7da6668..dbe299cef7 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -293,7 +293,7 @@ static void *httpAcceptHttpConnection(void *arg) { totalFds = 1; for (int i = 0; i < pServer->numOfThreads; ++i) { - totalFds += pServer->pThreads[i].numOfFds; + totalFds += pServer->pThreads[i].numOfContexts; } if (totalFds > tsHttpCacheSessions * 100) { @@ -332,9 +332,9 @@ static void *httpAcceptHttpConnection(void *arg) { } // notify the data process, add into the FdObj list - atomic_add_fetch_32(&pThread->numOfFds, 1); - httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfFds:%d totalFds:%d, accept a new connection", pContext, connFd, - pContext->ipstr, pThread->label, pThread->numOfFds, totalFds); + atomic_add_fetch_32(&pThread->numOfContexts, 1); + httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfContexts:%d totalFds:%d, accept a new connection", pContext, + connFd, pContext->ipstr, pThread->label, pThread->numOfContexts, totalFds); // pick up next thread for next connection threadId++; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 73124b8fc6..6cd6edd6db 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -172,6 +172,7 @@ typedef struct SQueryRuntimeEnv { bool topBotQuery; // false bool groupbyNormalCol; // denote if this is a groupby normal column query bool hasTagResults; // if there are tag values in final result or not + int32_t interBufSize; // intermediate buffer sizse int32_t prevGroupId; // previous executed group id SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file } SQueryRuntimeEnv; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 1ecca102ba..9b818b367f 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -15,6 +15,8 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +int32_t getOutputInterResultBufSize(SQuery* pQuery); + void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes); void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src); @@ -35,7 +37,7 @@ SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot); #define curTimeWindow(_winres) ((_winres)->curIndex) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); -void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo); +void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize); char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult); diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 340f6bc4f3..63b7abb379 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -272,7 +272,7 @@ bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *mi bool stableQueryFunctChanged(int32_t funcId); void resetResultInfo(SResultInfo *pResInfo); -void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable); +void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf); static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = true; // the this struct has been initialized flag diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3d7f6f95a0..53a32a2356 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -123,6 +123,14 @@ static void setQueryStatus(SQuery *pQuery, int8_t status); #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->intervalTime > 0) +// previous time window may not be of the same size of pQuery->intervalTime +#define GET_NEXT_TIMEWINDOW(_q, tw) \ + do { \ + int32_t factor = GET_FORWARD_DIRECTION_FACTOR((_q)->order.order); \ + (tw)->skey += ((_q)->slidingTime * factor); \ + (tw)->ekey = (tw)->skey + ((_q)->intervalTime - 1); \ + } while (0) + // todo move to utility static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); @@ -130,7 +138,6 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); -static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, SDataStatis *pStatis, void *param, int32_t colIndex); @@ -419,7 +426,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { SPosInfo pos = {-1, -1}; - createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &pos); + createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &pos, pRuntimeEnv->interBufSize); } pWindowResInfo->capacity = newCap; } @@ -551,19 +558,29 @@ static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int static int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, int16_t order, int64_t *pData) { - int32_t endPos = searchFn((char *)pData, numOfRows, ekey, order); int32_t forwardStep = 0; - if (endPos >= 0) { - forwardStep = (order == TSDB_ORDER_ASC) ? (endPos - pos) : (pos - endPos); - assert(forwardStep >= 0); + if (order == TSDB_ORDER_ASC) { + int32_t end = searchFn((char*) &pData[pos], numOfRows - pos, ekey, order); + if (end >= 0) { + forwardStep = end; - // endPos data is equalled to the key so, we do need to read the element in endPos - if (pData[endPos] == ekey) { - forwardStep += 1; + if (pData[end + pos] == ekey) { + forwardStep += 1; + } + } + } else { + int32_t end = searchFn((char *)pData, pos + 1, ekey, order); + if (end >= 0) { + forwardStep = pos - end; + + if (pData[end] == ekey) { + forwardStep += 1; + } } } + assert(forwardStep > 0); return forwardStep; } @@ -686,7 +703,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo } } - assert(num >= 0); + assert(num > 0); return num; } @@ -736,59 +753,60 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus } } -static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNextWin, - SDataBlockInfo *pDataBlockInfo, TSKEY *primaryKeys, - __block_search_fn_t searchFn) { +static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, + TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { SQuery *pQuery = pRuntimeEnv->pQuery; - // tumbling time window query, a special case of sliding time window query - if (pQuery->slidingTime == pQuery->intervalTime) { - // todo opt - } - - getNextTimeWindow(pQuery, pNextWin); + GET_NEXT_TIMEWINDOW(pQuery, pNext); // next time window is not in current block - if ((pNextWin->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (pNextWin->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (pNext->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { return -1; } TSKEY startKey = -1; if (QUERY_IS_ASC_QUERY(pQuery)) { - startKey = pNextWin->skey; + startKey = pNext->skey; if (startKey < pQuery->window.skey) { startKey = pQuery->window.skey; } } else { - startKey = pNextWin->ekey; + startKey = pNext->ekey; if (startKey > pQuery->window.skey) { startKey = pQuery->window.skey; } } - int32_t startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQuery->order.order); + int32_t startPos = 0; + // tumbling time window query, a special case of sliding time window query + if (pQuery->slidingTime == pQuery->intervalTime && prevPosition != -1) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + startPos = prevPosition + factor; + } else { + startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQuery->order.order); + } /* * This time window does not cover any data, try next time window, * this case may happen when the time window is too small */ - if (QUERY_IS_ASC_QUERY(pQuery) && primaryKeys[startPos] > pNextWin->ekey) { + if (QUERY_IS_ASC_QUERY(pQuery) && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; - pNextWin->ekey += ((next - pNextWin->ekey + pQuery->slidingTime - 1)/pQuery->slidingTime) * pQuery->slidingTime; - pNextWin->skey = pNextWin->ekey - pQuery->intervalTime + 1; - } else if ((!QUERY_IS_ASC_QUERY(pQuery)) && primaryKeys[startPos] < pNextWin->skey) { + pNext->ekey += ((next - pNext->ekey + pQuery->slidingTime - 1)/pQuery->slidingTime) * pQuery->slidingTime; + pNext->skey = pNext->ekey - pQuery->intervalTime + 1; + } else if ((!QUERY_IS_ASC_QUERY(pQuery)) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; - pNextWin->skey -= ((pNextWin->skey - next + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; - pNextWin->ekey = pNextWin->skey + pQuery->intervalTime - 1; + pNext->skey -= ((pNext->skey - next + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; + pNext->ekey = pNext->skey + pQuery->intervalTime - 1; } return startPos; } -static TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { +static FORCE_INLINE TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { TSKEY ekey = -1; if (QUERY_IS_ASC_QUERY(pQuery)) { ekey = pWindow->ekey; @@ -924,20 +942,23 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * return; } + int32_t forwardStep = 0; + int32_t startPos = pQuery->pos; + if (hasTimeWindow) { TSKEY ekey = reviseWindowEkey(pQuery, &win); - int32_t forwardStep = - getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); + forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, tsCols, pDataBlockInfo->rows); + doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; while (1) { - int32_t startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, tsCols, searchFn); + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, tsCols, searchFn, prevEndPos); if (startPos < 0) { break; } @@ -953,7 +974,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); - int32_t forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); + forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); SWindowStatus* pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); @@ -1224,7 +1245,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS int32_t index = pWindowResInfo->curIndex; while (1) { - getNextTimeWindow(pQuery, &nextWin); + GET_NEXT_TIMEWINDOW(pQuery, &nextWin); if (/*pWindowResInfo->startTime > nextWin.skey ||*/ (nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || (nextWin.skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { @@ -1236,7 +1257,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } // null data, failed to allocate more memory buffer - bool hasTimeWindow = false; + hasTimeWindow = false; if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { break; } @@ -1459,11 +1480,13 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p } } -static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery) { +static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) { + char* p = buf; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - assert(pQuery->pSelectExpr[i].interBytes <= DEFAULT_INTERN_BUF_PAGE_SIZE); - - setResultInfoBuf(&pResultInfo[i], pQuery->pSelectExpr[i].interBytes, isStableQuery); + int32_t size = pQuery->pSelectExpr[i].interBytes; + setResultInfoBuf(&pResultInfo[i], size, isStableQuery, p); + + p += size; } } @@ -1542,8 +1565,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } } + char* buf = calloc(1, pRuntimeEnv->interBufSize); + // set the intermediate result output buffer - setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery); + setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf); // if it is group by normal column, do not set output buffer, the output buffer is pResult if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !pRuntimeEnv->stableQuery) { @@ -1581,9 +1606,9 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tVariantDestroy(&pCtx->tag); tfree(pCtx->tagInfo.pTagCtxList); - tfree(pRuntimeEnv->resultInfo[i].interResultBuf); } + tfree(pRuntimeEnv->resultInfo[0].interResultBuf); tfree(pRuntimeEnv->resultInfo); tfree(pRuntimeEnv->pCtx); } @@ -2017,14 +2042,6 @@ static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFun return true; } -// previous time window may not be of the same size of pQuery->intervalTime -static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - pTimeWindow->skey += (pQuery->slidingTime * factor); - pTimeWindow->ekey = pTimeWindow->skey + (pQuery->intervalTime - 1); -} - SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis) { SQuery *pQuery = pRuntimeEnv->pQuery; @@ -2737,7 +2754,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery); + char* buf = calloc(1, pRuntimeEnv->interBufSize); + setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); int64_t lastTimestamp = -1; @@ -2823,11 +2841,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { tfree(pTree); pQInfo->offset = 0; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - tfree(pResultInfo[i].interResultBuf); - } tfree(pResultInfo); + tfree(buf); return pQInfo->numOfGroupResultPages; } @@ -2980,14 +2996,16 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { } } -void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo) { +void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize) { int32_t numOfCols = pQuery->numOfOutput; pResultRow->resultInfo = calloc((size_t)numOfCols, sizeof(SResultInfo)); pResultRow->pos = *posInfo; + char* buf = calloc(1, interBufSize); + // set the intermediate result output buffer - setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery); + setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery, buf); } void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { @@ -3365,7 +3383,7 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void // set more initial size of interval/groupby query if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { - int32_t initialSize = 20; + int32_t initialSize = 16; int32_t initialThreshold = 100; initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); } else { // in other aggregate query, do not initialize the windowResInfo @@ -3591,20 +3609,7 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { return loadPrimaryTS; } -static int32_t getNumOfSubset(SQInfo *pQInfo) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - int32_t totalSubset = 0; - if (pQInfo->runtimeEnv.groupbyNormalCol || (QUERY_IS_INTERVAL_QUERY(pQuery))) { - totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); - } else { - totalSubset = GET_NUM_OF_TABLEGROUP(pQInfo); - } - - return totalSubset; -} - -static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orderType) { +static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_t orderType) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -3613,17 +3618,18 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde int32_t step = -1; qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); - int32_t totalSubset = getNumOfSubset(pQInfo); + int32_t totalSet = numOfClosedTimeWindow(pResultInfo); + SWindowResult* result = pResultInfo->pResult; if (orderType == TSDB_ORDER_ASC) { startIdx = pQInfo->groupIndex; step = 1; } else { // desc order copy all data - startIdx = totalSubset - pQInfo->groupIndex - 1; + startIdx = totalSet - pQInfo->groupIndex - 1; step = -1; } - for (int32_t i = startIdx; (i < totalSubset) && (i >= 0); i += step) { + for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { if (result[i].numOfRows == 0) { pQInfo->offset = 0; pQInfo->groupIndex += 1; @@ -3678,11 +3684,11 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde * @param pQInfo * @param result */ -void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { +void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); + int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo, orderType); pQuery->rec.rows += numOfResult; @@ -4013,7 +4019,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { } STimeWindow tw = win; - getNextTimeWindow(pQuery, &tw); + GET_NEXT_TIMEWINDOW(pQuery, &tw); if (pQuery->limit.offset == 0) { if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || @@ -4025,7 +4031,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { tw = win; int32_t startPos = - getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey); + getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); assert(startPos >= 0); // set the abort info @@ -4068,7 +4074,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { tw = win; int32_t startPos = - getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey); + getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); assert(startPos >= 0); // set the abort info @@ -4197,7 +4203,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_INT; // group id } - initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 32, 4096, type); } } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { @@ -4469,6 +4475,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo); + taosArrayDestroy(g1); + taosArrayDestroy(tx); SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); @@ -4505,7 +4513,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQInfo->groupIndex = 0; ensureOutputBufferSimple(pRuntimeEnv, pWindowResInfo->size); - copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult); + copyFromWindowResToSData(pQInfo, pWindowResInfo); pQInfo->groupIndex = currentGroupIndex; //restore the group index assert(pQuery->rec.rows == pWindowResInfo->size); @@ -4520,7 +4528,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { * we need to return it to client in the first place. */ if (pQInfo->groupIndex > 0) { - copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); pQuery->rec.total += pQuery->rec.rows; if (pQuery->rec.rows > 0) { @@ -4721,7 +4729,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif } else { - copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); } qDebug("QInfo:%p current:%"PRId64", total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total); @@ -4772,7 +4780,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { #endif } } else { // not a interval query - copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); } // handle the limitation of output buffer @@ -4927,7 +4935,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) { pQInfo->groupIndex = 0; // always start from 0 pQuery->rec.rows = 0; - copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); } @@ -4956,7 +4964,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { if (pRuntimeEnv->groupbyNormalCol) { // todo refactor with merge interval time result pQInfo->groupIndex = 0; pQuery->rec.rows = 0; - copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); } @@ -4988,7 +4996,7 @@ static void tableQueryImpl(SQInfo *pQInfo) { pQInfo->groupIndex = 0; // always start from 0 if (pRuntimeEnv->windowResInfo.size > 0) { - copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); if (pQuery->rec.rows > 0) { @@ -5736,7 +5744,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, STimeWindow window = pQueryMsg->window; taosArraySort(pTableIdList, compareTableIdInfo); - // TODO optimize the STableQueryInfo malloc strategy + pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); int32_t index = 0; @@ -5851,6 +5859,18 @@ _error: return code; } +static void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { + if (pFilter == NULL) { + return; + } + for (int32_t i = 0; i < numOfFilters; i++) { + if (pFilter[i].filterstr) { + free((void*)(pFilter[i].pz)); + } + } + free(pFilter); +} + static void freeQInfo(SQInfo *pQInfo) { if (!isValidQInfo(pQInfo)) { return; @@ -5919,7 +5939,15 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->tagColList); tfree(pQuery->pFilterInfo); - tfree(pQuery->colList); + + if (pQuery->colList != NULL) { + for (int32_t i = 0; i < pQuery->numOfCols; i++) { + SColumnInfo* column = pQuery->colList + i; + freeColumnFilterInfo(column->filters, column->numOfFilters); + } + tfree(pQuery->colList); + } + tfree(pQuery->sdata); tfree(pQuery); @@ -6116,6 +6144,11 @@ _over: free(pExprMsg); taosArrayDestroy(pTableIdList); + for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { + SColumnInfo* column = pQueryMsg->colList + i; + freeColumnFilterInfo(column->filters, column->numOfFilters); + } + //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null; if (code != TSDB_CODE_SUCCESS) { *pQInfo = NULL; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 03574388b7..dce2c24ea0 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -17,15 +17,24 @@ #include "hash.h" #include "taosmsg.h" -#include "qextbuffer.h" -#include "ttime.h" - -#include "qfill.h" #include "ttime.h" #include "qExecutor.h" #include "qUtil.h" +int32_t getOutputInterResultBufSize(SQuery* pQuery) { + int32_t size = 0; + + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + assert(pQuery->pSelectExpr[i].interBytes <= DEFAULT_INTERN_BUF_PAGE_SIZE); + size += pQuery->pSelectExpr[i].interBytes; + } + + assert(size > 0); + + return size; +} + int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size, int32_t threshold, int16_t type) { pWindowResInfo->capacity = size; @@ -43,7 +52,7 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { SPosInfo posInfo = {-1, -1}; - createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo); + createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo, pRuntimeEnv->interBufSize); } return TSDB_CODE_SUCCESS; @@ -54,11 +63,7 @@ void destroyTimeWindowRes(SWindowResult *pWindowRes, int32_t nOutputCols) { return; } - // TODO opt malloc strategy - for (int32_t i = 0; i < nOutputCols; ++i) { - free(pWindowRes->resultInfo[i].interResultBuf); - } - + free(pWindowRes->resultInfo[0].interResultBuf); free(pWindowRes->resultInfo); } @@ -241,10 +246,9 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow } pWindowRes->numOfRows = 0; - // pWindowRes->nAlloc = 0; pWindowRes->pos = (SPosInfo){-1, -1}; pWindowRes->status.closed = false; - pWindowRes->window = (STimeWindow){0, 0}; + pWindowRes->window = TSWINDOW_INITIALIZER; } /** @@ -254,7 +258,6 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow */ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { dst->numOfRows = src->numOfRows; - // dst->nAlloc = src->nAlloc; dst->window = src->window; dst->status = src->status; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 0d5b80e8f3..5d67d5e615 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1366,6 +1366,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { } int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); + tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); /* * only the compressed size is less than the value of contLen - overhead, the compression is applied @@ -1378,7 +1379,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { memcpy(pCont + overhead, buf, compLen); pHead->comp = 1; - //tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); + tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); finalLen = compLen + overhead; } else { finalLen = contLen; diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index 5bddc76266..04e500743c 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -18,6 +18,7 @@ #ifdef TAOS_RANDOM_FILE_FAIL +void taosSetRandomFileFailFactor(int factor); ssize_t taos_tread(int fd, void *buf, size_t count); ssize_t taos_twrite(int fd, void *buf, size_t count); off_t taos_lseek(int fd, off_t offset, int whence); diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index eb7a2d5a66..92eeaef126 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -26,40 +26,51 @@ #include "os.h" -#define RANDOM_FILE_FAIL_FACTOR 5 +#ifdef TAOS_RANDOM_FILE_FAIL + +static int random_file_fail_factor = 20; + +void taosSetRandomFileFailFactor(int factor) +{ + random_file_fail_factor = factor; +} +#endif ssize_t taos_tread(int fd, void *buf, size_t count) { #ifdef TAOS_RANDOM_FILE_FAIL - if (rand() % RANDOM_FILE_FAIL_FACTOR == 0) { - errno = EIO; - return -1; + if (random_file_fail_factor > 0) { + if (rand() % random_file_fail_factor == 0) { + errno = EIO; + return -1; + } } #endif - return tread(fd, buf, count); } ssize_t taos_twrite(int fd, void *buf, size_t count) { #ifdef TAOS_RANDOM_FILE_FAIL - if (rand() % RANDOM_FILE_FAIL_FACTOR == 0) { - errno = EIO; - return -1; + if (random_file_fail_factor > 0) { + if (rand() % random_file_fail_factor == 0) { + errno = EIO; + return -1; + } } #endif - return twrite(fd, buf, count); } off_t taos_lseek(int fd, off_t offset, int whence) { #ifdef TAOS_RANDOM_FILE_FAIL - if (rand() % RANDOM_FILE_FAIL_FACTOR == 0) { - errno = EIO; - return -1; + if (random_file_fail_factor > 0) { + if (rand() % random_file_fail_factor == 0) { + errno = EIO; + return -1; + } } #endif - return lseek(fd, offset, whence); } diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c index e079653ab3..94a0fdc956 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walMain.c @@ -28,6 +28,7 @@ #include "taoserror.h" #include "twal.h" #include "tqueue.h" +#include "tfile.h" #define walPrefix "wal" @@ -180,7 +181,7 @@ int walWrite(void *handle, SWalHead *pHead) { taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead)); int contLen = pHead->len + sizeof(SWalHead); - if(write(pWal->fd, pHead, contLen) != contLen) { + if(twrite(pWal->fd, pHead, contLen) != contLen) { wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); } else { @@ -325,7 +326,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { wDebug("wal:%s, start to restore", name); while (1) { - int ret = read(fd, pHead, sizeof(SWalHead)); + int ret = tread(fd, pHead, sizeof(SWalHead)); if ( ret == 0) break; if (ret != sizeof(SWalHead)) { @@ -340,7 +341,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { break; } - ret = read(fd, pHead->cont, pHead->len); + ret = tread(fd, pHead->cont, pHead->len); if ( ret != pHead->len) { wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 53957fa96a..9af72af471 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1,4 +1,4 @@ -#-----!/usr/bin/python3.7 +# -----!/usr/bin/python3.7 ################################################################### # Copyright (c) 2016 by TAOS Technologies, Inc. # All rights reserved. @@ -11,7 +11,31 @@ ################################################################### # -*- coding: utf-8 -*- -from __future__ import annotations # For type hinting before definition, ref: https://stackoverflow.com/questions/33533148/how-do-i-specify-that-the-return-type-of-a-method-is-the-same-as-the-class-itsel +# For type hinting before definition, ref: +# https://stackoverflow.com/questions/33533148/how-do-i-specify-that-the-return-type-of-a-method-is-the-same-as-the-class-itsel +from __future__ import annotations +import taos +import crash_gen +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.log import * +from queue import Queue, Empty +from typing import IO +from typing import Set +from typing import Dict +from typing import List +from requests.auth import HTTPBasicAuth +import textwrap +import datetime +import logging +import time +import random +import threading +import requests +import copy +import argparse +import getopt import sys import os @@ -22,71 +46,48 @@ import traceback if sys.version_info[0] < 3: raise Exception("Must be using Python 3") -import getopt -import argparse -import copy -import requests -import threading -import random -import time -import logging -import datetime -import textwrap -import requests -from requests.auth import HTTPBasicAuth - -from typing import List -from typing import Dict -from typing import Set -from typing import IO -from queue import Queue, Empty - -from util.log import * -from util.dnodes import * -from util.cases import * -from util.sql import * - -import crash_gen -import taos - -# Global variables, tried to keep a small number. +# Global variables, tried to keep a small number. # Command-line/Environment Configurations, will set a bit later # ConfigNameSpace = argparse.Namespace -gConfig = argparse.Namespace() # Dummy value, will be replaced later +gConfig = argparse.Namespace() # Dummy value, will be replaced later logger = None -def runThread(wt: WorkerThread): + +def runThread(wt: WorkerThread): wt.run() + class CrashGenError(Exception): def __init__(self, msg=None, errno=None): - self.msg = msg + self.msg = msg self.errno = errno - + def __str__(self): return self.msg + class WorkerThread: - def __init__(self, pool: ThreadPool, tid, - tc: ThreadCoordinator, - # te: TaskExecutor, - ): # note: main thread context! - # self._curStep = -1 + def __init__(self, pool: ThreadPool, tid, + tc: ThreadCoordinator, + # te: TaskExecutor, + ): # note: main thread context! + # self._curStep = -1 self._pool = pool - self._tid = tid - self._tc = tc # type: ThreadCoordinator + self._tid = tid + self._tc = tc # type: ThreadCoordinator # self.threadIdent = threading.get_ident() self._thread = threading.Thread(target=runThread, args=(self,)) self._stepGate = threading.Event() # Let us have a DB connection of our own - if ( gConfig.per_thread_db_connection ): # type: ignore + if (gConfig.per_thread_db_connection): # type: ignore # print("connector_type = {}".format(gConfig.connector_type)) - self._dbConn = DbConn.createNative() if (gConfig.connector_type == 'native') else DbConn.createRest() + self._dbConn = DbConn.createNative() if ( + gConfig.connector_type == 'native') else DbConn.createRest() - self._dbInUse = False # if "use db" was executed already + self._dbInUse = False # if "use db" was executed already def logDebug(self, msg): logger.debug(" TRD[{}] {}".format(self._tid, msg)) @@ -98,137 +99,153 @@ class WorkerThread: return self._dbInUse def useDb(self): - if ( not self._dbInUse ): + if (not self._dbInUse): self.execSql("use db") self._dbInUse = True def getTaskExecutor(self): - return self._tc.getTaskExecutor() + return self._tc.getTaskExecutor() def start(self): self._thread.start() # AFTER the thread is recorded - def run(self): + def run(self): # initialization after thread starts, in the thread context # self.isSleeping = False logger.info("Starting to run thread: {}".format(self._tid)) - if ( gConfig.per_thread_db_connection ): # type: ignore + if (gConfig.per_thread_db_connection): # type: ignore logger.debug("Worker thread openning database connection") self._dbConn.open() - self._doTaskLoop() - + self._doTaskLoop() + # clean up - if ( gConfig.per_thread_db_connection ): # type: ignore + if (gConfig.per_thread_db_connection): # type: ignore self._dbConn.close() - def _doTaskLoop(self) : + def _doTaskLoop(self): # while self._curStep < self._pool.maxSteps: # tc = ThreadCoordinator(None) - while True: - tc = self._tc # Thread Coordinator, the overall master + while True: + tc = self._tc # Thread Coordinator, the overall master tc.crossStepBarrier() # shared barrier first, INCLUDING the last one - logger.debug("[TRD] Worker thread [{}] exited barrier...".format(self._tid)) + logger.debug( + "[TRD] Worker thread [{}] exited barrier...".format( + self._tid)) self.crossStepGate() # then per-thread gate, after being tapped - logger.debug("[TRD] Worker thread [{}] exited step gate...".format(self._tid)) + logger.debug( + "[TRD] Worker thread [{}] exited step gate...".format( + self._tid)) if not self._tc.isRunning(): - logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...") + logger.debug( + "[TRD] Thread Coordinator not running any more, worker thread now stopping...") break # Fetch a task from the Thread Coordinator - logger.debug("[TRD] Worker thread [{}] about to fetch task".format(self._tid)) + logger.debug( + "[TRD] Worker thread [{}] about to fetch task".format( + self._tid)) task = tc.fetchTask() # Execute such a task - logger.debug("[TRD] Worker thread [{}] about to execute task: {}".format(self._tid, task.__class__.__name__)) + logger.debug( + "[TRD] Worker thread [{}] about to execute task: {}".format( + self._tid, task.__class__.__name__)) task.execute(self) tc.saveExecutedTask(task) - logger.debug("[TRD] Worker thread [{}] finished executing task".format(self._tid)) + logger.debug( + "[TRD] Worker thread [{}] finished executing task".format( + self._tid)) - self._dbInUse = False # there may be changes between steps - - def verifyThreadSelf(self): # ensure we are called by this own thread - if ( threading.get_ident() != self._thread.ident ): + self._dbInUse = False # there may be changes between steps + + def verifyThreadSelf(self): # ensure we are called by this own thread + if (threading.get_ident() != self._thread.ident): raise RuntimeError("Unexpectly called from other threads") - def verifyThreadMain(self): # ensure we are called by the main thread - if ( threading.get_ident() != threading.main_thread().ident ): + def verifyThreadMain(self): # ensure we are called by the main thread + if (threading.get_ident() != threading.main_thread().ident): raise RuntimeError("Unexpectly called from other threads") def verifyThreadAlive(self): - if ( not self._thread.is_alive() ): + if (not self._thread.is_alive()): raise RuntimeError("Unexpected dead thread") # A gate is different from a barrier in that a thread needs to be "tapped" def crossStepGate(self): self.verifyThreadAlive() - self.verifyThreadSelf() # only allowed by ourselves - + self.verifyThreadSelf() # only allowed by ourselves + # Wait again at the "gate", waiting to be "tapped" - logger.debug("[TRD] Worker thread {} about to cross the step gate".format(self._tid)) - self._stepGate.wait() + logger.debug( + "[TRD] Worker thread {} about to cross the step gate".format( + self._tid)) + self._stepGate.wait() self._stepGate.clear() - + # self._curStep += 1 # off to a new step... - def tapStepGate(self): # give it a tap, release the thread waiting there + def tapStepGate(self): # give it a tap, release the thread waiting there self.verifyThreadAlive() - self.verifyThreadMain() # only allowed for main thread - - logger.debug("[TRD] Tapping worker thread {}".format(self._tid)) - self._stepGate.set() # wake up! - time.sleep(0) # let the released thread run a bit + self.verifyThreadMain() # only allowed for main thread - def execSql(self, sql): # TODO: expose DbConn directly - if ( gConfig.per_thread_db_connection ): - return self._dbConn.execute(sql) + logger.debug("[TRD] Tapping worker thread {}".format(self._tid)) + self._stepGate.set() # wake up! + time.sleep(0) # let the released thread run a bit + + def execSql(self, sql): # TODO: expose DbConn directly + if (gConfig.per_thread_db_connection): + return self._dbConn.execute(sql) else: return self._tc.getDbManager().getDbConn().execute(sql) - def querySql(self, sql): # TODO: expose DbConn directly - if ( gConfig.per_thread_db_connection ): - return self._dbConn.query(sql) + def querySql(self, sql): # TODO: expose DbConn directly + if (gConfig.per_thread_db_connection): + return self._dbConn.query(sql) else: return self._tc.getDbManager().getDbConn().query(sql) def getQueryResult(self): - if ( gConfig.per_thread_db_connection ): - return self._dbConn.getQueryResult() + if (gConfig.per_thread_db_connection): + return self._dbConn.getQueryResult() else: return self._tc.getDbManager().getDbConn().getQueryResult() def getDbConn(self): - if ( gConfig.per_thread_db_connection ): - return self._dbConn + if (gConfig.per_thread_db_connection): + return self._dbConn else: return self._tc.getDbManager().getDbConn() # def querySql(self, sql): # not "execute", since we are out side the DB context # if ( gConfig.per_thread_db_connection ): - # return self._dbConn.query(sql) + # return self._dbConn.query(sql) # else: # return self._tc.getDbState().getDbConn().query(sql) # The coordinator of all worker threads, mostly running in main thread + + class ThreadCoordinator: def __init__(self, pool: ThreadPool, dbManager): - self._curStep = -1 # first step is 0 + self._curStep = -1 # first step is 0 self._pool = pool # self._wd = wd - self._te = None # prepare for every new step + self._te = None # prepare for every new step self._dbManager = dbManager - self._executedTasks: List[Task] = [] # in a given step - self._lock = threading.RLock() # sync access for a few things + self._executedTasks: List[Task] = [] # in a given step + self._lock = threading.RLock() # sync access for a few things - self._stepBarrier = threading.Barrier(self._pool.numThreads + 1) # one barrier for all threads + self._stepBarrier = threading.Barrier( + self._pool.numThreads + 1) # one barrier for all threads self._execStats = ExecutionStats() self._runStatus = MainExec.STATUS_RUNNING def getTaskExecutor(self): return self._te - def getDbManager(self) -> DbManager : + def getDbManager(self) -> DbManager: return self._dbManager def crossStepBarrier(self): @@ -238,89 +255,103 @@ class ThreadCoordinator: self._runStatus = MainExec.STATUS_STOPPING self._execStats.registerFailure("User Interruption") - def run(self): + def run(self): self._pool.createAndStartThreads(self) # Coordinate all threads step by step - self._curStep = -1 # not started yet - maxSteps = gConfig.max_steps # type: ignore - self._execStats.startExec() # start the stop watch + self._curStep = -1 # not started yet + maxSteps = gConfig.max_steps # type: ignore + self._execStats.startExec() # start the stop watch transitionFailed = False hasAbortedTask = False - while(self._curStep < maxSteps-1 and - (not transitionFailed) and - (self._runStatus==MainExec.STATUS_RUNNING) and - (not hasAbortedTask)): # maxStep==10, last curStep should be 9 + while(self._curStep < maxSteps - 1 and + (not transitionFailed) and + (self._runStatus == MainExec.STATUS_RUNNING) and + (not hasAbortedTask)): # maxStep==10, last curStep should be 9 - if not gConfig.debug: - print(".", end="", flush=True) # print this only if we are not in debug mode + if not gConfig.debug: + # print this only if we are not in debug mode + print(".", end="", flush=True) logger.debug("[TRD] Main thread going to sleep") # Now main thread (that's us) is ready to enter a step - self.crossStepBarrier() # let other threads go past the pool barrier, but wait at the thread gate - self._stepBarrier.reset() # Other worker threads should now be at the "gate" + # let other threads go past the pool barrier, but wait at the + # thread gate + self.crossStepBarrier() + self._stepBarrier.reset() # Other worker threads should now be at the "gate" # At this point, all threads should be pass the overall "barrier" and before the per-thread "gate" - # We use this period to do house keeping work, when all worker threads are QUIET. + # We use this period to do house keeping work, when all worker + # threads are QUIET. hasAbortedTask = False - for task in self._executedTasks : - if task.isAborted() : + for task in self._executedTasks: + if task.isAborted(): print("Task aborted: {}".format(task)) hasAbortedTask = True break - if hasAbortedTask : # do transition only if tasks are error free + if hasAbortedTask: # do transition only if tasks are error free self._execStats.registerFailure("Aborted Task Encountered") - else: + else: try: sm = self._dbManager.getStateMachine() logger.debug("[STT] starting transitions") - sm.transition(self._executedTasks) # at end of step, transiton the DB state + # at end of step, transiton the DB state + sm.transition(self._executedTasks) logger.debug("[STT] transition ended") - # Due to limitation (or maybe not) of the Python library, we cannot share connections across threads - if sm.hasDatabase() : + # Due to limitation (or maybe not) of the Python library, + # we cannot share connections across threads + if sm.hasDatabase(): for t in self._pool.threadList: logger.debug("[DB] use db for all worker threads") t.useDb() - # t.execSql("use db") # main thread executing "use db" on behalf of every worker thread + # t.execSql("use db") # main thread executing "use + # db" on behalf of every worker thread except taos.error.ProgrammingError as err: - if ( err.msg == 'network unavailable' ): # broken DB connection + if (err.msg == 'network unavailable'): # broken DB connection logger.info("DB connection broken, execution failed") traceback.print_stack() transitionFailed = True - self._te = None # Not running any more + self._te = None # Not running any more self._execStats.registerFailure("Broken DB Connection") - # continue # don't do that, need to tap all threads at end, and maybe signal them to stop + # continue # don't do that, need to tap all threads at + # end, and maybe signal them to stop else: - raise + raise # finally: # pass - - self.resetExecutedTasks() # clear the tasks after we are done + + self.resetExecutedTasks() # clear the tasks after we are done # Get ready for next step logger.debug("<-- Step {} finished".format(self._curStep)) - self._curStep += 1 # we are about to get into next step. TODO: race condition here! - logger.debug("\r\n\n--> Step {} starts with main thread waking up".format(self._curStep)) # Now not all threads had time to go to sleep + self._curStep += 1 # we are about to get into next step. TODO: race condition here! + # Now not all threads had time to go to sleep + logger.debug( + "\r\n\n--> Step {} starts with main thread waking up".format(self._curStep)) # A new TE for the new step - if not transitionFailed: # only if not failed + if not transitionFailed: # only if not failed self._te = TaskExecutor(self._curStep) - logger.debug("[TRD] Main thread waking up at step {}, tapping worker threads".format(self._curStep)) # Now not all threads had time to go to sleep - self.tapAllThreads() # Worker threads will wake up at this point, and each execute it's own task + logger.debug( + "[TRD] Main thread waking up at step {}, tapping worker threads".format( + self._curStep)) # Now not all threads had time to go to sleep + # Worker threads will wake up at this point, and each execute it's + # own task + self.tapAllThreads() logger.debug("Main thread ready to finish up...") - if not transitionFailed: # only in regular situations - self.crossStepBarrier() # Cross it one last time, after all threads finish + if not transitionFailed: # only in regular situations + self.crossStepBarrier() # Cross it one last time, after all threads finish self._stepBarrier.reset() logger.debug("Main thread in exclusive zone...") - self._te = None # No more executor, time to end + self._te = None # No more executor, time to end logger.debug("Main thread tapping all threads one last time...") - self.tapAllThreads() # Let the threads run one last time + self.tapAllThreads() # Let the threads run one last time logger.debug("Main thread joining all threads") - self._pool.joinAll() # Get all threads to finish + self._pool.joinAll() # Get all threads to finish logger.info("\nAll worker threads finished") self._execStats.endExec() @@ -333,24 +364,27 @@ class ThreadCoordinator: def getExecStats(self): return self._execStats - def tapAllThreads(self): # in a deterministic manner + def tapAllThreads(self): # in a deterministic manner wakeSeq = [] - for i in range(self._pool.numThreads): # generate a random sequence - if Dice.throw(2) == 1 : + for i in range(self._pool.numThreads): # generate a random sequence + if Dice.throw(2) == 1: wakeSeq.append(i) else: wakeSeq.insert(0, i) - logger.debug("[TRD] Main thread waking up worker threads: {}".format(str(wakeSeq))) + logger.debug( + "[TRD] Main thread waking up worker threads: {}".format( + str(wakeSeq))) # TODO: set dice seed to a deterministic value for i in wakeSeq: - self._pool.threadList[i].tapStepGate() # TODO: maybe a bit too deep?! - time.sleep(0) # yield + # TODO: maybe a bit too deep?! + self._pool.threadList[i].tapStepGate() + time.sleep(0) # yield def isRunning(self): - return self._te != None + return self._te is not None - def fetchTask(self) -> Task : - if ( not self.isRunning() ): # no task + def fetchTask(self) -> Task: + if (not self.isRunning()): # no task raise RuntimeError("Cannot fetch task when not running") # return self._wd.pickTask() # Alternatively, let's ask the DbState for the appropriate task @@ -361,31 +395,36 @@ class ThreadCoordinator: # logger.debug(" (dice:{}/{}) ".format(i, nTasks)) # # return copy.copy(tasks[i]) # Needs a fresh copy, to save execution results, etc. # return tasks[i].clone() # TODO: still necessary? - taskType = self.getDbManager().getStateMachine().pickTaskType() # pick a task type for current state - return taskType(self.getDbManager(), self._execStats) # create a task from it + # pick a task type for current state + taskType = self.getDbManager().getStateMachine().pickTaskType() + return taskType( + self.getDbManager(), + self._execStats) # create a task from it def resetExecutedTasks(self): - self._executedTasks = [] # should be under single thread + self._executedTasks = [] # should be under single thread def saveExecutedTask(self, task): with self._lock: self._executedTasks.append(task) # We define a class to run a number of threads in locking steps. + + class ThreadPool: def __init__(self, numThreads, maxSteps): self.numThreads = numThreads self.maxSteps = maxSteps # Internal class variables self.curStep = 0 - self.threadList = [] # type: List[WorkerThread] - + self.threadList = [] # type: List[WorkerThread] + # starting to run all the threads, in locking steps def createAndStartThreads(self, tc: ThreadCoordinator): - for tid in range(0, self.numThreads): # Create the threads - workerThread = WorkerThread(self, tid, tc) + for tid in range(0, self.numThreads): # Create the threads + workerThread = WorkerThread(self, tid, tc) self.threadList.append(workerThread) - workerThread.start() # start, but should block immediately before step 0 + workerThread.start() # start, but should block immediately before step 0 def joinAll(self): for workerThread in self.threadList: @@ -394,21 +433,24 @@ class ThreadPool: # A queue of continguous POSITIVE integers, used by DbManager to generate continuous numbers # for new table names + + class LinearQueue(): def __init__(self): self.firstIndex = 1 # 1st ever element self.lastIndex = 0 - self._lock = threading.RLock() # our functions may call each other - self.inUse = set() # the indexes that are in use right now + self._lock = threading.RLock() # our functions may call each other + self.inUse = set() # the indexes that are in use right now def toText(self): - return "[{}..{}], in use: {}".format(self.firstIndex, self.lastIndex, self.inUse) + return "[{}..{}], in use: {}".format( + self.firstIndex, self.lastIndex, self.inUse) # Push (add new element, largest) to the tail, and mark it in use - def push(self): + def push(self): with self._lock: - # if ( self.isEmpty() ): - # self.lastIndex = self.firstIndex + # if ( self.isEmpty() ): + # self.lastIndex = self.firstIndex # return self.firstIndex # Otherwise we have something self.lastIndex += 1 @@ -418,12 +460,12 @@ class LinearQueue(): def pop(self): with self._lock: - if ( self.isEmpty() ): - # raise RuntimeError("Cannot pop an empty queue") - return False # TODO: None? - + if (self.isEmpty()): + # raise RuntimeError("Cannot pop an empty queue") + return False # TODO: None? + index = self.firstIndex - if ( index in self.inUse ): + if (index in self.inUse): return False self.firstIndex += 1 @@ -441,33 +483,35 @@ class LinearQueue(): def allocate(self, i): with self._lock: # logger.debug("LQ allocating item {}".format(i)) - if ( i in self.inUse ): - raise RuntimeError("Cannot re-use same index in queue: {}".format(i)) + if (i in self.inUse): + raise RuntimeError( + "Cannot re-use same index in queue: {}".format(i)) self.inUse.add(i) def release(self, i): with self._lock: # logger.debug("LQ releasing item {}".format(i)) - self.inUse.remove(i) # KeyError possible, TODO: why? + self.inUse.remove(i) # KeyError possible, TODO: why? def size(self): return self.lastIndex + 1 - self.firstIndex def pickAndAllocate(self): - if ( self.isEmpty() ): + if (self.isEmpty()): return None with self._lock: - cnt = 0 # counting the interations + cnt = 0 # counting the interations while True: cnt += 1 - if ( cnt > self.size()*10 ): # 10x iteration already + if (cnt > self.size() * 10): # 10x iteration already # raise RuntimeError("Failed to allocate LinearQueue element") return None - ret = Dice.throwRange(self.firstIndex, self.lastIndex+1) - if ( not ret in self.inUse ): + ret = Dice.throwRange(self.firstIndex, self.lastIndex + 1) + if (ret not in self.inUse): self.allocate(ret) return ret + class DbConn: TYPE_NATIVE = "native-c" TYPE_REST = "rest-api" @@ -480,7 +524,8 @@ class DbConn: elif connType == cls.TYPE_REST: return DbConnRest() else: - raise RuntimeError("Unexpected connection type: {}".format(connType)) + raise RuntimeError( + "Unexpected connection type: {}".format(connType)) @classmethod def createNative(cls): @@ -495,18 +540,21 @@ class DbConn: self._type = self.TYPE_INVALID def open(self): - if ( self.isOpen ): + if (self.isOpen): raise RuntimeError("Cannot re-open an existing DB connection") # below implemented by child classes self.openByType() - logger.debug("[DB] data connection opened, type = {}".format(self._type)) + logger.debug( + "[DB] data connection opened, type = {}".format( + self._type)) self.isOpen = True - def resetDb(self): # reset the whole database, etc. - if ( not self.isOpen ): - raise RuntimeError("Cannot reset database until connection is open") + def resetDb(self): # reset the whole database, etc. + if (not self.isOpen): + raise RuntimeError( + "Cannot reset database until connection is open") # self._tdSql.prepare() # Recreate database, etc. self.execute('drop database if exists db') @@ -515,83 +563,99 @@ class DbConn: # self._cursor.execute('use db') # tdSql.execute('show databases') - def queryScalar(self, sql) -> int : + def queryScalar(self, sql) -> int: return self._queryAny(sql) - def queryString(self, sql) -> str : + def queryString(self, sql) -> str: return self._queryAny(sql) - def _queryAny(self, sql) : # actual query result as an int - if ( not self.isOpen ): - raise RuntimeError("Cannot query database until connection is open") + def _queryAny(self, sql): # actual query result as an int + if (not self.isOpen): + raise RuntimeError( + "Cannot query database until connection is open") nRows = self.query(sql) - if nRows != 1 : - raise RuntimeError("Unexpected result for query: {}, rows = {}".format(sql, nRows)) + if nRows != 1: + raise RuntimeError( + "Unexpected result for query: {}, rows = {}".format( + sql, nRows)) if self.getResultRows() != 1 or self.getResultCols() != 1: - raise RuntimeError("Unexpected result set for query: {}".format(sql)) + raise RuntimeError( + "Unexpected result set for query: {}".format(sql)) return self.getQueryResult()[0][0] def execute(self, sql): raise RuntimeError("Unexpected execution, should be overriden") + def openByType(self): raise RuntimeError("Unexpected execution, should be overriden") + def getQueryResult(self): raise RuntimeError("Unexpected execution, should be overriden") + def getResultRows(self): raise RuntimeError("Unexpected execution, should be overriden") + def getResultCols(self): raise RuntimeError("Unexpected execution, should be overriden") # Sample: curl -u root:taosdata -d "show databases" localhost:6020/rest/sql + + class DbConnRest(DbConn): def __init__(self): super().__init__() self._type = self.TYPE_REST - self._url = "http://localhost:6020/rest/sql" # fixed for now + self._url = "http://localhost:6020/rest/sql" # fixed for now self._result = None - def openByType(self): # Open connection - pass # do nothing, always open - + def openByType(self): # Open connection + pass # do nothing, always open + def close(self): - if ( not self.isOpen ): - raise RuntimeError("Cannot clean up database until connection is open") + if (not self.isOpen): + raise RuntimeError( + "Cannot clean up database until connection is open") # Do nothing for REST logger.debug("[DB] REST Database connection closed") self.isOpen = False def _doSql(self, sql): - r = requests.post(self._url, - data = sql, - auth = HTTPBasicAuth('root', 'taosdata')) + r = requests.post(self._url, + data=sql, + auth=HTTPBasicAuth('root', 'taosdata')) rj = r.json() # Sanity check for the "Json Result" - if (not 'status' in rj): + if ('status' not in rj): raise RuntimeError("No status in REST response") - if rj['status'] == 'error': # clearly reported error - if (not 'code' in rj): # error without code - raise RuntimeError("REST error return without code") - errno = rj['code'] # May need to massage this in the future + if rj['status'] == 'error': # clearly reported error + if ('code' not in rj): # error without code + raise RuntimeError("REST error return without code") + errno = rj['code'] # May need to massage this in the future # print("Raising programming error with REST return: {}".format(rj)) - raise taos.error.ProgrammingError(rj['desc'], errno) # todo: check existance of 'desc' + raise taos.error.ProgrammingError( + rj['desc'], errno) # todo: check existance of 'desc' - if rj['status'] != 'succ': # better be this - raise RuntimeError("Unexpected REST return status: {}".format(rj['status'])) + if rj['status'] != 'succ': # better be this + raise RuntimeError( + "Unexpected REST return status: {}".format( + rj['status'])) nRows = rj['rows'] if ('rows' in rj) else 0 - self._result = rj + self._result = rj return nRows - def execute(self, sql): - if ( not self.isOpen ): - raise RuntimeError("Cannot execute database commands until connection is open") + def execute(self, sql): + if (not self.isOpen): + raise RuntimeError( + "Cannot execute database commands until connection is open") logger.debug("[SQL-REST] Executing SQL: {}".format(sql)) nRows = self._doSql(sql) - logger.debug("[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) + logger.debug( + "[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) return nRows - def query(self, sql) : # return rows affected + def query(self, sql): # return rows affected return self.execute(sql) def getQueryResult(self): @@ -605,48 +669,117 @@ class DbConnRest(DbConn): def getResultCols(self): print(self._result) raise RuntimeError("TBD") - + + # Duplicate code from TDMySQL, TODO: merge all this into DbConnNative + + +class MyTDSql: + def __init__(self): + self.queryRows = 0 + self.queryCols = 0 + self.affectedRows = 0 + + def init(self, cursor, log=True): + self.cursor = cursor + # if (log): + # caller = inspect.getframeinfo(inspect.stack()[1][0]) + # self.cursor.log(caller.filename + ".sql") + + def close(self): + self.cursor.close() + + def query(self, sql): + self.sql = sql + try: + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) + except Exception as e: + # caller = inspect.getframeinfo(inspect.stack()[1][0]) + # args = (caller.filename, caller.lineno, sql, repr(e)) + # tdLog.exit("%s(%d) failed: sql:%s, %s" % args) + raise + return self.queryRows + + def execute(self, sql): + self.sql = sql + try: + self.affectedRows = self.cursor.execute(sql) + except Exception as e: + # caller = inspect.getframeinfo(inspect.stack()[1][0]) + # args = (caller.filename, caller.lineno, sql, repr(e)) + # tdLog.exit("%s(%d) failed: sql:%s, %s" % args) + raise + return self.affectedRows + + class DbConnNative(DbConn): def __init__(self): super().__init__() self._type = self.TYPE_REST - self._conn = None + self._conn = None self._cursor = None - - def openByType(self): # Open connection - cfgPath = "../../build/test/cfg" - self._conn = taos.connect(host="127.0.0.1", config=cfgPath) # TODO: make configurable + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("communit")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def openByType(self): # Open connection + cfgPath = self.getBuildPath() + "/test/cfg" + self._conn = taos.connect( + host="127.0.0.1", + config=cfgPath) # TODO: make configurable self._cursor = self._conn.cursor() # Get the connection/cursor ready self._cursor.execute('reset query cache') - # self._cursor.execute('use db') # do this at the beginning of every step + # self._cursor.execute('use db') # do this at the beginning of every + # step # Open connection - self._tdSql = TDSql() + self._tdSql = MyTDSql() self._tdSql.init(self._cursor) - + def close(self): - if ( not self.isOpen ): - raise RuntimeError("Cannot clean up database until connection is open") + if (not self.isOpen): + raise RuntimeError( + "Cannot clean up database until connection is open") self._tdSql.close() logger.debug("[DB] Database connection closed") self.isOpen = False - def execute(self, sql): - if ( not self.isOpen ): - raise RuntimeError("Cannot execute database commands until connection is open") + def execute(self, sql): + if (not self.isOpen): + raise RuntimeError( + "Cannot execute database commands until connection is open") logger.debug("[SQL] Executing SQL: {}".format(sql)) nRows = self._tdSql.execute(sql) - logger.debug("[SQL] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) + logger.debug( + "[SQL] Execution Result, nRows = {}, SQL = {}".format( + nRows, sql)) return nRows - def query(self, sql) : # return rows affected - if ( not self.isOpen ): - raise RuntimeError("Cannot query database until connection is open") + def query(self, sql): # return rows affected + if (not self.isOpen): + raise RuntimeError( + "Cannot query database until connection is open") logger.debug("[SQL] Executing SQL: {}".format(sql)) nRows = self._tdSql.query(sql) - logger.debug("[SQL] Query Result, nRows = {}, SQL = {}".format(nRows, sql)) + logger.debug( + "[SQL] Query Result, nRows = {}, SQL = {}".format( + nRows, sql)) return nRows # results are in: return self._tdSql.queryResult @@ -659,13 +792,13 @@ class DbConnNative(DbConn): def getResultCols(self): return self._tdSql.queryCols - + class AnyState: - STATE_INVALID = -1 - STATE_EMPTY = 0 # nothing there, no even a DB - STATE_DB_ONLY = 1 # we have a DB, but nothing else + STATE_INVALID = -1 + STATE_EMPTY = 0 # nothing there, no even a DB + STATE_DB_ONLY = 1 # we have a DB, but nothing else STATE_TABLE_ONLY = 2 # we have a table, but totally empty - STATE_HAS_DATA = 3 # we have some data in the table + STATE_HAS_DATA = 3 # we have some data in the table _stateNames = ["Invalid", "Empty", "DB_Only", "Table_Only", "Has_Data"] STATE_VAL_IDX = 0 @@ -680,7 +813,8 @@ class AnyState: self._info = self.getInfo() def __str__(self): - return self._stateNames[self._info[self.STATE_VAL_IDX] + 1] # -1 hack to accomodate the STATE_INVALID case + # -1 hack to accomodate the STATE_INVALID case + return self._stateNames[self._info[self.STATE_VAL_IDX] + 1] def getInfo(self): raise RuntimeError("Must be overriden by child classes") @@ -691,7 +825,9 @@ class AnyState: elif isinstance(other, AnyState): return self.getValIndex() == other.getValIndex() else: - raise RuntimeError("Unexpected comparison, type = {}".format(type(other))) + raise RuntimeError( + "Unexpected comparison, type = {}".format( + type(other))) def verifyTasksToState(self, tasks, newState): raise RuntimeError("Must be overriden by child classes") @@ -701,55 +837,65 @@ class AnyState: def getValue(self): return self._info[self.STATE_VAL_IDX] + def canCreateDb(self): return self._info[self.CAN_CREATE_DB] + def canDropDb(self): return self._info[self.CAN_DROP_DB] + def canCreateFixedSuperTable(self): return self._info[self.CAN_CREATE_FIXED_SUPER_TABLE] + def canDropFixedSuperTable(self): return self._info[self.CAN_DROP_FIXED_SUPER_TABLE] + def canAddData(self): return self._info[self.CAN_ADD_DATA] + def canReadData(self): return self._info[self.CAN_READ_DATA] def assertAtMostOneSuccess(self, tasks, cls): sCnt = 0 - for task in tasks : + for task in tasks: if not isinstance(task, cls): continue if task.isSuccess(): # task.logDebug("Task success found") sCnt += 1 - if ( sCnt >= 2 ): - raise RuntimeError("Unexpected more than 1 success with task: {}".format(cls)) + if (sCnt >= 2): + raise RuntimeError( + "Unexpected more than 1 success with task: {}".format(cls)) def assertIfExistThenSuccess(self, tasks, cls): sCnt = 0 exists = False - for task in tasks : + for task in tasks: if not isinstance(task, cls): continue - exists = True # we have a valid instance + exists = True # we have a valid instance if task.isSuccess(): sCnt += 1 - if ( exists and sCnt <= 0 ): - raise RuntimeError("Unexpected zero success for task: {}".format(cls)) + if (exists and sCnt <= 0): + raise RuntimeError( + "Unexpected zero success for task: {}".format(cls)) def assertNoTask(self, tasks, cls): - for task in tasks : + for task in tasks: if isinstance(task, cls): - raise CrashGenError("This task: {}, is not expected to be present, given the success/failure of others".format(cls.__name__)) + raise CrashGenError( + "This task: {}, is not expected to be present, given the success/failure of others".format(cls.__name__)) def assertNoSuccess(self, tasks, cls): - for task in tasks : + for task in tasks: if isinstance(task, cls): if task.isSuccess(): - raise RuntimeError("Unexpected successful task: {}".format(cls)) + raise RuntimeError( + "Unexpected successful task: {}".format(cls)) def hasSuccess(self, tasks, cls): - for task in tasks : + for task in tasks: if not isinstance(task, cls): continue if task.isSuccess(): @@ -757,35 +903,40 @@ class AnyState: return False def hasTask(self, tasks, cls): - for task in tasks : + for task in tasks: if isinstance(task, cls): return True return False + class StateInvalid(AnyState): def getInfo(self): return [ self.STATE_INVALID, - False, False, # can create/drop Db - False, False, # can create/drop fixed table - False, False, # can insert/read data with fixed table + False, False, # can create/drop Db + False, False, # can create/drop fixed table + False, False, # can insert/read data with fixed table ] # def verifyTasksToState(self, tasks, newState): + class StateEmpty(AnyState): def getInfo(self): return [ self.STATE_EMPTY, - True, False, # can create/drop Db - False, False, # can create/drop fixed table - False, False, # can insert/read data with fixed table + True, False, # can create/drop Db + False, False, # can create/drop fixed table + False, False, # can insert/read data with fixed table ] - def verifyTasksToState(self, tasks, newState): - if ( self.hasSuccess(tasks, TaskCreateDb) ): # at EMPTY, if there's succes in creating DB - if ( not self.hasTask(tasks, TaskDropDb) ) : # and no drop_db tasks - self.assertAtMostOneSuccess(tasks, TaskCreateDb) # we must have at most one. TODO: compare numbers + def verifyTasksToState(self, tasks, newState): + if (self.hasSuccess(tasks, TaskCreateDb) + ): # at EMPTY, if there's succes in creating DB + if (not self.hasTask(tasks, TaskDropDb)): # and no drop_db tasks + # we must have at most one. TODO: compare numbers + self.assertAtMostOneSuccess(tasks, TaskCreateDb) + class StateDbOnly(AnyState): def getInfo(self): @@ -797,32 +948,34 @@ class StateDbOnly(AnyState): ] def verifyTasksToState(self, tasks, newState): - if ( not self.hasTask(tasks, TaskCreateDb) ): - self.assertAtMostOneSuccess(tasks, TaskDropDb) # only if we don't create any more + if (not self.hasTask(tasks, TaskCreateDb)): + # only if we don't create any more + self.assertAtMostOneSuccess(tasks, TaskDropDb) self.assertIfExistThenSuccess(tasks, TaskDropDb) # self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # not true in massively parrallel cases # Nothing to be said about adding data task # if ( self.hasSuccess(tasks, DropDbTask) ): # dropped the DB - # self.assertHasTask(tasks, DropDbTask) # implied by hasSuccess - # self.assertAtMostOneSuccess(tasks, DropDbTask) - # self._state = self.STATE_EMPTY + # self.assertHasTask(tasks, DropDbTask) # implied by hasSuccess + # self.assertAtMostOneSuccess(tasks, DropDbTask) + # self._state = self.STATE_EMPTY # if ( self.hasSuccess(tasks, TaskCreateSuperTable) ): # did not drop db, create table success # # self.assertHasTask(tasks, CreateFixedTableTask) # tried to create table - # if ( not self.hasTask(tasks, TaskDropSuperTable) ): + # if ( not self.hasTask(tasks, TaskDropSuperTable) ): # self.assertAtMostOneSuccess(tasks, TaskCreateSuperTable) # at most 1 attempt is successful, if we don't drop anything - # self.assertNoTask(tasks, DropDbTask) # should have have tried - # if ( not self.hasSuccess(tasks, AddFixedDataTask) ): # just created table, no data yet - # # can't say there's add-data attempts, since they may all fail - # self._state = self.STATE_TABLE_ONLY - # else: - # self._state = self.STATE_HAS_DATA + # self.assertNoTask(tasks, DropDbTask) # should have have tried + # if ( not self.hasSuccess(tasks, AddFixedDataTask) ): # just created table, no data yet + # # can't say there's add-data attempts, since they may all fail + # self._state = self.STATE_TABLE_ONLY + # else: + # self._state = self.STATE_HAS_DATA # What about AddFixedData? # elif ( self.hasSuccess(tasks, AddFixedDataTask) ): # self._state = self.STATE_HAS_DATA # else: # no success in dropping db tasks, no success in create fixed table? read data should also fail - # # raise RuntimeError("Unexpected no-success scenario") # We might just landed all failure tasks, + # # raise RuntimeError("Unexpected no-success scenario") # We might just landed all failure tasks, # self._state = self.STATE_DB_ONLY # no change + class StateSuperTableOnly(AnyState): def getInfo(self): return [ @@ -833,9 +986,11 @@ class StateSuperTableOnly(AnyState): ] def verifyTasksToState(self, tasks, newState): - if ( self.hasSuccess(tasks, TaskDropSuperTable) ): # we are able to drop the table + if (self.hasSuccess(tasks, TaskDropSuperTable) + ): # we are able to drop the table #self.assertAtMostOneSuccess(tasks, TaskDropSuperTable) - self.hasSuccess(tasks, TaskCreateSuperTable) # we must have had recreted it + # we must have had recreted it + self.hasSuccess(tasks, TaskCreateSuperTable) # self._state = self.STATE_DB_ONLY # elif ( self.hasSuccess(tasks, AddFixedDataTask) ): # no success dropping the table, but added data @@ -849,6 +1004,7 @@ class StateSuperTableOnly(AnyState): # raise RuntimeError("Unexpected no-success scenarios") # TODO: need to revamp!! + class StateHasData(AnyState): def getInfo(self): return [ @@ -859,13 +1015,15 @@ class StateHasData(AnyState): ] def verifyTasksToState(self, tasks, newState): - if ( newState.equals(AnyState.STATE_EMPTY) ): + if (newState.equals(AnyState.STATE_EMPTY)): self.hasSuccess(tasks, TaskDropDb) - if ( not self.hasTask(tasks, TaskCreateDb) ) : - self.assertAtMostOneSuccess(tasks, TaskDropDb) # TODO: dicy - elif ( newState.equals(AnyState.STATE_DB_ONLY) ): # in DB only - if ( not self.hasTask(tasks, TaskCreateDb)): # without a create_db task - self.assertNoTask(tasks, TaskDropDb) # we must have drop_db task + if (not self.hasTask(tasks, TaskCreateDb)): + self.assertAtMostOneSuccess(tasks, TaskDropDb) # TODO: dicy + elif (newState.equals(AnyState.STATE_DB_ONLY)): # in DB only + if (not self.hasTask(tasks, TaskCreateDb) + ): # without a create_db task + # we must have drop_db task + self.assertNoTask(tasks, TaskDropDb) self.hasSuccess(tasks, TaskDropSuperTable) # self.assertAtMostOneSuccess(tasks, DropFixedSuperTableTask) # TODO: dicy # elif ( newState.equals(AnyState.STATE_TABLE_ONLY) ): # data deleted @@ -873,19 +1031,26 @@ class StateHasData(AnyState): # self.assertNoTask(tasks, TaskDropSuperTable) # self.assertNoTask(tasks, TaskAddData) # self.hasSuccess(tasks, DeleteDataTasks) - else: # should be STATE_HAS_DATA - if (not self.hasTask(tasks, TaskCreateDb) ): # only if we didn't create one - self.assertNoTask(tasks, TaskDropDb) # we shouldn't have dropped it - if (not self.hasTask(tasks, TaskCreateSuperTable)) : # if we didn't create the table - self.assertNoTask(tasks, TaskDropSuperTable) # we should not have a task that drops it + else: # should be STATE_HAS_DATA + if (not self.hasTask(tasks, TaskCreateDb) + ): # only if we didn't create one + # we shouldn't have dropped it + self.assertNoTask(tasks, TaskDropDb) + if (not self.hasTask(tasks, TaskCreateSuperTable) + ): # if we didn't create the table + # we should not have a task that drops it + self.assertNoTask(tasks, TaskDropSuperTable) # self.assertIfExistThenSuccess(tasks, ReadFixedDataTask) + class StateMechine: def __init__(self, dbConn): self._dbConn = dbConn - self._curState = self._findCurrentState() # starting state - self._stateWeights = [1,3,5,15] # transitition target probabilities, indexed with value of STATE_EMPTY, STATE_DB_ONLY, etc. - + self._curState = self._findCurrentState() # starting state + # transitition target probabilities, indexed with value of STATE_EMPTY, + # STATE_DB_ONLY, etc. + self._stateWeights = [1, 3, 5, 15] + def getCurrentState(self): return self._curState @@ -893,142 +1058,178 @@ class StateMechine: return self._curState.canDropDb() # ha, can drop DB means it has one # May be slow, use cautionsly... - def getTaskTypes(self): # those that can run (directly/indirectly) from the current state + def getTaskTypes(self): # those that can run (directly/indirectly) from the current state def typesToStrings(types): ss = [] for t in types: ss.append(t.__name__) return ss - allTaskClasses = StateTransitionTask.__subclasses__() # all state transition tasks + allTaskClasses = StateTransitionTask.__subclasses__() # all state transition tasks firstTaskTypes = [] for tc in allTaskClasses: - # t = tc(self) # create task object + # t = tc(self) # create task object if tc.canBeginFrom(self._curState): firstTaskTypes.append(tc) - # now we have all the tasks that can begin directly from the current state, let's figure out the INDIRECT ones - taskTypes = firstTaskTypes.copy() # have to have these - for task1 in firstTaskTypes: # each task type gathered so far - endState = task1.getEndState() # figure the end state - if endState == None: # does not change end state - continue # no use, do nothing - for tc in allTaskClasses: # what task can further begin from there? + # now we have all the tasks that can begin directly from the current + # state, let's figure out the INDIRECT ones + taskTypes = firstTaskTypes.copy() # have to have these + for task1 in firstTaskTypes: # each task type gathered so far + endState = task1.getEndState() # figure the end state + if endState is None: # does not change end state + continue # no use, do nothing + for tc in allTaskClasses: # what task can further begin from there? if tc.canBeginFrom(endState) and (tc not in firstTaskTypes): - taskTypes.append(tc) # gather it + taskTypes.append(tc) # gather it if len(taskTypes) <= 0: - raise RuntimeError("No suitable task types found for state: {}".format(self._curState)) - logger.debug("[OPS] Tasks found for state {}: {}".format(self._curState, typesToStrings(taskTypes))) + raise RuntimeError( + "No suitable task types found for state: {}".format( + self._curState)) + logger.debug( + "[OPS] Tasks found for state {}: {}".format( + self._curState, + typesToStrings(taskTypes))) return taskTypes def _findCurrentState(self): dbc = self._dbConn - ts = time.time() # we use this to debug how fast/slow it is to do the various queries to find the current DB state - if dbc.query("show databases") == 0 : # no database?! + ts = time.time() # we use this to debug how fast/slow it is to do the various queries to find the current DB state + if dbc.query("show databases") == 0: # no database?! # logger.debug("Found EMPTY state") - logger.debug("[STT] empty database found, between {} and {}".format(ts, time.time())) + logger.debug( + "[STT] empty database found, between {} and {}".format( + ts, time.time())) return StateEmpty() - dbc.execute("use db") # did not do this when openning connection, and this is NOT the worker thread, which does this on their own - if dbc.query("show tables") == 0 : # no tables + # did not do this when openning connection, and this is NOT the worker + # thread, which does this on their own + dbc.execute("use db") + if dbc.query("show tables") == 0: # no tables # logger.debug("Found DB ONLY state") - logger.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time())) + logger.debug( + "[STT] DB_ONLY found, between {} and {}".format( + ts, time.time())) return StateDbOnly() - if dbc.query("SELECT * FROM db.{}".format(DbManager.getFixedSuperTableName()) ) == 0 : # no regular tables + if dbc.query("SELECT * FROM db.{}".format(DbManager.getFixedSuperTableName()) + ) == 0: # no regular tables # logger.debug("Found TABLE_ONLY state") - logger.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time())) + logger.debug( + "[STT] SUPER_TABLE_ONLY found, between {} and {}".format( + ts, time.time())) return StateSuperTableOnly() - else: # has actual tables + else: # has actual tables # logger.debug("Found HAS_DATA state") - logger.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time())) + logger.debug( + "[STT] HAS_DATA found, between {} and {}".format( + ts, time.time())) return StateHasData() def transition(self, tasks): - if ( len(tasks) == 0 ): # before 1st step, or otherwise empty + if (len(tasks) == 0): # before 1st step, or otherwise empty logger.debug("[STT] Starting State: {}".format(self._curState)) - return # do nothing + return # do nothing - self._dbConn.execute("show dnodes") # this should show up in the server log, separating steps + # this should show up in the server log, separating steps + self._dbConn.execute("show dnodes") # Generic Checks, first based on the start state if self._curState.canCreateDb(): self._curState.assertIfExistThenSuccess(tasks, TaskCreateDb) - # self.assertAtMostOneSuccess(tasks, CreateDbTask) # not really, in case of multiple creation and drops + # self.assertAtMostOneSuccess(tasks, CreateDbTask) # not really, in + # case of multiple creation and drops if self._curState.canDropDb(): self._curState.assertIfExistThenSuccess(tasks, TaskDropDb) - # self.assertAtMostOneSuccess(tasks, DropDbTask) # not really in case of drop-create-drop + # self.assertAtMostOneSuccess(tasks, DropDbTask) # not really in + # case of drop-create-drop # if self._state.canCreateFixedTable(): # self.assertIfExistThenSuccess(tasks, CreateFixedTableTask) # Not true, DB may be dropped - # self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # not really, in case of create-drop-create + # self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # not + # really, in case of create-drop-create # if self._state.canDropFixedTable(): # self.assertIfExistThenSuccess(tasks, DropFixedTableTask) # Not True, the whole DB may be dropped - # self.assertAtMostOneSuccess(tasks, DropFixedTableTask) # not really in case of drop-create-drop + # self.assertAtMostOneSuccess(tasks, DropFixedTableTask) # not + # really in case of drop-create-drop # if self._state.canAddData(): - # self.assertIfExistThenSuccess(tasks, AddFixedDataTask) # not true actually + # self.assertIfExistThenSuccess(tasks, AddFixedDataTask) # not true + # actually # if self._state.canReadData(): # Nothing for sure newState = self._findCurrentState() logger.debug("[STT] New DB state determined: {}".format(newState)) - self._curState.verifyTasksToState(tasks, newState) # can old state move to new state through the tasks? + # can old state move to new state through the tasks? + self._curState.verifyTasksToState(tasks, newState) self._curState = newState def pickTaskType(self): - taskTypes = self.getTaskTypes() # all the task types we can choose from at curent state + # all the task types we can choose from at curent state + taskTypes = self.getTaskTypes() weights = [] for tt in taskTypes: endState = tt.getEndState() - if endState != None : - weights.append(self._stateWeights[endState.getValIndex()]) # TODO: change to a method + if endState is not None: + # TODO: change to a method + weights.append(self._stateWeights[endState.getValIndex()]) else: - weights.append(10) # read data task, default to 10: TODO: change to a constant + # read data task, default to 10: TODO: change to a constant + weights.append(10) i = self._weighted_choice_sub(weights) - # logger.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes))) + # logger.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes))) return taskTypes[i] - def _weighted_choice_sub(self, weights): # ref: https://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python/ - rnd = random.random() * sum(weights) # TODO: use our dice to ensure it being determinstic? + # ref: + # https://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python/ + def _weighted_choice_sub(self, weights): + # TODO: use our dice to ensure it being determinstic? + rnd = random.random() * sum(weights) for i, w in enumerate(weights): rnd -= w if rnd < 0: return i # Manager of the Database Data/Connection -class DbManager(): - def __init__(self, resetDb = True): + + +class DbManager(): + def __init__(self, resetDb=True): self.tableNumQueue = LinearQueue() - self._lastTick = self.setupLastTick() # datetime.datetime(2019, 1, 1) # initial date time tick - self._lastInt = 0 # next one is initial integer + # datetime.datetime(2019, 1, 1) # initial date time tick + self._lastTick = self.setupLastTick() + self._lastInt = 0 # next one is initial integer self._lock = threading.RLock() - + # self.openDbServerConnection() - self._dbConn = DbConn.createNative() if (gConfig.connector_type=='native') else DbConn.createRest() + self._dbConn = DbConn.createNative() if ( + gConfig.connector_type == 'native') else DbConn.createRest() try: - self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected + self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected except taos.error.ProgrammingError as err: # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err)) - if ( err.msg == 'client disconnected' ): # cannot open DB connection - print("Cannot establish DB connection, please re-run script without parameter, and follow the instructions.") + if (err.msg == 'client disconnected'): # cannot open DB connection + print( + "Cannot establish DB connection, please re-run script without parameter, and follow the instructions.") sys.exit(2) else: - raise - except: + raise + except BaseException: print("[=] Unexpected exception") - raise + raise - if resetDb : - self._dbConn.resetDb() # drop and recreate DB + if resetDb: + self._dbConn.resetDb() # drop and recreate DB + + # Do this after dbConn is in proper shape + self._stateMachine = StateMechine(self._dbConn) - self._stateMachine = StateMechine(self._dbConn) # Do this after dbConn is in proper shape - def getDbConn(self): return self._dbConn - def getStateMachine(self) -> StateMechine : + def getStateMachine(self) -> StateMechine: return self._stateMachine # def getState(self): @@ -1043,15 +1244,18 @@ class DbManager(): def setupLastTick(self): t1 = datetime.datetime(2020, 6, 1) t2 = datetime.datetime.now() - elSec = int(t2.timestamp() - t1.timestamp()) # maybe a very large number, takes 69 years to exceed Python int range - elSec2 = ( elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500 ) ) * 500 # a number representing seconds within 10 years + # maybe a very large number, takes 69 years to exceed Python int range + elSec = int(t2.timestamp() - t1.timestamp()) + elSec2 = (elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500)) * \ + 500 # a number representing seconds within 10 years # print("elSec = {}".format(elSec)) - t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years - t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above + t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years + t4 = datetime.datetime.fromtimestamp( + t3.timestamp() + elSec2) # see explanation above logger.info("Setting up TICKS to start from: {}".format(t4)) return t4 - def pickAndAllocateTable(self): # pick any table, and "use" it + def pickAndAllocateTable(self): # pick any table, and "use" it return self.tableNumQueue.pickAndAllocate() def addTable(self): @@ -1063,15 +1267,16 @@ class DbManager(): def getFixedSuperTableName(cls): return "fs_table" - def releaseTable(self, i): # return the table back, so others can use it + def releaseTable(self, i): # return the table back, so others can use it self.tableNumQueue.release(i) def getNextTick(self): - with self._lock: # prevent duplicate tick - if Dice.throw(10) == 0 : # 1 in 10 chance + with self._lock: # prevent duplicate tick + if Dice.throw(10) == 0: # 1 in 10 chance return self._lastTick + datetime.timedelta(0, -100) - else: # regular - self._lastTick += datetime.timedelta(0, 1) # add one second to it + else: # regular + # add one second to it + self._lastTick += datetime.timedelta(0, 1) return self._lastTick def getNextInt(self): @@ -1080,29 +1285,31 @@ class DbManager(): return self._lastInt def getNextBinary(self): - return "Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_{}".format(self.getNextInt()) + return "Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_{}".format( + self.getNextInt()) def getNextFloat(self): return 0.9 + self.getNextInt() - + def getTableNameToDelete(self): - tblNum = self.tableNumQueue.pop() # TODO: race condition! - if ( not tblNum ): # maybe false + tblNum = self.tableNumQueue.pop() # TODO: race condition! + if (not tblNum): # maybe false return False - + return "table_{}".format(tblNum) def cleanUp(self): - self._dbConn.close() + self._dbConn.close() + class TaskExecutor(): class BoundedList: - def __init__(self, size = 10): + def __init__(self, size=10): self._size = size self._list = [] - def add(self, n: int) : - if not self._list: # empty + def add(self, n: int): + if not self._list: # empty self._list.append(n) return # now we should insert @@ -1110,22 +1317,22 @@ class TaskExecutor(): insPos = 0 for i in range(nItems): insPos = i - if n <= self._list[i] : # smaller than this item, time to insert - break # found the insertion point - insPos += 1 # insert to the right + if n <= self._list[i]: # smaller than this item, time to insert + break # found the insertion point + insPos += 1 # insert to the right - if insPos == 0 : # except for the 1st item, # TODO: elimiate first item as gating item - return # do nothing + if insPos == 0: # except for the 1st item, # TODO: elimiate first item as gating item + return # do nothing # print("Inserting at postion {}, value: {}".format(insPos, n)) - self._list.insert(insPos, n) # insert - + self._list.insert(insPos, n) # insert + newLen = len(self._list) - if newLen <= self._size : - return # do nothing - elif newLen == (self._size + 1) : - del self._list[0] # remove the first item - else : + if newLen <= self._size: + return # do nothing + elif newLen == (self._size + 1): + del self._list[0] # remove the first item + else: raise RuntimeError("Corrupt Bounded List") def __str__(self): @@ -1143,7 +1350,7 @@ class TaskExecutor(): def getCurStep(self): return self._curStep - def execute(self, task: Task, wt: WorkerThread): # execute a task on a thread + def execute(self, task: Task, wt: WorkerThread): # execute a task on a thread task.execute(wt) def recordDataMark(self, n: int): @@ -1156,137 +1363,164 @@ class TaskExecutor(): # def logDebug(self, msg): # logger.debug(" T[{}.x]: ".format(self._curStep) + msg) + class Task(): taskSn = 100 @classmethod def allocTaskNum(cls): - Task.taskSn += 1 # IMPORTANT: cannot use cls.taskSn, since each sub class will have a copy + Task.taskSn += 1 # IMPORTANT: cannot use cls.taskSn, since each sub class will have a copy # logger.debug("Allocating taskSN: {}".format(Task.taskSn)) return Task.taskSn - def __init__(self, dbManager: DbManager, execStats: ExecutionStats): + def __init__(self, dbManager: DbManager, execStats: ExecutionStats): self._dbManager = dbManager - self._workerThread = None + self._workerThread = None self._err = None self._aborted = False self._curStep = None - self._numRows = None # Number of rows affected + self._numRows = None # Number of rows affected - # Assign an incremental task serial number + # Assign an incremental task serial number self._taskNum = self.allocTaskNum() # logger.debug("Creating new task {}...".format(self._taskNum)) self._execStats = execStats - self._lastSql = "" # last SQL executed/attempted + self._lastSql = "" # last SQL executed/attempted def isSuccess(self): - return self._err == None + return self._err is None def isAborted(self): return self._aborted - def clone(self): # TODO: why do we need this again? + def clone(self): # TODO: why do we need this again? newTask = self.__class__(self._dbManager, self._execStats) return newTask def logDebug(self, msg): - self._workerThread.logDebug("Step[{}.{}] {}".format(self._curStep, self._taskNum, msg)) + self._workerThread.logDebug( + "Step[{}.{}] {}".format( + self._curStep, self._taskNum, msg)) def logInfo(self, msg): - self._workerThread.logInfo("Step[{}.{}] {}".format(self._curStep, self._taskNum, msg)) + self._workerThread.logInfo( + "Step[{}.{}] {}".format( + self._curStep, self._taskNum, msg)) def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - raise RuntimeError("To be implemeted by child classes, class name: {}".format(self.__class__.__name__)) + raise RuntimeError( + "To be implemeted by child classes, class name: {}".format( + self.__class__.__name__)) def execute(self, wt: WorkerThread): wt.verifyThreadSelf() - self._workerThread = wt # type: ignore + self._workerThread = wt # type: ignore te = wt.getTaskExecutor() self._curStep = te.getCurStep() - self.logDebug("[-] executing task {}...".format(self.__class__.__name__)) + self.logDebug( + "[-] executing task {}...".format(self.__class__.__name__)) self._err = None - self._execStats.beginTaskType(self.__class__.__name__) # mark beginning + self._execStats.beginTaskType( + self.__class__.__name__) # mark beginning try: - self._executeInternal(te, wt) # TODO: no return value? + self._executeInternal(te, wt) # TODO: no return value? except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correct error scheme - if ( errno2 in [ - 0x05, # TSDB_CODE_RPC_NOT_READY - 0x200, 0x360, 0x362, 0x36A, 0x36B, 0x36D, 0x381, 0x380, 0x383, 0x503, - 0x510, # vnode not in ready state + errno2 = err.errno if ( + err.errno > 0) else 0x80000000 + err.errno # correct error scheme + if (gConfig.continue_on_exception): # user choose to continue + self.logDebug( + "[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format( + errno2, err, self._lastSql)) + self._err = err + elif (errno2 in [ + 0x05, # TSDB_CODE_RPC_NOT_READY + 0x200, 0x360, 0x362, 0x36A, 0x36B, 0x36D, + 0x381, 0x380, 0x383, + 0x386, # DB is being dropped?! + 0x503, + 0x510, # vnode not in ready state 0x600, - 1000 # REST catch-all error - ]) : # allowed errors - self.logDebug("[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, self._lastSql)) + 1000 # REST catch-all error + ]): # allowed errors + self.logDebug( + "[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format( + errno2, err, self._lastSql)) print("_", end="", flush=True) - self._err = err + self._err = err else: - errMsg = "[=] Unexpected Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, self._lastSql) + errMsg = "[=] Unexpected Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format( + errno2, err, self._lastSql) self.logDebug(errMsg) - if gConfig.debug : + if gConfig.debug: # raise # so that we see full stack traceback.print_exc() - print("\n\n----------------------------\nProgram ABORTED Due to Unexpected TAOS Error: \n\n{}\n".format(errMsg) + + print( + "\n\n----------------------------\nProgram ABORTED Due to Unexpected TAOS Error: \n\n{}\n".format(errMsg) + "----------------------------\n") # sys.exit(-1) self._err = err self._aborted = True - except Exception as e : + except Exception as e: self.logInfo("Non-TAOS exception encountered") - self._err = e + self._err = e self._aborted = True traceback.print_exc() - except BaseException as e : + except BaseException as e: self.logInfo("Python base exception encountered") - self._err = e + self._err = e self._aborted = True traceback.print_exc() - except : - self.logDebug("[=] Unexpected exception, SQL: {}".format(self._lastSql)) + except BaseException: + self.logDebug( + "[=] Unexpected exception, SQL: {}".format( + self._lastSql)) raise self._execStats.endTaskType(self.__class__.__name__, self.isSuccess()) - - self.logDebug("[X] task execution completed, {}, status: {}".format(self.__class__.__name__, "Success" if self.isSuccess() else "Failure")) - self._execStats.incExecCount(self.__class__.__name__, self.isSuccess()) # TODO: merge with above. + + self.logDebug("[X] task execution completed, {}, status: {}".format( + self.__class__.__name__, "Success" if self.isSuccess() else "Failure")) + # TODO: merge with above. + self._execStats.incExecCount(self.__class__.__name__, self.isSuccess()) def execSql(self, sql): self._lastSql = sql return self._dbManager.execute(sql) - def execWtSql(self, wt: WorkerThread, sql): # execute an SQL on the worker thread + def execWtSql(self, wt: WorkerThread, sql): # execute an SQL on the worker thread self._lastSql = sql return wt.execSql(sql) - def queryWtSql(self, wt: WorkerThread, sql): # execute an SQL on the worker thread + def queryWtSql(self, wt: WorkerThread, sql): # execute an SQL on the worker thread self._lastSql = sql return wt.querySql(sql) - def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread + def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread return wt.getQueryResult() - class ExecutionStats: def __init__(self): - self._execTimes: Dict[str, [int, int]] = {} # total/success times for a task + # total/success times for a task + self._execTimes: Dict[str, [int, int]] = {} self._tasksInProgress = 0 self._lock = threading.Lock() self._firstTaskStartTime = None self._execStartTime = None - self._elapsedTime = 0.0 # total elapsed time - self._accRunTime = 0.0 # accumulated run time + self._elapsedTime = 0.0 # total elapsed time + self._accRunTime = 0.0 # accumulated run time self._failed = False self._failureReason = None def __str__(self): - return "[ExecStats: _failed={}, _failureReason={}".format(self._failed, self._failureReason) + return "[ExecStats: _failed={}, _failureReason={}".format( + self._failed, self._failureReason) def isFailed(self): - return self._failed == True + return self._failed def startExec(self): self._execStartTime = time.time() @@ -1294,24 +1528,24 @@ class ExecutionStats: def endExec(self): self._elapsedTime = time.time() - self._execStartTime - def incExecCount(self, klassName, isSuccess): # TODO: add a lock here + def incExecCount(self, klassName, isSuccess): # TODO: add a lock here if klassName not in self._execTimes: self._execTimes[klassName] = [0, 0] - t = self._execTimes[klassName] # tuple for the data - t[0] += 1 # index 0 has the "total" execution times + t = self._execTimes[klassName] # tuple for the data + t[0] += 1 # index 0 has the "total" execution times if isSuccess: - t[1] += 1 # index 1 has the "success" execution times + t[1] += 1 # index 1 has the "success" execution times def beginTaskType(self, klassName): with self._lock: - if self._tasksInProgress == 0 : # starting a new round - self._firstTaskStartTime = time.time() # I am now the first task + if self._tasksInProgress == 0: # starting a new round + self._firstTaskStartTime = time.time() # I am now the first task self._tasksInProgress += 1 def endTaskType(self, klassName, isSuccess): with self._lock: self._tasksInProgress -= 1 - if self._tasksInProgress == 0 : # all tasks have stopped + if self._tasksInProgress == 0: # all tasks have stopped self._accRunTime += (time.time() - self._firstTaskStartTime) self._firstTaskStartTime = None @@ -1320,23 +1554,36 @@ class ExecutionStats: self._failureReason = reason def printStats(self): - logger.info("----------------------------------------------------------------------") - logger.info("| Crash_Gen test {}, with the following stats:". - format("FAILED (reason: {})".format(self._failureReason) if self._failed else "SUCCEEDED")) + logger.info( + "----------------------------------------------------------------------") + logger.info( + "| Crash_Gen test {}, with the following stats:". format( + "FAILED (reason: {})".format( + self._failureReason) if self._failed else "SUCCEEDED")) logger.info("| Task Execution Times (success/total):") execTimesAny = 0 - for k, n in self._execTimes.items(): + for k, n in self._execTimes.items(): execTimesAny += n[0] - logger.info("| {0:<24}: {1}/{2}".format(k,n[1],n[0])) - - logger.info("| Total Tasks Executed (success or not): {} ".format(execTimesAny)) - logger.info("| Total Tasks In Progress at End: {}".format(self._tasksInProgress)) - logger.info("| Total Task Busy Time (elapsed time when any task is in progress): {:.3f} seconds".format(self._accRunTime)) - logger.info("| Average Per-Task Execution Time: {:.3f} seconds".format(self._accRunTime/execTimesAny)) - logger.info("| Total Elapsed Time (from wall clock): {:.3f} seconds".format(self._elapsedTime)) - logger.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList())) - logger.info("----------------------------------------------------------------------") - + logger.info("| {0:<24}: {1}/{2}".format(k, n[1], n[0])) + + logger.info( + "| Total Tasks Executed (success or not): {} ".format(execTimesAny)) + logger.info( + "| Total Tasks In Progress at End: {}".format( + self._tasksInProgress)) + logger.info( + "| Total Task Busy Time (elapsed time when any task is in progress): {:.3f} seconds".format( + self._accRunTime)) + logger.info( + "| Average Per-Task Execution Time: {:.3f} seconds".format(self._accRunTime / execTimesAny)) + logger.info( + "| Total Elapsed Time (from wall clock): {:.3f} seconds".format( + self._elapsedTime)) + logger.info( + "| Top numbers written: {}".format( + TaskExecutor.getBoundedList())) + logger.info( + "----------------------------------------------------------------------") class StateTransitionTask(Task): @@ -1346,12 +1593,12 @@ class StateTransitionTask(Task): SMALL_NUMBER_OF_RECORDS = 3 @classmethod - def getInfo(cls): # each sub class should supply their own information + def getInfo(cls): # each sub class should supply their own information raise RuntimeError("Overriding method expected") - _endState = None + _endState = None @classmethod - def getEndState(cls): # TODO: optimize by calling it fewer times + def getEndState(cls): # TODO: optimize by calling it fewer times raise RuntimeError("Overriding method expected") # @classmethod @@ -1373,18 +1620,20 @@ class StateTransitionTask(Task): def execute(self, wt: WorkerThread): super().execute(wt) - + + class TaskCreateDb(StateTransitionTask): @classmethod def getEndState(cls): - return StateDbOnly() + return StateDbOnly() @classmethod def canBeginFrom(cls, state: AnyState): return state.canCreateDb() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - self.execWtSql(wt, "create database db") + self.execWtSql(wt, "create database db") + class TaskDropDb(StateTransitionTask): @classmethod @@ -1399,6 +1648,7 @@ class TaskDropDb(StateTransitionTask): self.execWtSql(wt, "drop database db") logger.debug("[OPS] database dropped at {}".format(time.time())) + class TaskCreateSuperTable(StateTransitionTask): @classmethod def getEndState(cls): @@ -1409,115 +1659,135 @@ class TaskCreateSuperTable(StateTransitionTask): return state.canCreateFixedSuperTable() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - if not wt.dbInUse(): # no DB yet, to the best of our knowledge + if not wt.dbInUse(): # no DB yet, to the best of our knowledge logger.debug("Skipping task, no DB yet") return - tblName = self._dbManager.getFixedSuperTableName() + tblName = self._dbManager.getFixedSuperTableName() # wt.execSql("use db") # should always be in place - self.execWtSql(wt, "create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) - # No need to create the regular tables, INSERT will do that automatically + self.execWtSql( + wt, + "create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) + # No need to create the regular tables, INSERT will do that + # automatically class TaskReadData(StateTransitionTask): @classmethod def getEndState(cls): - return None # meaning doesn't affect state + return None # meaning doesn't affect state @classmethod def canBeginFrom(cls, state: AnyState): return state.canReadData() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - sTbName = self._dbManager.getFixedSuperTableName() - self.queryWtSql(wt, "select TBNAME from db.{}".format(sTbName)) # TODO: analyze result set later + sTbName = self._dbManager.getFixedSuperTableName() + self.queryWtSql(wt, "select TBNAME from db.{}".format( + sTbName)) # TODO: analyze result set later - if random.randrange(5) == 0 : # 1 in 5 chance, simulate a broken connection. TODO: break connection in all situations + if random.randrange( + 5) == 0: # 1 in 5 chance, simulate a broken connection. TODO: break connection in all situations wt.getDbConn().close() wt.getDbConn().open() else: - rTables = self.getQueryResult(wt) # wt.getDbConn().getQueryResult() + # wt.getDbConn().getQueryResult() + rTables = self.getQueryResult(wt) # print("rTables[0] = {}, type = {}".format(rTables[0], type(rTables[0]))) - for rTbName in rTables : # regular tables + for rTbName in rTables: # regular tables self.execWtSql(wt, "select * from db.{}".format(rTbName[0])) # tdSql.query(" cars where tbname in ('carzero', 'carone')") + class TaskDropSuperTable(StateTransitionTask): @classmethod def getEndState(cls): - return StateDbOnly() + return StateDbOnly() @classmethod def canBeginFrom(cls, state: AnyState): return state.canDropFixedSuperTable() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - # 1/2 chance, we'll drop the regular tables one by one, in a randomized sequence - if Dice.throw(2) == 0 : - tblSeq = list(range(2 + (self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))) - random.shuffle(tblSeq) - tickOutput = False # if we have spitted out a "d" character for "drop regular table" + # 1/2 chance, we'll drop the regular tables one by one, in a randomized + # sequence + if Dice.throw(2) == 0: + tblSeq = list(range( + 2 + (self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))) + random.shuffle(tblSeq) + tickOutput = False # if we have spitted out a "d" character for "drop regular table" isSuccess = True - for i in tblSeq: - regTableName = self.getRegTableName(i); # "db.reg_table_{}".format(i) + for i in tblSeq: + regTableName = self.getRegTableName( + i) # "db.reg_table_{}".format(i) try: - self.execWtSql(wt, "drop table {}".format(regTableName)) # nRows always 0, like MySQL - except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correcting for strange error number scheme - if ( errno2 in [0x362]) : # mnode invalid table name + self.execWtSql(wt, "drop table {}".format( + regTableName)) # nRows always 0, like MySQL + except taos.error.ProgrammingError as err: + # correcting for strange error number scheme + errno2 = err.errno if ( + err.errno > 0) else 0x80000000 + err.errno + if (errno2 in [0x362]): # mnode invalid table name isSuccess = False - logger.debug("[DB] Acceptable error when dropping a table") - continue # try to delete next regular table + logger.debug( + "[DB] Acceptable error when dropping a table") + continue # try to delete next regular table if (not tickOutput): - tickOutput = True # Print only one time - if isSuccess : + tickOutput = True # Print only one time + if isSuccess: print("d", end="", flush=True) else: - print("f", end="", flush=True) + print("f", end="", flush=True) # Drop the super table itself - tblName = self._dbManager.getFixedSuperTableName() + tblName = self._dbManager.getFixedSuperTableName() self.execWtSql(wt, "drop table db.{}".format(tblName)) + class TaskAlterTags(StateTransitionTask): @classmethod def getEndState(cls): - return None # meaning doesn't affect state + return None # meaning doesn't affect state @classmethod def canBeginFrom(cls, state: AnyState): - return state.canDropFixedSuperTable() # if we can drop it, we can alter tags + return state.canDropFixedSuperTable() # if we can drop it, we can alter tags def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - tblName = self._dbManager.getFixedSuperTableName() + tblName = self._dbManager.getFixedSuperTableName() dice = Dice.throw(4) - if dice == 0 : + if dice == 0: sql = "alter table db.{} add tag extraTag int".format(tblName) - elif dice == 1 : + elif dice == 1: sql = "alter table db.{} drop tag extraTag".format(tblName) - elif dice == 2 : + elif dice == 2: sql = "alter table db.{} drop tag newTag".format(tblName) - else: # dice == 3 - sql = "alter table db.{} change tag extraTag newTag".format(tblName) + else: # dice == 3 + sql = "alter table db.{} change tag extraTag newTag".format( + tblName) self.execWtSql(wt, sql) -class TaskAddData(StateTransitionTask): - activeTable : Set[int] = set() # Track which table is being actively worked on - # We use these two files to record operations to DB, useful for power-off tests +class TaskAddData(StateTransitionTask): + # Track which table is being actively worked on + activeTable: Set[int] = set() + + # We use these two files to record operations to DB, useful for power-off + # tests fAddLogReady = None fAddLogDone = None @classmethod def prepToRecordOps(cls): - if gConfig.record_ops : - if ( cls.fAddLogReady == None ): - logger.info("Recording in a file operations to be performed...") + if gConfig.record_ops: + if (cls.fAddLogReady is None): + logger.info( + "Recording in a file operations to be performed...") cls.fAddLogReady = open("add_log_ready.txt", "w") - if ( cls.fAddLogDone == None ): + if (cls.fAddLogDone is None): logger.info("Recording in a file operations completed...") cls.fAddLogDone = open("add_log_done.txt", "w") @@ -1528,78 +1798,92 @@ class TaskAddData(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): return state.canAddData() - + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): ds = self._dbManager - # wt.execSql("use db") # TODO: seems to be an INSERT bug to require this - tblSeq = list(range(self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES)) - random.shuffle(tblSeq) - for i in tblSeq: - if ( i in self.activeTable ): # wow already active - # logger.info("Concurrent data insertion into table: {}".format(i)) - # print("ct({})".format(i), end="", flush=True) # Concurrent insertion into table + # wt.execSql("use db") # TODO: seems to be an INSERT bug to require + # this + tblSeq = list( + range( + self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES)) + random.shuffle(tblSeq) + for i in tblSeq: + if (i in self.activeTable): # wow already active + # logger.info("Concurrent data insertion into table: {}".format(i)) + # print("ct({})".format(i), end="", flush=True) # Concurrent + # insertion into table print("x", end="", flush=True) else: - self.activeTable.add(i) # marking it active - # No need to shuffle data sequence, unless later we decide to do non-increment insertion - regTableName = self.getRegTableName(i); # "db.reg_table_{}".format(i) - for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS) : # number of records per table - nextInt = ds.getNextInt() + self.activeTable.add(i) # marking it active + # No need to shuffle data sequence, unless later we decide to do + # non-increment insertion + regTableName = self.getRegTableName( + i) # "db.reg_table_{}".format(i) + for j in range( + self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table + nextInt = ds.getNextInt() if gConfig.record_ops: self.prepToRecordOps() - self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) + self.fAddLogReady.write( + "Ready to write {} to {}\n".format( + nextInt, regTableName)) self.fAddLogReady.flush() os.fsync(self.fAddLogReady) sql = "insert into {} using {} tags ('{}', {}) values ('{}', {});".format( - regTableName, - ds.getFixedSuperTableName(), + regTableName, + ds.getFixedSuperTableName(), ds.getNextBinary(), ds.getNextFloat(), ds.getNextTick(), nextInt) - self.execWtSql(wt, sql) - # Successfully wrote the data into the DB, let's record it somehow + self.execWtSql(wt, sql) + # Successfully wrote the data into the DB, let's record it + # somehow te.recordDataMark(nextInt) if gConfig.record_ops: - self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) + self.fAddLogDone.write( + "Wrote {} to {}\n".format( + nextInt, regTableName)) self.fAddLogDone.flush() os.fsync(self.fAddLogDone) - self.activeTable.discard(i) # not raising an error, unlike remove + self.activeTable.discard(i) # not raising an error, unlike remove # Deterministic random number generator class Dice(): - seeded = False # static, uninitialized + seeded = False # static, uninitialized @classmethod - def seed(cls, s): # static + def seed(cls, s): # static if (cls.seeded): - raise RuntimeError("Cannot seed the random generator more than once") + raise RuntimeError( + "Cannot seed the random generator more than once") cls.verifyRNG() random.seed(s) cls.seeded = True # TODO: protect against multi-threading @classmethod - def verifyRNG(cls): # Verify that the RNG is determinstic + def verifyRNG(cls): # Verify that the RNG is determinstic random.seed(0) x1 = random.randrange(0, 1000) x2 = random.randrange(0, 1000) x3 = random.randrange(0, 1000) - if ( x1 != 864 or x2!=394 or x3!=776 ): + if (x1 != 864 or x2 != 394 or x3 != 776): raise RuntimeError("System RNG is not deterministic") @classmethod - def throw(cls, stop): # get 0 to stop-1 + def throw(cls, stop): # get 0 to stop-1 return cls.throwRange(0, stop) @classmethod - def throwRange(cls, start, stop): # up to stop-1 - if ( not cls.seeded ): + def throwRange(cls, start, stop): # up to stop-1 + if (not cls.seeded): raise RuntimeError("Cannot throw dice before seeding it") return random.randrange(start, stop) + class LoggingFilter(logging.Filter): def filter(self, record: logging.LogRecord): - if ( record.levelno >= logging.INFO ) : - return True # info or above always log + if (record.levelno >= logging.INFO): + return True # info or above always log # Commenting out below to adjust... @@ -1607,20 +1891,23 @@ class LoggingFilter(logging.Filter): # return False return True -class MyLoggingAdapter(logging.LoggerAdapter): + +class MyLoggingAdapter(logging.LoggerAdapter): def process(self, msg, kwargs): return "[{}]{}".format(threading.get_ident() % 10000, msg), kwargs # return '[%s] %s' % (self.extra['connid'], msg), kwargs -class SvcManager: + +class SvcManager: def __init__(self): print("Starting TDengine Service Manager") signal.signal(signal.SIGTERM, self.sigIntHandler) - signal.signal(signal.SIGINT, self.sigIntHandler) - signal.signal(signal.SIGUSR1, self.sigUsrHandler) # different handler! - + signal.signal(signal.SIGINT, self.sigIntHandler) + signal.signal(signal.SIGUSR1, self.sigUsrHandler) # different handler! + self.inSigHandler = False - # self._status = MainExec.STATUS_RUNNING # set inside _startTaosService() + # self._status = MainExec.STATUS_RUNNING # set inside + # _startTaosService() self.svcMgrThread = None def _doMenu(self): @@ -1631,30 +1918,32 @@ class SvcManager: print("2: Terminate") print("3: Restart") # Remember to update the if range below - # print("Enter Choice: ", end="", flush=True) + # print("Enter Choice: ", end="", flush=True) while choice == "": choice = input("Enter Choice: ") if choice != "": - break # done with reading repeated input - if choice in ["1", "2", "3"]: - break # we are done with whole method + break # done with reading repeated input + if choice in ["1", "2", "3"]: + break # we are done with whole method print("Invalid choice, please try again.") - choice = "" # reset + choice = "" # reset return choice - def sigUsrHandler(self, signalNumber, frame) : + def sigUsrHandler(self, signalNumber, frame): print("Interrupting main thread execution upon SIGUSR1") - if self.inSigHandler : # already + if self.inSigHandler: # already print("Ignoring repeated SIG...") - return # do nothing if it's already not running + return # do nothing if it's already not running self.inSigHandler = True choice = self._doMenu() - if choice == "1" : - self.sigHandlerResume() # TODO: can the sub-process be blocked due to us not reading from queue? - elif choice == "2" : + if choice == "1": + # TODO: can the sub-process be blocked due to us not reading from + # queue? + self.sigHandlerResume() + elif choice == "2": self.stopTaosService() - elif choice == "3" : + elif choice == "3": self.stopTaosService() self.startTaosService() else: @@ -1664,48 +1953,52 @@ class SvcManager: def sigIntHandler(self, signalNumber, frame): print("Sig INT Handler starting...") - if self.inSigHandler : + if self.inSigHandler: print("Ignoring repeated SIG_INT...") return self.inSigHandler = True - - self.stopTaosService() - print("INT signal handler returning...") + + self.stopTaosService() + print("INT signal handler returning...") self.inSigHandler = False - def sigHandlerResume(self) : + def sigHandlerResume(self): print("Resuming TDengine service manager thread (main thread)...\n\n") - + def _checkServiceManagerThread(self): - if self.svcMgrThread: # valid svc mgr thread - if self.svcMgrThread.isStopped(): # done? - self.svcMgrThread.procIpcBatch() # one last time. TODO: appropriate? - self.svcMgrThread = None # no more + if self.svcMgrThread: # valid svc mgr thread + if self.svcMgrThread.isStopped(): # done? + self.svcMgrThread.procIpcBatch() # one last time. TODO: appropriate? + self.svcMgrThread = None # no more def _procIpcAll(self): - while self.svcMgrThread : # for as long as the svc mgr thread is still here - self.svcMgrThread.procIpcBatch() # regular processing, - time.sleep(0.5) # pause, before next round - self._checkServiceManagerThread() - print("Service Manager Thread (with subprocess) has ended, main thread now exiting...") + while self.svcMgrThread: # for as long as the svc mgr thread is still here + self.svcMgrThread.procIpcBatch() # regular processing, + time.sleep(0.5) # pause, before next round + self._checkServiceManagerThread() + print( + "Service Manager Thread (with subprocess) has ended, main thread now exiting...") - def startTaosService(self): + def startTaosService(self): if self.svcMgrThread: - raise RuntimeError("Cannot start TAOS service when one may already be running") - self.svcMgrThread = ServiceManagerThread() # create the object + raise RuntimeError( + "Cannot start TAOS service when one may already be running") + self.svcMgrThread = ServiceManagerThread() # create the object self.svcMgrThread.start() print("TAOS service started, printing out output...") - self.svcMgrThread.procIpcBatch(trimToTarget=10, forceOutput=True) # for printing 10 lines + self.svcMgrThread.procIpcBatch( + trimToTarget=10, + forceOutput=True) # for printing 10 lines print("TAOS service started") - - def stopTaosService(self, outputLines = 20): + + def stopTaosService(self, outputLines=20): print("Terminating Service Manager Thread (SMT) execution...") if not self.svcMgrThread: raise RuntimeError("Unexpected empty svc mgr thread") self.svcMgrThread.stop() if self.svcMgrThread.isStopped(): - self.svcMgrThread.procIpcBatch(outputLines) # one last time - self.svcMgrThread = None + self.svcMgrThread.procIpcBatch(outputLines) # one last time + self.svcMgrThread = None print("----- End of TDengine Service Output -----\n") print("SMT execution terminated") else: @@ -1713,16 +2006,17 @@ class SvcManager: def run(self): self.startTaosService() - self._procIpcAll() # pump/process all the messages - if self.svcMgrThread: # if sig handler hasn't destroyed it by now - self.stopTaosService() # should have started already - + self._procIpcAll() # pump/process all the messages + if self.svcMgrThread: # if sig handler hasn't destroyed it by now + self.stopTaosService() # should have started already + + class ServiceManagerThread: MAX_QUEUE_SIZE = 10000 def __init__(self): self._tdeSubProcess = None - self._thread = None + self._thread = None self._status = None def getStatus(self): @@ -1740,98 +2034,107 @@ class ServiceManagerThread: # Start the thread (with sub process), and wait for the sub service # to become fully operational - def start(self): - if self._thread : + def start(self): + if self._thread: raise RuntimeError("Unexpected _thread") - if self._tdeSubProcess : + if self._tdeSubProcess: raise RuntimeError("TDengine sub process already created/running") self._status = MainExec.STATUS_STARTING - self._tdeSubProcess = TdeSubProcess() + self._tdeSubProcess = TdeSubProcess() self._tdeSubProcess.start() self._ipcQueue = Queue() self._thread = threading.Thread( - target=self.svcOutputReader, + target=self.svcOutputReader, args=(self._tdeSubProcess.getStdOut(), self._ipcQueue)) - self._thread.daemon = True # thread dies with the program + self._thread.daemon = True # thread dies with the program self._thread.start() # wait for service to start - for i in range(0, 10) : + for i in range(0, 10): time.sleep(1.0) # self.procIpcBatch() # don't pump message during start up print("_zz_", end="", flush=True) - if self._status == MainExec.STATUS_RUNNING : + if self._status == MainExec.STATUS_RUNNING: logger.info("[] TDengine service READY to process requests") - return # now we've started - raise RuntimeError("TDengine service did not start successfully") # TODO: handle this better? + return # now we've started + # TODO: handle this better? + raise RuntimeError("TDengine service did not start successfully") def stop(self): # can be called from both main thread or signal handler print("Terminating TDengine service running as the sub process...") if self.isStopped(): print("Service already stopped") - return + return if self.isStopping(): print("Service is already being stopped") return - # Linux will send Control-C generated SIGINT to the TDengine process already, ref: https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes - if not self._tdeSubProcess : + # Linux will send Control-C generated SIGINT to the TDengine process + # already, ref: + # https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes + if not self._tdeSubProcess: raise RuntimeError("sub process object missing") self._status = MainExec.STATUS_STOPPING self._tdeSubProcess.stop() - if self._tdeSubProcess.isRunning(): # still running - print("FAILED to stop sub process, it is still running... pid = {}".format(self.subProcess.pid)) + if self._tdeSubProcess.isRunning(): # still running + print( + "FAILED to stop sub process, it is still running... pid = {}".format( + self.subProcess.pid)) else: - self._tdeSubProcess = None # not running any more - self.join() # stop the thread, change the status, etc. - + self._tdeSubProcess = None # not running any more + self.join() # stop the thread, change the status, etc. + def join(self): # TODO: sanity check if not self.isStopping(): - raise RuntimeError("Unexpected status when ending svc mgr thread: {}".format(self._status)) + raise RuntimeError( + "Unexpected status when ending svc mgr thread: {}".format( + self._status)) - if self._thread : + if self._thread: self._thread.join() - self._thread = None + self._thread = None self._status = MainExec.STATUS_STOPPED - else : + else: print("Joining empty thread, doing nothing") def _trimQueue(self, targetSize): if targetSize <= 0: - return # do nothing + return # do nothing q = self._ipcQueue - if (q.qsize() <= targetSize ) : # no need to trim + if (q.qsize() <= targetSize): # no need to trim return logger.debug("Triming IPC queue to target size: {}".format(targetSize)) itemsToTrim = q.qsize() - targetSize - for i in range(0, itemsToTrim) : + for i in range(0, itemsToTrim): try: q.get_nowait() except Empty: - break # break out of for loop, no more trimming - + break # break out of for loop, no more trimming + TD_READY_MSG = "TDengine is initialized successfully" - def procIpcBatch(self, trimToTarget = 0, forceOutput = False): - self._trimQueue(trimToTarget) # trim if necessary - # Process all the output generated by the underlying sub process, managed by IO thread + + def procIpcBatch(self, trimToTarget=0, forceOutput=False): + self._trimQueue(trimToTarget) # trim if necessary + # Process all the output generated by the underlying sub process, + # managed by IO thread print("<", end="", flush=True) - while True : - try: - line = self._ipcQueue.get_nowait() # getting output at fast speed + while True: + try: + line = self._ipcQueue.get_nowait() # getting output at fast speed self._printProgress("_o") except Empty: # time.sleep(2.3) # wait only if there's no output # no more output print(".>", end="", flush=True) - return # we are done with THIS BATCH - else: # got line, printing out + return # we are done with THIS BATCH + else: # got line, printing out if forceOutput: logger.info(line) else: @@ -1839,8 +2142,9 @@ class ServiceManagerThread: print(">", end="", flush=True) _ProgressBars = ["--", "//", "||", "\\\\"] - def _printProgress(self, msg): # TODO: assuming 2 chars - print(msg, end="", flush=True) + + def _printProgress(self, msg): # TODO: assuming 2 chars + print(msg, end="", flush=True) pBar = self._ProgressBars[Dice.throw(4)] print(pBar, end="", flush=True) print('\b\b\b\b', end="", flush=True) @@ -1848,29 +2152,33 @@ class ServiceManagerThread: def svcOutputReader(self, out: IO, queue): # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python # print("This is the svcOutput Reader...") - # for line in out : + # for line in out : for line in iter(out.readline, b''): # print("Finished reading a line: {}".format(line)) # print("Adding item to queue...") line = line.decode("utf-8").rstrip() - queue.put(line) # This might block, and then causing "out" buffer to block + # This might block, and then causing "out" buffer to block + queue.put(line) self._printProgress("_i") - if self._status == MainExec.STATUS_STARTING : # we are starting, let's see if we have started - if line.find(self.TD_READY_MSG) != -1 : # found - self._status = MainExec.STATUS_RUNNING + if self._status == MainExec.STATUS_STARTING: # we are starting, let's see if we have started + if line.find(self.TD_READY_MSG) != -1: # found + self._status = MainExec.STATUS_RUNNING # Trim the queue if necessary: TODO: try this 1 out of 10 times - self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size + self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size - if self.isStopping() : # TODO: use thread status instead - print("_w", end="", flush=True) # WAITING for stopping sub process to finish its outptu + if self.isStopping(): # TODO: use thread status instead + # WAITING for stopping sub process to finish its outptu + print("_w", end="", flush=True) # queue.put(line) - print("\nNo more output from IO thread managing TDengine service") # meaning sub process must have died + # meaning sub process must have died + print("\nNo more output from IO thread managing TDengine service") out.close() -class TdeSubProcess: + +class TdeSubProcess: def __init__(self): self.subProcess = None @@ -1878,20 +2186,39 @@ class TdeSubProcess: return self.subProcess.stdout def isRunning(self): - return self.subProcess != None + return self.subProcess is not None + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("communit")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath def start(self): ON_POSIX = 'posix' in sys.builtin_module_names - svcCmd = ['../../build/build/bin/taosd', '-c', '../../build/test/cfg'] + + taosdPath = self.getBuildPath() + "/build/bin/taosd" + cfgPath = self.getBuildPath() + "/test/cfg" + + svcCmd = [taosdPath, '-c', cfgPath] # svcCmd = ['vmstat', '1'] - if self.subProcess : # already there + if self.subProcess: # already there raise RuntimeError("Corrupt process state") self.subProcess = subprocess.Popen( - svcCmd, - stdout=subprocess.PIPE, + svcCmd, + stdout=subprocess.PIPE, # bufsize=1, # not supported in binary mode - close_fds=ON_POSIX) # had text=True, which interferred with reading EOF + close_fds=ON_POSIX) # had text=True, which interferred with reading EOF def stop(self): if not self.subProcess: @@ -1899,12 +2226,15 @@ class TdeSubProcess: return retCode = self.subProcess.poll() - if retCode : # valid return code, process ended + if retCode: # valid return code, process ended self.subProcess = None - else: # process still alive, let's interrupt it - print("Sub process is running, sending SIG_INT and waiting for it to terminate...") - self.subProcess.send_signal(signal.SIGINT) # sub process should end, then IPC queue should end, causing IO thread to end - try : + else: # process still alive, let's interrupt it + print( + "Sub process is running, sending SIG_INT and waiting for it to terminate...") + # sub process should end, then IPC queue should end, causing IO + # thread to end + self.subProcess.send_signal(signal.SIGINT) + try: self.subProcess.wait(10) except subprocess.TimeoutExpired as err: print("Time out waiting for TDengine service process to exit") @@ -1912,6 +2242,7 @@ class TdeSubProcess: print("TDengine service process terminated successfully from SIG_INT") self.subProcess = None + class ClientManager: def __init__(self): print("Starting service manager") @@ -1922,76 +2253,78 @@ class ClientManager: self.tc = None def sigIntHandler(self, signalNumber, frame): - if self._status != MainExec.STATUS_RUNNING : + if self._status != MainExec.STATUS_RUNNING: print("Ignoring repeated SIGINT...") - return # do nothing if it's already not running - self._status = MainExec.STATUS_STOPPING # immediately set our status + return # do nothing if it's already not running + self._status = MainExec.STATUS_STOPPING # immediately set our status print("Terminating program...") self.tc.requestToStop() - def _printLastNumbers(self): # to verify data durability + def _printLastNumbers(self): # to verify data durability dbManager = DbManager(resetDb=False) dbc = dbManager.getDbConn() - if dbc.query("show databases") == 0 : # no databae + if dbc.query("show databases") == 0: # no databae return - if dbc.query("show tables") == 0 : # no tables + if dbc.query("show tables") == 0: # no tables return dbc.execute("use db") - sTbName = dbManager.getFixedSuperTableName() + sTbName = dbManager.getFixedSuperTableName() # get all regular tables - dbc.query("select TBNAME from db.{}".format(sTbName)) # TODO: analyze result set later + # TODO: analyze result set later + dbc.query("select TBNAME from db.{}".format(sTbName)) rTables = dbc.getQueryResult() bList = TaskExecutor.BoundedList() - for rTbName in rTables : # regular tables + for rTbName in rTables: # regular tables dbc.query("select speed from db.{}".format(rTbName[0])) numbers = dbc.getQueryResult() - for row in numbers : + for row in numbers: # print("<{}>".format(n), end="", flush=True) bList.add(row[0]) print("Top numbers in DB right now: {}".format(bList)) print("TDengine client execution is about to start in 2 seconds...") time.sleep(2.0) - dbManager = None # release? + dbManager = None # release? def prepare(self): self._printLastNumbers() def run(self): - if gConfig.auto_start_service : + if gConfig.auto_start_service: svcMgr = SvcManager() svcMgr.startTaosService() self._printLastNumbers() - dbManager = DbManager() # Regular function + dbManager = DbManager() # Regular function thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps) self.tc = ThreadCoordinator(thPool, dbManager) - + self.tc.run() # print("exec stats: {}".format(self.tc.getExecStats())) - # print("TC failed = {}".format(self.tc.isFailed())) - if gConfig.auto_start_service : + # print("TC failed = {}".format(self.tc.isFailed())) + if gConfig.auto_start_service: svcMgr.stopTaosService() # Print exec status, etc., AFTER showing messages from the server self.conclude() # print("TC failed (2) = {}".format(self.tc.isFailed())) - return 1 if self.tc.isFailed() else 0 # Linux return code: ref https://shapeshed.com/unix-exit-codes/ + # Linux return code: ref https://shapeshed.com/unix-exit-codes/ + return 1 if self.tc.isFailed() else 0 def conclude(self): self.tc.printStats() - self.tc.getDbManager().cleanUp() + self.tc.getDbManager().cleanUp() class MainExec: STATUS_STARTING = 1 - STATUS_RUNNING = 2 + STATUS_RUNNING = 2 STATUS_STOPPING = 3 - STATUS_STOPPED = 4 + STATUS_STOPPED = 4 @classmethod def runClient(cls): @@ -2004,13 +2337,13 @@ class MainExec: svcManager.run() @classmethod - def runTemp(cls): # for debugging purposes + def runTemp(cls): # for debugging purposes # # Hack to exercise reading from disk, imcreasing coverage. TODO: fix # dbc = dbState.getDbConn() - # sTbName = dbState.getFixedSuperTableName() + # sTbName = dbState.getFixedSuperTableName() # dbc.execute("create database if not exists db") # if not dbState.getState().equals(StateEmpty()): - # dbc.execute("use db") + # dbc.execute("use db") # rTables = None # try: # the super table may not exist @@ -2022,7 +2355,7 @@ class MainExec: # logger.info("Result: {}".format(rTables)) # except taos.error.ProgrammingError as err: # logger.info("Initial Super table OPS error: {}".format(err)) - + # # sys.exit() # if ( not rTables == None): # # print("rTables[0] = {}, type = {}".format(rTables[0], type(rTables[0]))) @@ -2031,24 +2364,26 @@ class MainExec: # ds = dbState # logger.info("Inserting into table: {}".format(rTbName[0])) # sql = "insert into db.{} values ('{}', {});".format( - # rTbName[0], + # rTbName[0], # ds.getNextTick(), ds.getNextInt()) # dbc.execute(sql) - # for rTbName in rTables : # regular tables + # for rTbName in rTables : # regular tables # dbc.query("select * from db.{}".format(rTbName[0])) # TODO: check success failure - # logger.info("Initial READING operation is successful") + # logger.info("Initial READING operation is successful") # except taos.error.ProgrammingError as err: - # logger.info("Initial WRITE/READ error: {}".format(err)) - + # logger.info("Initial WRITE/READ error: {}".format(err)) + # Sandbox testing code # dbc = dbState.getDbConn() # while True: - # rows = dbc.query("show databases") + # rows = dbc.query("show databases") # print("Rows: {}, time={}".format(rows, time.time())) - return + return + def main(): - # Super cool Python argument library: https://docs.python.org/3/library/argparse.html + # Super cool Python argument library: + # https://docs.python.org/3/library/argparse.html parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent('''\ @@ -2059,50 +2394,90 @@ def main(): ''')) - parser.add_argument('-a', '--auto-start-service', action='store_true', - help='Automatically start/stop the TDengine service (default: false)') - parser.add_argument('-c', '--connector-type', action='store', default='native', type=str, - help='Connector type to use: native, rest, or mixed (default: 10)') - parser.add_argument('-d', '--debug', action='store_true', - help='Turn on DEBUG mode for more logging (default: false)') - parser.add_argument('-e', '--run-tdengine', action='store_true', - help='Run TDengine service in foreground (default: false)') - parser.add_argument('-l', '--larger-data', action='store_true', - help='Write larger amount of data during write operations (default: false)') - parser.add_argument('-p', '--per-thread-db-connection', action='store_true', - help='Use a single shared db connection (default: false)') - parser.add_argument('-r', '--record-ops', action='store_true', - help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)') - parser.add_argument('-s', '--max-steps', action='store', default=1000, type=int, - help='Maximum number of steps to run (default: 100)') - parser.add_argument('-t', '--num-threads', action='store', default=5, type=int, - help='Number of threads to run (default: 10)') + parser.add_argument( + '-a', + '--auto-start-service', + action='store_true', + help='Automatically start/stop the TDengine service (default: false)') + parser.add_argument( + '-c', + '--connector-type', + action='store', + default='native', + type=str, + help='Connector type to use: native, rest, or mixed (default: 10)') + parser.add_argument( + '-d', + '--debug', + action='store_true', + help='Turn on DEBUG mode for more logging (default: false)') + parser.add_argument( + '-e', + '--run-tdengine', + action='store_true', + help='Run TDengine service in foreground (default: false)') + parser.add_argument( + '-l', + '--larger-data', + action='store_true', + help='Write larger amount of data during write operations (default: false)') + parser.add_argument( + '-p', + '--per-thread-db-connection', + action='store_true', + help='Use a single shared db connection (default: false)') + parser.add_argument( + '-r', + '--record-ops', + action='store_true', + help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)') + parser.add_argument( + '-s', + '--max-steps', + action='store', + default=1000, + type=int, + help='Maximum number of steps to run (default: 100)') + parser.add_argument( + '-t', + '--num-threads', + action='store', + default=5, + type=int, + help='Number of threads to run (default: 10)') + parser.add_argument( + '-x', + '--continue-on-exception', + action='store_true', + help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)') global gConfig gConfig = parser.parse_args() - + # Logging Stuff global logger - _logger = logging.getLogger('CrashGen') # real logger - _logger.addFilter(LoggingFilter()) + _logger = logging.getLogger('CrashGen') # real logger + _logger.addFilter(LoggingFilter()) ch = logging.StreamHandler() _logger.addHandler(ch) - logger = MyLoggingAdapter(_logger, []) # Logging adapter, to be used as a logger + # Logging adapter, to be used as a logger + logger = MyLoggingAdapter(_logger, []) - if ( gConfig.debug ): - logger.setLevel(logging.DEBUG) # default seems to be INFO + if (gConfig.debug): + logger.setLevel(logging.DEBUG) # default seems to be INFO else: logger.setLevel(logging.INFO) - - Dice.seed(0) # initial seeding of dice - + + Dice.seed(0) # initial seeding of dice + # Run server or client - if gConfig.run_tdengine : # run server + if gConfig.run_tdengine: # run server MainExec.runService() - else : + else: return MainExec.runClient() + if __name__ == "__main__": exitCode = main() # print("Exiting with code: {}".format(exitCode)) diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index de80361aa3..f6be6aae49 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -31,11 +31,22 @@ then exit -1 fi +CURR_DIR=`pwd` +IN_TDINTERNAL="community" +if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + TAOS_DIR=$CURR_DIR/../../.. +else + TAOS_DIR=$CURR_DIR/../.. +fi +TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` + +LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib + # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 # Then let us set up the library path so that our compiled SO file can be loaded by Python -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/../../build/build/lib +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR # Now we are all let, and let's see if we can find a crash. Note we pass all params python3 ./crash_gen.py $@ diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 83f94f727a..7a0f2cb825 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -121,7 +121,7 @@ python3 ./test.py -f import_merge/importTORestart.py python3 ./test.py -f import_merge/importTPORestart.py python3 ./test.py -f import_merge/importTRestart.py python3 ./test.py -f import_merge/importInsertThenImport.py - +python3 ./test.py -f import_merge/importCSV.py # user python3 ./test.py -f user/user_create.py python3 ./test.py -f user/pass_len.py diff --git a/tests/pytest/import_merge/importCSV.py b/tests/pytest/import_merge/importCSV.py new file mode 100644 index 0000000000..b4441949a1 --- /dev/null +++ b/tests/pytest/import_merge/importCSV.py @@ -0,0 +1,94 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import csv +import random +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.csvfile = "/tmp/file.csv" + self.rows = 10000 + self.ntables = 1 + self.startTime = 1520000010000 + def genRandomStr(self, maxLen): + H = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' + salt = '' + if maxLen <= 1: + maxLen = 2 + l = random.randint(1,maxLen) + for i in range(l): + salt += random.choice(H) + return salt + def createCSVFile(self): + f = open(self.csvfile,'w',encoding='utf-8') + csv_writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC) + for i in range(self.rows): + csv_writer.writerow([self.startTime + i, + self.genRandomStr(5), + self.genRandomStr(6), + self.genRandomStr(7), + self.genRandomStr(8), + self.genRandomStr(9), + self.genRandomStr(10), + self.genRandomStr(11), + self.genRandomStr(12), + self.genRandomStr(13), + self.genRandomStr(14)]) + f.close() + def destroyCSVFile(self): + os.remove(self.csvfile) + def run(self): + self.createCSVFile() + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + tdSql.execute('''create table tbx (ts TIMESTAMP, + collect_area NCHAR(5), + device_id BINARY(6), + imsi BINARY(7), + imei BINARY(8), + mdn BINARY(9), + net_type BINARY(10), + mno NCHAR(11), + province NCHAR(12), + city NCHAR(13), + alarm BINARY(14))''') + + tdSql.execute("import into tbx file \'%s\'"%(self.csvfile)) + tdSql.query('select * from tbx') + tdSql.checkRows(self.rows) + + def stop(self): + self.destroyCSVFile() + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + tdDnodes.stop(1) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh index d3a8deaf47..46098f4040 100755 --- a/tests/pytest/regressiontest.sh +++ b/tests/pytest/regressiontest.sh @@ -121,7 +121,7 @@ python3 ./test.py -f import_merge/importTORestart.py python3 ./test.py -f import_merge/importTPORestart.py python3 ./test.py -f import_merge/importTRestart.py python3 ./test.py -f import_merge/importInsertThenImport.py - +python3 ./test.py -f import_merge/importCSV.py # user python3 ./test.py -f user/user_create.py python3 ./test.py -f user/pass_len.py diff --git a/tests/pytest/testCompress.py b/tests/pytest/testCompress.py new file mode 100644 index 0000000000..0f5d9ef3b1 --- /dev/null +++ b/tests/pytest/testCompress.py @@ -0,0 +1,136 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/linux/python2/ + +# -*- coding: utf-8 -*- +import sys +import getopt +import subprocess +from distutils.log import warn as printf + +from util.log import * +from util.dnodes import * +from util.cases import * +from util.sql import * + +import taos + + +if __name__ == "__main__": + fileName = "all" + deployPath = "" + testCluster = False + valgrind = 0 + logSql = True + stop = 0 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'l:sgh', [ + 'logSql', 'stop', 'valgrind', 'help']) + for key, value in opts: + if key in ['-h', '--help']: + tdLog.printNoPrefix( + 'A collection of test cases written using Python') + tdLog.printNoPrefix('-l logSql Flag') + tdLog.printNoPrefix('-s stop All dnodes') + tdLog.printNoPrefix('-g valgrind Test Flag') + sys.exit(0) + + if key in ['-l', '--logSql']: + if (value.upper() == "TRUE"): + logSql = True + elif (value.upper() == "FALSE"): + logSql = False + else: + tdLog.printNoPrefix("logSql value %s is invalid" % logSql) + sys.exit(0) + + if key in ['-g', '--valgrind']: + valgrind = 1 + + if key in ['-s', '--stop']: + stop = 1 + + if (stop != 0): + if (valgrind == 0): + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP > /dev/null 2>&1" % toBeKilled + + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output(psCmd, shell=True) + + while(processID): + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output(psCmd, shell=True) + + for port in range(6030, 6041): + usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port + processID = subprocess.check_output(usePortPID, shell=True) + + if processID: + killCmd = "kill -9 %s" % processID + os.system(killCmd) + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if valgrind: + time.sleep(2) + + tdLog.info('stop All dnodes') + sys.exit(0) + + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + + tdDnodes.stopAll() + tdDnodes.addSimExtraCfg("compressMsgSize", "10240") + tdDnodes.deploy(1) + tdDnodes.start(1) + + host = '127.0.0.1' + + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + + tdCases.logSql(logSql) + + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + + tdSql.init(conn.cursor(), True) + + tdSql.execute("CREATE DATABASE IF NOT EXISTS t10b") + tdSql.execute("USE t10b") + tdSql.execute( + "CREATE TABLE IF NOT EXISTS s_sensor_info (ts TIMESTAMP, temperature INT, humidity FLOAT)") + + start_time = int(round(time.time() * 1000)) + for i in range(1, 1000): + tdSql.execute("IMPORT INTO s_sensor_info VALUES (1575129600000, 16, 19.405090) (1575129601000, 22, 14.377142) (1575129602000, 16, 16.868231) (1575129603000, 20, 11.565193) (1575129604000, 31, 13.009119) (1575129605000, 29, 18.136400) (1575129606000, 17, 13.806572) (1575129607000, 23, 14.688898) (1575129608000, 26, 12.931019) (1575129609000, 32, 12.185531) (1575129610000, 30, 13.608714) (1575129611000, 23, 18.624914) (1575129612000, 22, 12.970826) (1575129613000, 22, 12.065827) (1575129614000, 25, 16.967192) (1575129615000, 16, 10.283031) (1575129616000, 22, 16.072535) (1575129617000, 24, 10.794536) (1575129618000, 32, 10.591207) (1575129619000, 20, 13.015227) (1575129620000, 28, 15.410999) (1575129621000, 29, 12.785076) (1575129622000, 28, 15.305857) (1575129623000, 33, 12.820810) (1575129624000, 34, 13.618055) (1575129625000, 32, 12.971123) (1575129626000, 24, 10.974546) (1575129627000, 15, 10.742910) (1575129628000, 23, 16.810783) (1575129629000, 18, 13.115224) (1575129630000, 26, 17.418489) (1575129631000, 20, 17.302315) (1575129632000, 21, 14.283571) (1575129633000, 16, 16.826534) (1575129634000, 18, 19.222122) (1575129635000, 18, 14.931420) (1575129636000, 17, 19.549454) (1575129637000, 22, 16.908388) (1575129638000, 32, 15.637796) (1575129639000, 31, 15.517650) (1575129640000, 18, 14.038033) (1575129641000, 32, 19.859648) (1575129642000, 16, 13.220840) (1575129643000, 28, 16.445398) (1575129644000, 26, 16.695753) (1575129645000, 33, 13.696928) (1575129646000, 21, 15.352819) (1575129647000, 15, 12.388407) (1575129648000, 27, 11.267529) (1575129649000, 20, 14.103228) (1575129650000, 20, 16.250950) (1575129651000, 30, 16.236088) (1575129652000, 22, 18.305340) (1575129653000, 25, 17.360685) (1575129654000, 25, 14.978681) (1575129655000, 33, 14.096183) (1575129656000, 26, 10.019039) (1575129657000, 19, 19.158213) (1575129658000, 22, 15.593924) (1575129659000, 26, 18.780119) (1575129660000, 21, 16.001656) (1575129661000, 16, 18.458328) (1575129662000, 21, 16.417843) (1575129663000, 28, 11.736558) (1575129664000, 34, 18.143946) (1575129665000, 27, 10.303225) (1575129666000, 20, 19.756748) (1575129667000, 22, 12.940063) (1575129668000, 23, 11.509640) (1575129669000, 19, 18.319309) (1575129670000, 19, 16.278345) (1575129671000, 27, 10.898361) (1575129672000, 31, 13.922162) (1575129673000, 15, 19.296116) (1575129674000, 26, 15.885763) (1575129675000, 15, 15.525804) (1575129676000, 19, 19.579538) (1575129677000, 20, 11.073811) (1575129678000, 16, 13.932510) (1575129679000, 17, 11.900328) (1575129680000, 22, 16.540415) (1575129681000, 33, 15.203803) (1575129682000, 17, 11.518434) (1575129683000, 17, 13.152081) (1575129684000, 18, 11.378041) (1575129685000, 21, 15.390745) (1575129686000, 30, 15.127818) (1575129687000, 19, 16.530401) (1575129688000, 32, 16.542702) (1575129689000, 26, 16.366442) (1575129690000, 25, 10.306822) (1575129691000, 15, 13.691117) (1575129692000, 15, 13.476817) (1575129693000, 25, 12.529998) (1575129694000, 22, 15.550021) (1575129695000, 20, 15.064971) (1575129696000, 24, 13.313683) (1575129697000, 23, 17.002878) (1575129698000, 30, 19.991594) (1575129699000, 15, 11.116746) (1575129699990, 16, 19.405090) (1575129700990, 22, 14.377142) (1575129701990, 16, 16.868231) (1575129702990, 20, 11.565193) (1575129703990, 31, 13.009119) (1575129704990, 29, 18.136400) (1575129705990, 17, 13.806572) (1575129706990, 23, 14.688898) (1575129707990, 26, 12.931019) (1575129708990, 32, 12.185531) (1575129709990, 30, 13.608714) (1575129710990, 23, 18.624914) (1575129711990, 22, 12.970826) (1575129712990, 22, 12.065827) (1575129713990, 25, 16.967192) (1575129714990, 16, 10.283031) (1575129715990, 22, 16.072535) (1575129716990, 24, 10.794536) (1575129717990, 32, 10.591207) (1575129718990, 20, 13.015227) (1575129719990, 28, 15.410999) (1575129720990, 29, 12.785076) (1575129721990, 28, 15.305857) (1575129722990, 33, 12.820810) (1575129723990, 34, 13.618055) (1575129724990, 32, 12.971123) (1575129725990, 24, 10.974546) (1575129726990, 15, 10.742910) (1575129727990, 23, 16.810783) (1575129728990, 18, 13.115224) (1575129729990, 26, 17.418489) (1575129730990, 20, 17.302315) (1575129731990, 21, 14.283571) (1575129732990, 16, 16.826534) (1575129733990, 18, 19.222122) (1575129734990, 18, 14.931420) (1575129735990, 17, 19.549454) (1575129736990, 22, 16.908388) (1575129737990, 32, 15.637796) (1575129738990, 31, 15.517650) (1575129739990, 18, 14.038033) (1575129740990, 32, 19.859648) (1575129741990, 16, 13.220840) (1575129742990, 28, 16.445398) (1575129743990, 26, 16.695753) (1575129744990, 33, 13.696928) (1575129745990, 21, 15.352819) (1575129746990, 15, 12.388407) (1575129747990, 27, 11.267529) (1575129748990, 20, 14.103228) (1575129749990, 20, 16.250950) (1575129750990, 30, 16.236088) (1575129751990, 22, 18.305340) (1575129752990, 25, 17.360685) (1575129753990, 25, 14.978681) (1575129754990, 33, 14.096183) (1575129755990, 26, 10.019039) (1575129756990, 19, 19.158213) (1575129757990, 22, 15.593924) (1575129758990, 26, 18.780119) (1575129759990, 21, 16.001656) (1575129760990, 16, 18.458328) (1575129761990, 21, 16.417843) (1575129762990, 28, 11.736558) (1575129763990, 34, 18.143946) (1575129764990, 27, 10.303225) (1575129765990, 20, 19.756748) (1575129766990, 22, 12.940063) (1575129767990, 23, 11.509640) (1575129768990, 19, 18.319309) (1575129769990, 19, 16.278345) (1575129770990, 27, 10.898361) (1575129771990, 31, 13.922162) (1575129772990, 15, 19.296116) (1575129773990, 26, 15.885763) (1575129774990, 15, 15.525804) (1575129775990, 19, 19.579538) (1575129776990, 20, 11.073811) (1575129777990, 16, 13.932510) (1575129778990, 17, 11.900328) (1575129779990, 22, 16.540415) (1575129780990, 33, 15.203803) (1575129781990, 17, 11.518434) (1575129782990, 17, 13.152081) (1575129783990, 18, 11.378041) (1575129784990, 21, 15.390745) (1575129785990, 30, 15.127818) (1575129786990, 19, 16.530401) (1575129787990, 32, 16.542702) (1575129788990, 26, 16.366442) (1575129789990, 25, 10.306822) (1575129790990, 15, 13.691117) (1575129791990, 15, 13.476817) (1575129792990, 25, 12.529998) (1575129793990, 22, 15.550021) (1575129794990, 20, 15.064971) (1575129795990, 24, 13.313683) (1575129796990, 23, 17.002878) (1575129797990, 30, 19.991594) (1575129798990, 15, 11.116746) (1575129799980, 16, 19.405090) (1575129800980, 22, 14.377142) (1575129801980, 16, 16.868231) (1575129802980, 20, 11.565193) (1575129803980, 31, 13.009119) (1575129804980, 29, 18.136400) (1575129805980, 17, 13.806572) (1575129806980, 23, 14.688898) (1575129807980, 26, 12.931019) (1575129808980, 32, 12.185531) (1575129809980, 30, 13.608714) (1575129810980, 23, 18.624914) (1575129811980, 22, 12.970826) (1575129812980, 22, 12.065827) (1575129813980, 25, 16.967192) (1575129814980, 16, 10.283031) (1575129815980, 22, 16.072535) (1575129816980, 24, 10.794536) (1575129817980, 32, 10.591207) (1575129818980, 20, 13.015227) (1575129819980, 28, 15.410999) (1575129820980, 29, 12.785076) (1575129821980, 28, 15.305857) (1575129822980, 33, 12.820810) (1575129823980, 34, 13.618055) (1575129824980, 32, 12.971123) (1575129825980, 24, 10.974546) (1575129826980, 15, 10.742910) (1575129827980, 23, 16.810783) (1575129828980, 18, 13.115224) (1575129829980, 26, 17.418489) (1575129830980, 20, 17.302315) (1575129831980, 21, 14.283571) (1575129832980, 16, 16.826534) (1575129833980, 18, 19.222122) (1575129834980, 18, 14.931420) (1575129835980, 17, 19.549454) (1575129836980, 22, 16.908388) (1575129837980, 32, 15.637796) (1575129838980, 31, 15.517650) (1575129839980, 18, 14.038033) (1575129840980, 32, 19.859648) (1575129841980, 16, 13.220840) (1575129842980, 28, 16.445398) (1575129843980, 26, 16.695753) (1575129844980, 33, 13.696928) (1575129845980, 21, 15.352819) (1575129846980, 15, 12.388407) (1575129847980, 27, 11.267529) (1575129848980, 20, 14.103228) (1575129849980, 20, 16.250950) (1575129850980, 30, 16.236088) (1575129851980, 22, 18.305340) (1575129852980, 25, 17.360685) (1575129853980, 25, 14.978681) (1575129854980, 33, 14.096183) (1575129855980, 26, 10.019039) (1575129856980, 19, 19.158213) (1575129857980, 22, 15.593924) (1575129858980, 26, 18.780119) (1575129859980, 21, 16.001656) (1575129860980, 16, 18.458328) (1575129861980, 21, 16.417843) (1575129862980, 28, 11.736558) (1575129863980, 34, 18.143946) (1575129864980, 27, 10.303225) (1575129865980, 20, 19.756748) (1575129866980, 22, 12.940063) (1575129867980, 23, 11.509640) (1575129868980, 19, 18.319309) (1575129869980, 19, 16.278345) (1575129870980, 27, 10.898361) (1575129871980, 31, 13.922162) (1575129872980, 15, 19.296116) (1575129873980, 26, 15.885763) (1575129874980, 15, 15.525804) (1575129875980, 19, 19.579538) (1575129876980, 20, 11.073811) (1575129877980, 16, 13.932510) (1575129878980, 17, 11.900328) (1575129879980, 22, 16.540415) (1575129880980, 33, 15.203803) (1575129881980, 17, 11.518434) (1575129882980, 17, 13.152081) (1575129883980, 18, 11.378041) (1575129884980, 21, 15.390745) (1575129885980, 30, 15.127818) (1575129886980, 19, 16.530401) (1575129887980, 32, 16.542702) (1575129888980, 26, 16.366442) (1575129889980, 25, 10.306822) (1575129890980, 15, 13.691117) (1575129891980, 15, 13.476817) (1575129892980, 25, 12.529998) (1575129893980, 22, 15.550021) (1575129894980, 20, 15.064971) (1575129895980, 24, 13.313683) (1575129896980, 23, 17.002878) (1575129897980, 30, 19.991594) (1575129898980, 15, 11.116746) (1575129899970, 16, 19.405090) (1575129900970, 22, 14.377142) (1575129901970, 16, 16.868231) (1575129902970, 20, 11.565193) (1575129903970, 31, 13.009119) (1575129904970, 29, 18.136400) (1575129905970, 17, 13.806572) (1575129906970, 23, 14.688898) (1575129907970, 26, 12.931019) (1575129908970, 32, 12.185531) (1575129909970, 30, 13.608714) (1575129910970, 23, 18.624914) (1575129911970, 22, 12.970826) (1575129912970, 22, 12.065827) (1575129913970, 25, 16.967192) (1575129914970, 16, 10.283031) (1575129915970, 22, 16.072535) (1575129916970, 24, 10.794536) (1575129917970, 32, 10.591207) (1575129918970, 20, 13.015227) (1575129919970, 28, 15.410999) (1575129920970, 29, 12.785076) (1575129921970, 28, 15.305857) (1575129922970, 33, 12.820810) (1575129923970, 34, 13.618055) (1575129924970, 32, 12.971123) (1575129925970, 24, 10.974546) (1575129926970, 15, 10.742910) (1575129927970, 23, 16.810783) (1575129928970, 18, 13.115224) (1575129929970, 26, 17.418489) (1575129930970, 20, 17.302315) (1575129931970, 21, 14.283571) (1575129932970, 16, 16.826534) (1575129933970, 18, 19.222122) (1575129934970, 18, 14.931420) (1575129935970, 17, 19.549454) (1575129936970, 22, 16.908388) (1575129937970, 32, 15.637796) (1575129938970, 31, 15.517650) (1575129939970, 18, 14.038033) (1575129940970, 32, 19.859648) (1575129941970, 16, 13.220840) (1575129942970, 28, 16.445398) (1575129943970, 26, 16.695753) (1575129944970, 33, 13.696928) (1575129945970, 21, 15.352819) (1575129946970, 15, 12.388407) (1575129947970, 27, 11.267529) (1575129948970, 20, 14.103228) (1575129949970, 20, 16.250950) (1575129950970, 30, 16.236088) (1575129951970, 22, 18.305340) (1575129952970, 25, 17.360685) (1575129953970, 25, 14.978681) (1575129954970, 33, 14.096183) (1575129955970, 26, 10.019039) (1575129956970, 19, 19.158213) (1575129957970, 22, 15.593924) (1575129958970, 26, 18.780119) (1575129959970, 21, 16.001656) (1575129960970, 16, 18.458328) (1575129961970, 21, 16.417843) (1575129962970, 28, 11.736558) (1575129963970, 34, 18.143946) (1575129964970, 27, 10.303225) (1575129965970, 20, 19.756748) (1575129966970, 22, 12.940063) (1575129967970, 23, 11.509640) (1575129968970, 19, 18.319309) (1575129969970, 19, 16.278345) (1575129970970, 27, 10.898361) (1575129971970, 31, 13.922162) (1575129972970, 15, 19.296116) (1575129973970, 26, 15.885763) (1575129974970, 15, 15.525804) (1575129975970, 19, 19.579538) (1575129976970, 20, 11.073811) (1575129977970, 16, 13.932510) (1575129978970, 17, 11.900328) (1575129979970, 22, 16.540415) (1575129980970, 33, 15.203803) (1575129981970, 17, 11.518434) (1575129982970, 17, 13.152081) (1575129983970, 18, 11.378041) (1575129984970, 21, 15.390745) (1575129985970, 30, 15.127818) (1575129986970, 19, 16.530401) (1575129987970, 32, 16.542702) (1575129988970, 26, 16.366442) (1575129989970, 25, 10.306822) (1575129990970, 15, 13.691117) (1575129991970, 15, 13.476817) (1575129992970, 25, 12.529998) (1575129993970, 22, 15.550021) (1575129994970, 20, 15.064971) (1575129995970, 24, 13.313683) (1575129996970, 23, 17.002878) (1575129997970, 30, 19.991594) (1575129998970, 15, 11.116746) (1575129999960, 16, 19.405090) (1575130000960, 22, 14.377142) (1575130001960, 16, 16.868231) (1575130002960, 20, 11.565193) (1575130003960, 31, 13.009119) (1575130004960, 29, 18.136400) (1575130005960, 17, 13.806572) (1575130006960, 23, 14.688898) (1575130007960, 26, 12.931019) (1575130008960, 32, 12.185531) (1575130009960, 30, 13.608714) (1575130010960, 23, 18.624914) (1575130011960, 22, 12.970826) (1575130012960, 22, 12.065827) (1575130013960, 25, 16.967192) (1575130014960, 16, 10.283031) (1575130015960, 22, 16.072535) (1575130016960, 24, 10.794536) (1575130017960, 32, 10.591207) (1575130018960, 20, 13.015227) (1575130019960, 28, 15.410999) (1575130020960, 29, 12.785076) (1575130021960, 28, 15.305857) (1575130022960, 33, 12.820810) (1575130023960, 34, 13.618055) (1575130024960, 32, 12.971123) (1575130025960, 24, 10.974546) (1575130026960, 15, 10.742910) (1575130027960, 23, 16.810783) (1575130028960, 18, 13.115224) (1575130029960, 26, 17.418489) (1575130030960, 20, 17.302315) (1575130031960, 21, 14.283571) (1575130032960, 16, 16.826534) (1575130033960, 18, 19.222122) (1575130034960, 18, 14.931420) (1575130035960, 17, 19.549454) (1575130036960, 22, 16.908388) (1575130037960, 32, 15.637796) (1575130038960, 31, 15.517650) (1575130039960, 18, 14.038033) (1575130040960, 32, 19.859648) (1575130041960, 16, 13.220840) (1575130042960, 28, 16.445398) (1575130043960, 26, 16.695753) (1575130044960, 33, 13.696928) (1575130045960, 21, 15.352819) (1575130046960, 15, 12.388407) (1575130047960, 27, 11.267529) (1575130048960, 20, 14.103228) (1575130049960, 20, 16.250950) (1575130050960, 30, 16.236088) (1575130051960, 22, 18.305340) (1575130052960, 25, 17.360685) (1575130053960, 25, 14.978681) (1575130054960, 33, 14.096183) (1575130055960, 26, 10.019039) (1575130056960, 19, 19.158213) (1575130057960, 22, 15.593924) (1575130058960, 26, 18.780119) (1575130059960, 21, 16.001656) (1575130060960, 16, 18.458328) (1575130061960, 21, 16.417843) (1575130062960, 28, 11.736558) (1575130063960, 34, 18.143946) (1575130064960, 27, 10.303225) (1575130065960, 20, 19.756748) (1575130066960, 22, 12.940063) (1575130067960, 23, 11.509640) (1575130068960, 19, 18.319309) (1575130069960, 19, 16.278345) (1575130070960, 27, 10.898361) (1575130071960, 31, 13.922162) (1575130072960, 15, 19.296116) (1575130073960, 26, 15.885763) (1575130074960, 15, 15.525804) (1575130075960, 19, 19.579538) (1575130076960, 20, 11.073811) (1575130077960, 16, 13.932510) (1575130078960, 17, 11.900328) (1575130079960, 22, 16.540415) (1575130080960, 33, 15.203803) (1575130081960, 17, 11.518434) (1575130082960, 17, 13.152081) (1575130083960, 18, 11.378041) (1575130084960, 21, 15.390745) (1575130085960, 30, 15.127818) (1575130086960, 19, 16.530401) (1575130087960, 32, 16.542702) (1575130088960, 26, 16.366442) (1575130089960, 25, 10.306822) (1575130090960, 15, 13.691117) (1575130091960, 15, 13.476817) (1575130092960, 25, 12.529998) (1575130093960, 22, 15.550021) (1575130094960, 20, 15.064971) (1575130095960, 24, 13.313683) (1575130096960, 23, 17.002878) (1575130097960, 30, 19.991594) (1575130098960, 15, 11.116746) (1575130099950, 16, 19.405090) (1575130100950, 22, 14.377142) (1575130101950, 16, 16.868231) (1575130102950, 20, 11.565193) (1575130103950, 31, 13.009119) (1575130104950, 29, 18.136400) (1575130105950, 17, 13.806572) (1575130106950, 23, 14.688898) (1575130107950, 26, 12.931019) (1575130108950, 32, 12.185531) (1575130109950, 30, 13.608714) (1575130110950, 23, 18.624914) (1575130111950, 22, 12.970826) (1575130112950, 22, 12.065827) (1575130113950, 25, 16.967192) (1575130114950, 16, 10.283031) (1575130115950, 22, 16.072535) (1575130116950, 24, 10.794536) (1575130117950, 32, 10.591207) (1575130118950, 20, 13.015227) (1575130119950, 28, 15.410999) (1575130120950, 29, 12.785076) (1575130121950, 28, 15.305857) (1575130122950, 33, 12.820810) (1575130123950, 34, 13.618055) (1575130124950, 32, 12.971123) (1575130125950, 24, 10.974546) (1575130126950, 15, 10.742910) (1575130127950, 23, 16.810783) (1575130128950, 18, 13.115224) (1575130129950, 26, 17.418489) (1575130130950, 20, 17.302315) (1575130131950, 21, 14.283571) (1575130132950, 16, 16.826534) (1575130133950, 18, 19.222122) (1575130134950, 18, 14.931420) (1575130135950, 17, 19.549454) (1575130136950, 22, 16.908388) (1575130137950, 32, 15.637796) (1575130138950, 31, 15.517650) (1575130139950, 18, 14.038033) (1575130140950, 32, 19.859648) (1575130141950, 16, 13.220840) (1575130142950, 28, 16.445398) (1575130143950, 26, 16.695753) (1575130144950, 33, 13.696928) (1575130145950, 21, 15.352819) (1575130146950, 15, 12.388407) (1575130147950, 27, 11.267529) (1575130148950, 20, 14.103228) (1575130149950, 20, 16.250950) (1575130150950, 30, 16.236088) (1575130151950, 22, 18.305340) (1575130152950, 25, 17.360685) (1575130153950, 25, 14.978681) (1575130154950, 33, 14.096183) (1575130155950, 26, 10.019039) (1575130156950, 19, 19.158213) (1575130157950, 22, 15.593924) (1575130158950, 26, 18.780119) (1575130159950, 21, 16.001656) (1575130160950, 16, 18.458328) (1575130161950, 21, 16.417843) (1575130162950, 28, 11.736558) (1575130163950, 34, 18.143946) (1575130164950, 27, 10.303225) (1575130165950, 20, 19.756748) (1575130166950, 22, 12.940063) (1575130167950, 23, 11.509640) (1575130168950, 19, 18.319309) (1575130169950, 19, 16.278345) (1575130170950, 27, 10.898361) (1575130171950, 31, 13.922162) (1575130172950, 15, 19.296116) (1575130173950, 26, 15.885763) (1575130174950, 15, 15.525804) (1575130175950, 19, 19.579538) (1575130176950, 20, 11.073811) (1575130177950, 16, 13.932510) (1575130178950, 17, 11.900328) (1575130179950, 22, 16.540415) (1575130180950, 33, 15.203803) (1575130181950, 17, 11.518434) (1575130182950, 17, 13.152081) (1575130183950, 18, 11.378041) (1575130184950, 21, 15.390745) (1575130185950, 30, 15.127818) (1575130186950, 19, 16.530401) (1575130187950, 32, 16.542702) (1575130188950, 26, 16.366442) (1575130189950, 25, 10.306822) (1575130190950, 15, 13.691117) (1575130191950, 15, 13.476817) (1575130192950, 25, 12.529998) (1575130193950, 22, 15.550021) (1575130194950, 20, 15.064971) (1575130195950, 24, 13.313683) (1575130196950, 23, 17.002878) (1575130197950, 30, 19.991594) (1575130198950, 15, 11.116746) (1575130199940, 16, 19.405090) (1575130200940, 22, 14.377142) (1575130201940, 16, 16.868231) (1575130202940, 20, 11.565193) (1575130203940, 31, 13.009119) (1575130204940, 29, 18.136400) (1575130205940, 17, 13.806572) (1575130206940, 23, 14.688898) (1575130207940, 26, 12.931019) (1575130208940, 32, 12.185531) (1575130209940, 30, 13.608714) (1575130210940, 23, 18.624914) (1575130211940, 22, 12.970826) (1575130212940, 22, 12.065827) (1575130213940, 25, 16.967192) (1575130214940, 16, 10.283031) (1575130215940, 22, 16.072535) (1575130216940, 24, 10.794536) (1575130217940, 32, 10.591207) (1575130218940, 20, 13.015227) (1575130219940, 28, 15.410999) (1575130220940, 29, 12.785076) (1575130221940, 28, 15.305857) (1575130222940, 33, 12.820810) (1575130223940, 34, 13.618055) (1575130224940, 32, 12.971123) (1575130225940, 24, 10.974546) (1575130226940, 15, 10.742910) (1575130227940, 23, 16.810783) (1575130228940, 18, 13.115224) (1575130229940, 26, 17.418489) (1575130230940, 20, 17.302315) (1575130231940, 21, 14.283571) (1575130232940, 16, 16.826534) (1575130233940, 18, 19.222122) (1575130234940, 18, 14.931420) (1575130235940, 17, 19.549454) (1575130236940, 22, 16.908388) (1575130237940, 32, 15.637796) (1575130238940, 31, 15.517650) (1575130239940, 18, 14.038033) (1575130240940, 32, 19.859648) (1575130241940, 16, 13.220840) (1575130242940, 28, 16.445398) (1575130243940, 26, 16.695753) (1575130244940, 33, 13.696928) (1575130245940, 21, 15.352819) (1575130246940, 15, 12.388407) (1575130247940, 27, 11.267529) (1575130248940, 20, 14.103228) (1575130249940, 20, 16.250950) (1575130250940, 30, 16.236088) (1575130251940, 22, 18.305340) (1575130252940, 25, 17.360685) (1575130253940, 25, 14.978681) (1575130254940, 33, 14.096183) (1575130255940, 26, 10.019039) (1575130256940, 19, 19.158213) (1575130257940, 22, 15.593924) (1575130258940, 26, 18.780119) (1575130259940, 21, 16.001656) (1575130260940, 16, 18.458328) (1575130261940, 21, 16.417843) (1575130262940, 28, 11.736558) (1575130263940, 34, 18.143946) (1575130264940, 27, 10.303225) (1575130265940, 20, 19.756748) (1575130266940, 22, 12.940063) (1575130267940, 23, 11.509640) (1575130268940, 19, 18.319309) (1575130269940, 19, 16.278345) (1575130270940, 27, 10.898361) (1575130271940, 31, 13.922162) (1575130272940, 15, 19.296116) (1575130273940, 26, 15.885763) (1575130274940, 15, 15.525804) (1575130275940, 19, 19.579538) (1575130276940, 20, 11.073811) (1575130277940, 16, 13.932510) (1575130278940, 17, 11.900328) (1575130279940, 22, 16.540415) (1575130280940, 33, 15.203803) (1575130281940, 17, 11.518434) (1575130282940, 17, 13.152081) (1575130283940, 18, 11.378041) (1575130284940, 21, 15.390745) (1575130285940, 30, 15.127818) (1575130286940, 19, 16.530401) (1575130287940, 32, 16.542702) (1575130288940, 26, 16.366442) (1575130289940, 25, 10.306822) (1575130290940, 15, 13.691117) (1575130291940, 15, 13.476817) (1575130292940, 25, 12.529998) (1575130293940, 22, 15.550021) (1575130294940, 20, 15.064971) (1575130295940, 24, 13.313683) (1575130296940, 23, 17.002878) (1575130297940, 30, 19.991594) (1575130298940, 15, 11.116746) (1575130299930, 16, 19.405090) (1575130300930, 22, 14.377142) (1575130301930, 16, 16.868231) (1575130302930, 20, 11.565193) (1575130303930, 31, 13.009119) (1575130304930, 29, 18.136400) (1575130305930, 17, 13.806572) (1575130306930, 23, 14.688898) (1575130307930, 26, 12.931019) (1575130308930, 32, 12.185531) (1575130309930, 30, 13.608714) (1575130310930, 23, 18.624914) (1575130311930, 22, 12.970826) (1575130312930, 22, 12.065827) (1575130313930, 25, 16.967192) (1575130314930, 16, 10.283031) (1575130315930, 22, 16.072535) (1575130316930, 24, 10.794536) (1575130317930, 32, 10.591207) (1575130318930, 20, 13.015227) (1575130319930, 28, 15.410999) (1575130320930, 29, 12.785076) (1575130321930, 28, 15.305857) (1575130322930, 33, 12.820810) (1575130323930, 34, 13.618055) (1575130324930, 32, 12.971123) (1575130325930, 24, 10.974546) (1575130326930, 15, 10.742910) (1575130327930, 23, 16.810783) (1575130328930, 18, 13.115224) (1575130329930, 26, 17.418489) (1575130330930, 20, 17.302315) (1575130331930, 21, 14.283571) (1575130332930, 16, 16.826534) (1575130333930, 18, 19.222122) (1575130334930, 18, 14.931420) (1575130335930, 17, 19.549454) (1575130336930, 22, 16.908388) (1575130337930, 32, 15.637796) (1575130338930, 31, 15.517650) (1575130339930, 18, 14.038033) (1575130340930, 32, 19.859648) (1575130341930, 16, 13.220840) (1575130342930, 28, 16.445398) (1575130343930, 26, 16.695753) (1575130344930, 33, 13.696928) (1575130345930, 21, 15.352819) (1575130346930, 15, 12.388407) (1575130347930, 27, 11.267529) (1575130348930, 20, 14.103228) (1575130349930, 20, 16.250950) (1575130350930, 30, 16.236088) (1575130351930, 22, 18.305340) (1575130352930, 25, 17.360685) (1575130353930, 25, 14.978681) (1575130354930, 33, 14.096183) (1575130355930, 26, 10.019039) (1575130356930, 19, 19.158213) (1575130357930, 22, 15.593924) (1575130358930, 26, 18.780119) (1575130359930, 21, 16.001656) (1575130360930, 16, 18.458328) (1575130361930, 21, 16.417843) (1575130362930, 28, 11.736558) (1575130363930, 34, 18.143946) (1575130364930, 27, 10.303225) (1575130365930, 20, 19.756748) (1575130366930, 22, 12.940063) (1575130367930, 23, 11.509640) (1575130368930, 19, 18.319309) (1575130369930, 19, 16.278345) (1575130370930, 27, 10.898361) (1575130371930, 31, 13.922162) (1575130372930, 15, 19.296116) (1575130373930, 26, 15.885763) (1575130374930, 15, 15.525804) (1575130375930, 19, 19.579538) (1575130376930, 20, 11.073811) (1575130377930, 16, 13.932510) (1575130378930, 17, 11.900328) (1575130379930, 22, 16.540415) (1575130380930, 33, 15.203803) (1575130381930, 17, 11.518434) (1575130382930, 17, 13.152081) (1575130383930, 18, 11.378041) (1575130384930, 21, 15.390745) (1575130385930, 30, 15.127818) (1575130386930, 19, 16.530401) (1575130387930, 32, 16.542702) (1575130388930, 26, 16.366442) (1575130389930, 25, 10.306822) (1575130390930, 15, 13.691117) (1575130391930, 15, 13.476817) (1575130392930, 25, 12.529998) (1575130393930, 22, 15.550021) (1575130394930, 20, 15.064971) (1575130395930, 24, 13.313683) (1575130396930, 23, 17.002878) (1575130397930, 30, 19.991594) (1575130398930, 15, 11.116746) (1575130399920, 16, 19.405090) (1575130400920, 22, 14.377142) (1575130401920, 16, 16.868231) (1575130402920, 20, 11.565193) (1575130403920, 31, 13.009119) (1575130404920, 29, 18.136400) (1575130405920, 17, 13.806572) (1575130406920, 23, 14.688898) (1575130407920, 26, 12.931019) (1575130408920, 32, 12.185531) (1575130409920, 30, 13.608714) (1575130410920, 23, 18.624914) (1575130411920, 22, 12.970826) (1575130412920, 22, 12.065827) (1575130413920, 25, 16.967192) (1575130414920, 16, 10.283031) (1575130415920, 22, 16.072535) (1575130416920, 24, 10.794536) (1575130417920, 32, 10.591207) (1575130418920, 20, 13.015227) (1575130419920, 28, 15.410999) (1575130420920, 29, 12.785076) (1575130421920, 28, 15.305857) (1575130422920, 33, 12.820810) (1575130423920, 34, 13.618055) (1575130424920, 32, 12.971123) (1575130425920, 24, 10.974546) (1575130426920, 15, 10.742910) (1575130427920, 23, 16.810783) (1575130428920, 18, 13.115224) (1575130429920, 26, 17.418489) (1575130430920, 20, 17.302315) (1575130431920, 21, 14.283571) (1575130432920, 16, 16.826534) (1575130433920, 18, 19.222122) (1575130434920, 18, 14.931420) (1575130435920, 17, 19.549454) (1575130436920, 22, 16.908388) (1575130437920, 32, 15.637796) (1575130438920, 31, 15.517650) (1575130439920, 18, 14.038033) (1575130440920, 32, 19.859648) (1575130441920, 16, 13.220840) (1575130442920, 28, 16.445398) (1575130443920, 26, 16.695753) (1575130444920, 33, 13.696928) (1575130445920, 21, 15.352819) (1575130446920, 15, 12.388407) (1575130447920, 27, 11.267529) (1575130448920, 20, 14.103228) (1575130449920, 20, 16.250950) (1575130450920, 30, 16.236088) (1575130451920, 22, 18.305340) (1575130452920, 25, 17.360685) (1575130453920, 25, 14.978681) (1575130454920, 33, 14.096183) (1575130455920, 26, 10.019039) (1575130456920, 19, 19.158213) (1575130457920, 22, 15.593924) (1575130458920, 26, 18.780119) (1575130459920, 21, 16.001656) (1575130460920, 16, 18.458328) (1575130461920, 21, 16.417843) (1575130462920, 28, 11.736558) (1575130463920, 34, 18.143946) (1575130464920, 27, 10.303225) (1575130465920, 20, 19.756748) (1575130466920, 22, 12.940063) (1575130467920, 23, 11.509640) (1575130468920, 19, 18.319309) (1575130469920, 19, 16.278345) (1575130470920, 27, 10.898361) (1575130471920, 31, 13.922162) (1575130472920, 15, 19.296116) (1575130473920, 26, 15.885763) (1575130474920, 15, 15.525804) (1575130475920, 19, 19.579538) (1575130476920, 20, 11.073811) (1575130477920, 16, 13.932510) (1575130478920, 17, 11.900328) (1575130479920, 22, 16.540415) (1575130480920, 33, 15.203803) (1575130481920, 17, 11.518434) (1575130482920, 17, 13.152081) (1575130483920, 18, 11.378041) (1575130484920, 21, 15.390745) (1575130485920, 30, 15.127818) (1575130486920, 19, 16.530401) (1575130487920, 32, 16.542702) (1575130488920, 26, 16.366442) (1575130489920, 25, 10.306822) (1575130490920, 15, 13.691117) (1575130491920, 15, 13.476817) (1575130492920, 25, 12.529998) (1575130493920, 22, 15.550021) (1575130494920, 20, 15.064971) (1575130495920, 24, 13.313683) (1575130496920, 23, 17.002878) (1575130497920, 30, 19.991594) (1575130498920, 15, 11.116746) (1575130499910, 16, 19.405090) (1575130500910, 22, 14.377142) (1575130501910, 16, 16.868231) (1575130502910, 20, 11.565193) (1575130503910, 31, 13.009119) (1575130504910, 29, 18.136400) (1575130505910, 17, 13.806572) (1575130506910, 23, 14.688898) (1575130507910, 26, 12.931019) (1575130508910, 32, 12.185531) (1575130509910, 30, 13.608714) (1575130510910, 23, 18.624914) (1575130511910, 22, 12.970826) (1575130512910, 22, 12.065827) (1575130513910, 25, 16.967192) (1575130514910, 16, 10.283031) (1575130515910, 22, 16.072535) (1575130516910, 24, 10.794536) (1575130517910, 32, 10.591207) (1575130518910, 20, 13.015227) (1575130519910, 28, 15.410999) (1575130520910, 29, 12.785076) (1575130521910, 28, 15.305857) (1575130522910, 33, 12.820810) (1575130523910, 34, 13.618055) (1575130524910, 32, 12.971123) (1575130525910, 24, 10.974546) (1575130526910, 15, 10.742910) (1575130527910, 23, 16.810783) (1575130528910, 18, 13.115224) (1575130529910, 26, 17.418489) (1575130530910, 20, 17.302315) (1575130531910, 21, 14.283571) (1575130532910, 16, 16.826534) (1575130533910, 18, 19.222122) (1575130534910, 18, 14.931420) (1575130535910, 17, 19.549454) (1575130536910, 22, 16.908388) (1575130537910, 32, 15.637796) (1575130538910, 31, 15.517650) (1575130539910, 18, 14.038033) (1575130540910, 32, 19.859648) (1575130541910, 16, 13.220840) (1575130542910, 28, 16.445398) (1575130543910, 26, 16.695753) (1575130544910, 33, 13.696928) (1575130545910, 21, 15.352819) (1575130546910, 15, 12.388407) (1575130547910, 27, 11.267529) (1575130548910, 20, 14.103228) (1575130549910, 20, 16.250950) (1575130550910, 30, 16.236088) (1575130551910, 22, 18.305340) (1575130552910, 25, 17.360685) (1575130553910, 25, 14.978681) (1575130554910, 33, 14.096183) (1575130555910, 26, 10.019039) (1575130556910, 19, 19.158213) (1575130557910, 22, 15.593924) (1575130558910, 26, 18.780119) (1575130559910, 21, 16.001656) (1575130560910, 16, 18.458328) (1575130561910, 21, 16.417843) (1575130562910, 28, 11.736558) (1575130563910, 34, 18.143946) (1575130564910, 27, 10.303225) (1575130565910, 20, 19.756748) (1575130566910, 22, 12.940063) (1575130567910, 23, 11.509640) (1575130568910, 19, 18.319309) (1575130569910, 19, 16.278345) (1575130570910, 27, 10.898361) (1575130571910, 31, 13.922162) (1575130572910, 15, 19.296116) (1575130573910, 26, 15.885763) (1575130574910, 15, 15.525804) (1575130575910, 19, 19.579538) (1575130576910, 20, 11.073811) (1575130577910, 16, 13.932510) (1575130578910, 17, 11.900328) (1575130579910, 22, 16.540415) (1575130580910, 33, 15.203803) (1575130581910, 17, 11.518434) (1575130582910, 17, 13.152081) (1575130583910, 18, 11.378041) (1575130584910, 21, 15.390745) (1575130585910, 30, 15.127818) (1575130586910, 19, 16.530401) (1575130587910, 32, 16.542702) (1575130588910, 26, 16.366442) (1575130589910, 25, 10.306822) (1575130590910, 15, 13.691117) (1575130591910, 15, 13.476817) (1575130592910, 25, 12.529998) (1575130593910, 22, 15.550021) (1575130594910, 20, 15.064971) (1575130595910, 24, 13.313683) (1575130596910, 23, 17.002878) (1575130597910, 30, 19.991594) (1575130598910, 15, 11.116746) (1575130599900, 16, 19.405090) (1575130600900, 22, 14.377142) (1575130601900, 16, 16.868231) (1575130602900, 20, 11.565193) (1575130603900, 31, 13.009119) (1575130604900, 29, 18.136400) (1575130605900, 17, 13.806572) (1575130606900, 23, 14.688898) (1575130607900, 26, 12.931019) (1575130608900, 32, 12.185531) (1575130609900, 30, 13.608714) (1575130610900, 23, 18.624914) (1575130611900, 22, 12.970826) (1575130612900, 22, 12.065827) (1575130613900, 25, 16.967192) (1575130614900, 16, 10.283031) (1575130615900, 22, 16.072535) (1575130616900, 24, 10.794536) (1575130617900, 32, 10.591207) (1575130618900, 20, 13.015227) (1575130619900, 28, 15.410999) (1575130620900, 29, 12.785076) (1575130621900, 28, 15.305857) (1575130622900, 33, 12.820810) (1575130623900, 34, 13.618055) (1575130624900, 32, 12.971123) (1575130625900, 24, 10.974546) (1575130626900, 15, 10.742910) (1575130627900, 23, 16.810783) (1575130628900, 18, 13.115224) (1575130629900, 26, 17.418489) (1575130630900, 20, 17.302315) (1575130631900, 21, 14.283571) (1575130632900, 16, 16.826534) (1575130633900, 18, 19.222122) (1575130634900, 18, 14.931420) (1575130635900, 17, 19.549454) (1575130636900, 22, 16.908388) (1575130637900, 32, 15.637796) (1575130638900, 31, 15.517650) (1575130639900, 18, 14.038033) (1575130640900, 32, 19.859648) (1575130641900, 16, 13.220840) (1575130642900, 28, 16.445398) (1575130643900, 26, 16.695753) (1575130644900, 33, 13.696928) (1575130645900, 21, 15.352819) (1575130646900, 15, 12.388407) (1575130647900, 27, 11.267529) (1575130648900, 20, 14.103228) (1575130649900, 20, 16.250950) (1575130650900, 30, 16.236088) (1575130651900, 22, 18.305340) (1575130652900, 25, 17.360685) (1575130653900, 25, 14.978681) (1575130654900, 33, 14.096183) (1575130655900, 26, 10.019039) (1575130656900, 19, 19.158213) (1575130657900, 22, 15.593924) (1575130658900, 26, 18.780119) (1575130659900, 21, 16.001656) (1575130660900, 16, 18.458328) (1575130661900, 21, 16.417843) (1575130662900, 28, 11.736558) (1575130663900, 34, 18.143946) (1575130664900, 27, 10.303225) (1575130665900, 20, 19.756748) (1575130666900, 22, 12.940063) (1575130667900, 23, 11.509640) (1575130668900, 19, 18.319309) (1575130669900, 19, 16.278345) (1575130670900, 27, 10.898361) (1575130671900, 31, 13.922162) (1575130672900, 15, 19.296116) (1575130673900, 26, 15.885763) (1575130674900, 15, 15.525804) (1575130675900, 19, 19.579538) (1575130676900, 20, 11.073811) (1575130677900, 16, 13.932510) (1575130678900, 17, 11.900328) (1575130679900, 22, 16.540415) (1575130680900, 33, 15.203803) (1575130681900, 17, 11.518434) (1575130682900, 17, 13.152081) (1575130683900, 18, 11.378041) (1575130684900, 21, 15.390745) (1575130685900, 30, 15.127818) (1575130686900, 19, 16.530401) (1575130687900, 32, 16.542702) (1575130688900, 26, 16.366442) (1575130689900, 25, 10.306822) (1575130690900, 15, 13.691117) (1575130691900, 15, 13.476817) (1575130692900, 25, 12.529998) (1575130693900, 22, 15.550021) (1575130694900, 20, 15.064971) (1575130695900, 24, 13.313683) (1575130696900, 23, 17.002878) (1575130697900, 30, 19.991594) (1575130698900, 15, 11.116746) (1575130699890, 16, 19.405090) (1575130700890, 22, 14.377142) (1575130701890, 16, 16.868231) (1575130702890, 20, 11.565193) (1575130703890, 31, 13.009119) (1575130704890, 29, 18.136400) (1575130705890, 17, 13.806572) (1575130706890, 23, 14.688898) (1575130707890, 26, 12.931019) (1575130708890, 32, 12.185531) (1575130709890, 30, 13.608714) (1575130710890, 23, 18.624914) (1575130711890, 22, 12.970826) (1575130712890, 22, 12.065827) (1575130713890, 25, 16.967192) (1575130714890, 16, 10.283031) (1575130715890, 22, 16.072535) (1575130716890, 24, 10.794536) (1575130717890, 32, 10.591207) (1575130718890, 20, 13.015227) (1575130719890, 28, 15.410999) (1575130720890, 29, 12.785076) (1575130721890, 28, 15.305857) (1575130722890, 33, 12.820810) (1575130723890, 34, 13.618055) (1575130724890, 32, 12.971123) (1575130725890, 24, 10.974546) (1575130726890, 15, 10.742910) (1575130727890, 23, 16.810783) (1575130728890, 18, 13.115224) (1575130729890, 26, 17.418489) (1575130730890, 20, 17.302315) (1575130731890, 21, 14.283571) (1575130732890, 16, 16.826534) (1575130733890, 18, 19.222122) (1575130734890, 18, 14.931420) (1575130735890, 17, 19.549454) (1575130736890, 22, 16.908388) (1575130737890, 32, 15.637796) (1575130738890, 31, 15.517650) (1575130739890, 18, 14.038033) (1575130740890, 32, 19.859648) (1575130741890, 16, 13.220840) (1575130742890, 28, 16.445398) (1575130743890, 26, 16.695753) (1575130744890, 33, 13.696928) (1575130745890, 21, 15.352819) (1575130746890, 15, 12.388407) (1575130747890, 27, 11.267529) (1575130748890, 20, 14.103228) (1575130749890, 20, 16.250950) (1575130750890, 30, 16.236088) (1575130751890, 22, 18.305340) (1575130752890, 25, 17.360685) (1575130753890, 25, 14.978681) (1575130754890, 33, 14.096183) (1575130755890, 26, 10.019039) (1575130756890, 19, 19.158213) (1575130757890, 22, 15.593924) (1575130758890, 26, 18.780119) (1575130759890, 21, 16.001656) (1575130760890, 16, 18.458328) (1575130761890, 21, 16.417843) (1575130762890, 28, 11.736558) (1575130763890, 34, 18.143946) (1575130764890, 27, 10.303225) (1575130765890, 20, 19.756748) (1575130766890, 22, 12.940063) (1575130767890, 23, 11.509640) (1575130768890, 19, 18.319309) (1575130769890, 19, 16.278345) (1575130770890, 27, 10.898361) (1575130771890, 31, 13.922162) (1575130772890, 15, 19.296116) (1575130773890, 26, 15.885763) (1575130774890, 15, 15.525804) (1575130775890, 19, 19.579538) (1575130776890, 20, 11.073811) (1575130777890, 16, 13.932510) (1575130778890, 17, 11.900328) (1575130779890, 22, 16.540415) (1575130780890, 33, 15.203803) (1575130781890, 17, 11.518434) (1575130782890, 17, 13.152081) (1575130783890, 18, 11.378041) (1575130784890, 21, 15.390745) (1575130785890, 30, 15.127818) (1575130786890, 19, 16.530401) (1575130787890, 32, 16.542702) (1575130788890, 26, 16.366442) (1575130789890, 25, 10.306822) (1575130790890, 15, 13.691117) (1575130791890, 15, 13.476817) (1575130792890, 25, 12.529998) (1575130793890, 22, 15.550021) (1575130794890, 20, 15.064971) (1575130795890, 24, 13.313683) (1575130796890, 23, 17.002878) (1575130797890, 30, 19.991594) (1575130798890, 15, 11.116746) (1575130799880, 16, 19.405090) (1575130800880, 22, 14.377142) (1575130801880, 16, 16.868231) (1575130802880, 20, 11.565193) (1575130803880, 31, 13.009119) (1575130804880, 29, 18.136400) (1575130805880, 17, 13.806572) (1575130806880, 23, 14.688898) (1575130807880, 26, 12.931019) (1575130808880, 32, 12.185531) (1575130809880, 30, 13.608714) (1575130810880, 23, 18.624914) (1575130811880, 22, 12.970826) (1575130812880, 22, 12.065827) (1575130813880, 25, 16.967192) (1575130814880, 16, 10.283031) (1575130815880, 22, 16.072535) (1575130816880, 24, 10.794536) (1575130817880, 32, 10.591207) (1575130818880, 20, 13.015227) (1575130819880, 28, 15.410999) (1575130820880, 29, 12.785076) (1575130821880, 28, 15.305857) (1575130822880, 33, 12.820810) (1575130823880, 34, 13.618055) (1575130824880, 32, 12.971123) (1575130825880, 24, 10.974546) (1575130826880, 15, 10.742910) (1575130827880, 23, 16.810783) (1575130828880, 18, 13.115224) (1575130829880, 26, 17.418489) (1575130830880, 20, 17.302315) (1575130831880, 21, 14.283571) (1575130832880, 16, 16.826534) (1575130833880, 18, 19.222122) (1575130834880, 18, 14.931420) (1575130835880, 17, 19.549454) (1575130836880, 22, 16.908388) (1575130837880, 32, 15.637796) (1575130838880, 31, 15.517650) (1575130839880, 18, 14.038033) (1575130840880, 32, 19.859648) (1575130841880, 16, 13.220840) (1575130842880, 28, 16.445398) (1575130843880, 26, 16.695753) (1575130844880, 33, 13.696928) (1575130845880, 21, 15.352819) (1575130846880, 15, 12.388407) (1575130847880, 27, 11.267529) (1575130848880, 20, 14.103228) (1575130849880, 20, 16.250950) (1575130850880, 30, 16.236088) (1575130851880, 22, 18.305340) (1575130852880, 25, 17.360685) (1575130853880, 25, 14.978681) (1575130854880, 33, 14.096183) (1575130855880, 26, 10.019039) (1575130856880, 19, 19.158213) (1575130857880, 22, 15.593924) (1575130858880, 26, 18.780119) (1575130859880, 21, 16.001656) (1575130860880, 16, 18.458328) (1575130861880, 21, 16.417843) (1575130862880, 28, 11.736558) (1575130863880, 34, 18.143946) (1575130864880, 27, 10.303225) (1575130865880, 20, 19.756748) (1575130866880, 22, 12.940063) (1575130867880, 23, 11.509640) (1575130868880, 19, 18.319309) (1575130869880, 19, 16.278345) (1575130870880, 27, 10.898361) (1575130871880, 31, 13.922162) (1575130872880, 15, 19.296116) (1575130873880, 26, 15.885763) (1575130874880, 15, 15.525804) (1575130875880, 19, 19.579538) (1575130876880, 20, 11.073811) (1575130877880, 16, 13.932510) (1575130878880, 17, 11.900328) (1575130879880, 22, 16.540415) (1575130880880, 33, 15.203803) (1575130881880, 17, 11.518434) (1575130882880, 17, 13.152081) (1575130883880, 18, 11.378041) (1575130884880, 21, 15.390745) (1575130885880, 30, 15.127818) (1575130886880, 19, 16.530401) (1575130887880, 32, 16.542702) (1575130888880, 26, 16.366442) (1575130889880, 25, 10.306822) (1575130890880, 15, 13.691117) (1575130891880, 15, 13.476817) (1575130892880, 25, 12.529998) (1575130893880, 22, 15.550021) (1575130894880, 20, 15.064971) (1575130895880, 24, 13.313683) (1575130896880, 23, 17.002878) (1575130897880, 30, 19.991594) (1575130898880, 15, 11.116746) (1575130899870, 16, 19.405090) (1575130900870, 22, 14.377142) (1575130901870, 16, 16.868231) (1575130902870, 20, 11.565193) (1575130903870, 31, 13.009119) (1575130904870, 29, 18.136400) (1575130905870, 17, 13.806572) (1575130906870, 23, 14.688898) (1575130907870, 26, 12.931019) (1575130908870, 32, 12.185531) (1575130909870, 30, 13.608714) (1575130910870, 23, 18.624914) (1575130911870, 22, 12.970826) (1575130912870, 22, 12.065827) (1575130913870, 25, 16.967192) (1575130914870, 16, 10.283031) (1575130915870, 22, 16.072535) (1575130916870, 24, 10.794536) (1575130917870, 32, 10.591207) (1575130918870, 20, 13.015227) (1575130919870, 28, 15.410999) (1575130920870, 29, 12.785076) (1575130921870, 28, 15.305857) (1575130922870, 33, 12.820810) (1575130923870, 34, 13.618055) (1575130924870, 32, 12.971123) (1575130925870, 24, 10.974546) (1575130926870, 15, 10.742910) (1575130927870, 23, 16.810783) (1575130928870, 18, 13.115224) (1575130929870, 26, 17.418489) (1575130930870, 20, 17.302315) (1575130931870, 21, 14.283571) (1575130932870, 16, 16.826534) (1575130933870, 18, 19.222122) (1575130934870, 18, 14.931420) (1575130935870, 17, 19.549454) (1575130936870, 22, 16.908388) (1575130937870, 32, 15.637796) (1575130938870, 31, 15.517650) (1575130939870, 18, 14.038033) (1575130940870, 32, 19.859648) (1575130941870, 16, 13.220840) (1575130942870, 28, 16.445398) (1575130943870, 26, 16.695753) (1575130944870, 33, 13.696928) (1575130945870, 21, 15.352819) (1575130946870, 15, 12.388407) (1575130947870, 27, 11.267529) (1575130948870, 20, 14.103228) (1575130949870, 20, 16.250950) (1575130950870, 30, 16.236088) (1575130951870, 22, 18.305340) (1575130952870, 25, 17.360685) (1575130953870, 25, 14.978681) (1575130954870, 33, 14.096183) (1575130955870, 26, 10.019039) (1575130956870, 19, 19.158213) (1575130957870, 22, 15.593924) (1575130958870, 26, 18.780119) (1575130959870, 21, 16.001656) (1575130960870, 16, 18.458328) (1575130961870, 21, 16.417843) (1575130962870, 28, 11.736558) (1575130963870, 34, 18.143946) (1575130964870, 27, 10.303225) (1575130965870, 20, 19.756748) (1575130966870, 22, 12.940063) (1575130967870, 23, 11.509640) (1575130968870, 19, 18.319309) (1575130969870, 19, 16.278345) (1575130970870, 27, 10.898361) (1575130971870, 31, 13.922162) (1575130972870, 15, 19.296116) (1575130973870, 26, 15.885763) (1575130974870, 15, 15.525804) (1575130975870, 19, 19.579538) (1575130976870, 20, 11.073811) (1575130977870, 16, 13.932510) (1575130978870, 17, 11.900328) (1575130979870, 22, 16.540415) (1575130980870, 33, 15.203803) (1575130981870, 17, 11.518434) (1575130982870, 17, 13.152081) (1575130983870, 18, 11.378041) (1575130984870, 21, 15.390745) (1575130985870, 30, 15.127818) (1575130986870, 19, 16.530401) (1575130987870, 32, 16.542702) (1575130988870, 26, 16.366442) (1575130989870, 25, 10.306822) (1575130990870, 15, 13.691117) (1575130991870, 15, 13.476817) (1575130992870, 25, 12.529998) (1575130993870, 22, 15.550021) (1575130994870, 20, 15.064971) (1575130995870, 24, 13.313683) (1575130996870, 23, 17.002878) (1575130997870, 30, 19.991594) (1575130998870, 15, 11.116746) (1575130999860, 16, 19.405090) (1575131000860, 22, 14.377142) (1575131001860, 16, 16.868231) (1575131002860, 20, 11.565193) (1575131003860, 31, 13.009119) (1575131004860, 29, 18.136400) (1575131005860, 17, 13.806572) (1575131006860, 23, 14.688898) (1575131007860, 26, 12.931019) (1575131008860, 32, 12.185531) (1575131009860, 30, 13.608714) (1575131010860, 23, 18.624914) (1575131011860, 22, 12.970826) (1575131012860, 22, 12.065827) (1575131013860, 25, 16.967192) (1575131014860, 16, 10.283031) (1575131015860, 22, 16.072535) (1575131016860, 24, 10.794536) (1575131017860, 32, 10.591207) (1575131018860, 20, 13.015227) (1575131019860, 28, 15.410999) (1575131020860, 29, 12.785076) (1575131021860, 28, 15.305857) (1575131022860, 33, 12.820810) (1575131023860, 34, 13.618055) (1575131024860, 32, 12.971123) (1575131025860, 24, 10.974546) (1575131026860, 15, 10.742910) (1575131027860, 23, 16.810783) (1575131028860, 18, 13.115224) (1575131029860, 26, 17.418489) (1575131030860, 20, 17.302315) (1575131031860, 21, 14.283571) (1575131032860, 16, 16.826534) (1575131033860, 18, 19.222122) (1575131034860, 18, 14.931420) (1575131035860, 17, 19.549454) (1575131036860, 22, 16.908388) (1575131037860, 32, 15.637796) (1575131038860, 31, 15.517650) (1575131039860, 18, 14.038033) (1575131040860, 32, 19.859648) (1575131041860, 16, 13.220840) (1575131042860, 28, 16.445398) (1575131043860, 26, 16.695753) (1575131044860, 33, 13.696928) (1575131045860, 21, 15.352819) (1575131046860, 15, 12.388407) (1575131047860, 27, 11.267529) (1575131048860, 20, 14.103228) (1575131049860, 20, 16.250950) (1575131050860, 30, 16.236088) (1575131051860, 22, 18.305340) (1575131052860, 25, 17.360685) (1575131053860, 25, 14.978681) (1575131054860, 33, 14.096183) (1575131055860, 26, 10.019039) (1575131056860, 19, 19.158213) (1575131057860, 22, 15.593924) (1575131058860, 26, 18.780119) (1575131059860, 21, 16.001656) (1575131060860, 16, 18.458328) (1575131061860, 21, 16.417843) (1575131062860, 28, 11.736558) (1575131063860, 34, 18.143946) (1575131064860, 27, 10.303225) (1575131065860, 20, 19.756748) (1575131066860, 22, 12.940063) (1575131067860, 23, 11.509640) (1575131068860, 19, 18.319309) (1575131069860, 19, 16.278345) (1575131070860, 27, 10.898361) (1575131071860, 31, 13.922162) (1575131072860, 15, 19.296116) (1575131073860, 26, 15.885763) (1575131074860, 15, 15.525804) (1575131075860, 19, 19.579538) (1575131076860, 20, 11.073811) (1575131077860, 16, 13.932510) (1575131078860, 17, 11.900328) (1575131079860, 22, 16.540415) (1575131080860, 33, 15.203803) (1575131081860, 17, 11.518434) (1575131082860, 17, 13.152081) (1575131083860, 18, 11.378041) (1575131084860, 21, 15.390745) (1575131085860, 30, 15.127818) (1575131086860, 19, 16.530401) (1575131087860, 32, 16.542702) (1575131088860, 26, 16.366442) (1575131089860, 25, 10.306822) (1575131090860, 15, 13.691117) (1575131091860, 15, 13.476817) (1575131092860, 25, 12.529998) (1575131093860, 22, 15.550021) (1575131094860, 20, 15.064971) (1575131095860, 24, 13.313683) (1575131096860, 23, 17.002878) (1575131097860, 30, 19.991594) (1575131098860, 15, 11.116746) (1575131099850, 16, 19.405090) (1575131100850, 22, 14.377142) (1575131101850, 16, 16.868231) (1575131102850, 20, 11.565193) (1575131103850, 31, 13.009119) (1575131104850, 29, 18.136400) (1575131105850, 17, 13.806572) (1575131106850, 23, 14.688898) (1575131107850, 26, 12.931019) (1575131108850, 32, 12.185531) (1575131109850, 30, 13.608714) (1575131110850, 23, 18.624914) (1575131111850, 22, 12.970826) (1575131112850, 22, 12.065827) (1575131113850, 25, 16.967192) (1575131114850, 16, 10.283031) (1575131115850, 22, 16.072535) (1575131116850, 24, 10.794536) (1575131117850, 32, 10.591207) (1575131118850, 20, 13.015227) (1575131119850, 28, 15.410999) (1575131120850, 29, 12.785076) (1575131121850, 28, 15.305857) (1575131122850, 33, 12.820810) (1575131123850, 34, 13.618055) (1575131124850, 32, 12.971123) (1575131125850, 24, 10.974546) (1575131126850, 15, 10.742910) (1575131127850, 23, 16.810783) (1575131128850, 18, 13.115224) (1575131129850, 26, 17.418489) (1575131130850, 20, 17.302315) (1575131131850, 21, 14.283571) (1575131132850, 16, 16.826534) (1575131133850, 18, 19.222122) (1575131134850, 18, 14.931420) (1575131135850, 17, 19.549454) (1575131136850, 22, 16.908388) (1575131137850, 32, 15.637796) (1575131138850, 31, 15.517650) (1575131139850, 18, 14.038033) (1575131140850, 32, 19.859648) (1575131141850, 16, 13.220840) (1575131142850, 28, 16.445398) (1575131143850, 26, 16.695753) (1575131144850, 33, 13.696928) (1575131145850, 21, 15.352819) (1575131146850, 15, 12.388407) (1575131147850, 27, 11.267529) (1575131148850, 20, 14.103228) (1575131149850, 20, 16.250950) (1575131150850, 30, 16.236088) (1575131151850, 22, 18.305340) (1575131152850, 25, 17.360685) (1575131153850, 25, 14.978681) (1575131154850, 33, 14.096183) (1575131155850, 26, 10.019039) (1575131156850, 19, 19.158213) (1575131157850, 22, 15.593924) (1575131158850, 26, 18.780119) (1575131159850, 21, 16.001656) (1575131160850, 16, 18.458328) (1575131161850, 21, 16.417843) (1575131162850, 28, 11.736558) (1575131163850, 34, 18.143946) (1575131164850, 27, 10.303225) (1575131165850, 20, 19.756748) (1575131166850, 22, 12.940063) (1575131167850, 23, 11.509640) (1575131168850, 19, 18.319309) (1575131169850, 19, 16.278345) (1575131170850, 27, 10.898361) (1575131171850, 31, 13.922162) (1575131172850, 15, 19.296116) (1575131173850, 26, 15.885763) (1575131174850, 15, 15.525804) (1575131175850, 19, 19.579538) (1575131176850, 20, 11.073811) (1575131177850, 16, 13.932510) (1575131178850, 17, 11.900328) (1575131179850, 22, 16.540415) (1575131180850, 33, 15.203803) (1575131181850, 17, 11.518434) (1575131182850, 17, 13.152081) (1575131183850, 18, 11.378041) (1575131184850, 21, 15.390745) (1575131185850, 30, 15.127818) (1575131186850, 19, 16.530401) (1575131187850, 32, 16.542702) (1575131188850, 26, 16.366442) (1575131189850, 25, 10.306822) (1575131190850, 15, 13.691117) (1575131191850, 15, 13.476817) (1575131192850, 25, 12.529998) (1575131193850, 22, 15.550021) (1575131194850, 20, 15.064971) (1575131195850, 24, 13.313683) (1575131196850, 23, 17.002878) (1575131197850, 30, 19.991594) (1575131198850, 15, 11.116746) (1575131199840, 16, 19.405090) (1575131200840, 22, 14.377142) (1575131201840, 16, 16.868231) (1575131202840, 20, 11.565193) (1575131203840, 31, 13.009119) (1575131204840, 29, 18.136400) (1575131205840, 17, 13.806572) (1575131206840, 23, 14.688898) (1575131207840, 26, 12.931019) (1575131208840, 32, 12.185531) (1575131209840, 30, 13.608714) (1575131210840, 23, 18.624914) (1575131211840, 22, 12.970826) (1575131212840, 22, 12.065827) (1575131213840, 25, 16.967192) (1575131214840, 16, 10.283031) (1575131215840, 22, 16.072535) (1575131216840, 24, 10.794536) (1575131217840, 32, 10.591207) (1575131218840, 20, 13.015227) (1575131219840, 28, 15.410999) (1575131220840, 29, 12.785076) (1575131221840, 28, 15.305857) (1575131222840, 33, 12.820810) (1575131223840, 34, 13.618055) (1575131224840, 32, 12.971123) (1575131225840, 24, 10.974546) (1575131226840, 15, 10.742910) (1575131227840, 23, 16.810783) (1575131228840, 18, 13.115224) (1575131229840, 26, 17.418489) (1575131230840, 20, 17.302315) (1575131231840, 21, 14.283571) (1575131232840, 16, 16.826534) (1575131233840, 18, 19.222122) (1575131234840, 18, 14.931420) (1575131235840, 17, 19.549454) (1575131236840, 22, 16.908388) (1575131237840, 32, 15.637796) (1575131238840, 31, 15.517650) (1575131239840, 18, 14.038033) (1575131240840, 32, 19.859648) (1575131241840, 16, 13.220840) (1575131242840, 28, 16.445398) (1575131243840, 26, 16.695753) (1575131244840, 33, 13.696928) (1575131245840, 21, 15.352819) (1575131246840, 15, 12.388407) (1575131247840, 27, 11.267529) (1575131248840, 20, 14.103228) (1575131249840, 20, 16.250950) (1575131250840, 30, 16.236088) (1575131251840, 22, 18.305340) (1575131252840, 25, 17.360685) (1575131253840, 25, 14.978681) (1575131254840, 33, 14.096183) (1575131255840, 26, 10.019039) (1575131256840, 19, 19.158213) (1575131257840, 22, 15.593924) (1575131258840, 26, 18.780119) (1575131259840, 21, 16.001656) (1575131260840, 16, 18.458328) (1575131261840, 21, 16.417843) (1575131262840, 28, 11.736558) (1575131263840, 34, 18.143946) (1575131264840, 27, 10.303225) (1575131265840, 20, 19.756748) (1575131266840, 22, 12.940063) (1575131267840, 23, 11.509640) (1575131268840, 19, 18.319309) (1575131269840, 19, 16.278345) (1575131270840, 27, 10.898361) (1575131271840, 31, 13.922162) (1575131272840, 15, 19.296116) (1575131273840, 26, 15.885763) (1575131274840, 15, 15.525804) (1575131275840, 19, 19.579538) (1575131276840, 20, 11.073811) (1575131277840, 16, 13.932510) (1575131278840, 17, 11.900328) (1575131279840, 22, 16.540415) (1575131280840, 33, 15.203803) (1575131281840, 17, 11.518434) (1575131282840, 17, 13.152081) (1575131283840, 18, 11.378041) (1575131284840, 21, 15.390745) (1575131285840, 30, 15.127818) (1575131286840, 19, 16.530401) (1575131287840, 32, 16.542702) (1575131288840, 26, 16.366442) (1575131289840, 25, 10.306822) (1575131290840, 15, 13.691117) (1575131291840, 15, 13.476817) (1575131292840, 25, 12.529998) (1575131293840, 22, 15.550021) (1575131294840, 20, 15.064971) (1575131295840, 24, 13.313683) (1575131296840, 23, 17.002878) (1575131297840, 30, 19.991594) (1575131298840, 15, 11.116746) (1575131299830, 16, 19.405090) (1575131300830, 22, 14.377142) (1575131301830, 16, 16.868231) (1575131302830, 20, 11.565193) (1575131303830, 31, 13.009119) (1575131304830, 29, 18.136400) (1575131305830, 17, 13.806572) (1575131306830, 23, 14.688898) (1575131307830, 26, 12.931019) (1575131308830, 32, 12.185531) (1575131309830, 30, 13.608714) (1575131310830, 23, 18.624914) (1575131311830, 22, 12.970826) (1575131312830, 22, 12.065827) (1575131313830, 25, 16.967192) (1575131314830, 16, 10.283031) (1575131315830, 22, 16.072535) (1575131316830, 24, 10.794536) (1575131317830, 32, 10.591207) (1575131318830, 20, 13.015227) (1575131319830, 28, 15.410999) (1575131320830, 29, 12.785076) (1575131321830, 28, 15.305857) (1575131322830, 33, 12.820810) (1575131323830, 34, 13.618055) (1575131324830, 32, 12.971123) (1575131325830, 24, 10.974546) (1575131326830, 15, 10.742910) (1575131327830, 23, 16.810783) (1575131328830, 18, 13.115224) (1575131329830, 26, 17.418489) (1575131330830, 20, 17.302315) (1575131331830, 21, 14.283571) (1575131332830, 16, 16.826534) (1575131333830, 18, 19.222122) (1575131334830, 18, 14.931420) (1575131335830, 17, 19.549454) (1575131336830, 22, 16.908388) (1575131337830, 32, 15.637796) (1575131338830, 31, 15.517650) (1575131339830, 18, 14.038033) (1575131340830, 32, 19.859648) (1575131341830, 16, 13.220840) (1575131342830, 28, 16.445398) (1575131343830, 26, 16.695753) (1575131344830, 33, 13.696928) (1575131345830, 21, 15.352819) (1575131346830, 15, 12.388407) (1575131347830, 27, 11.267529) (1575131348830, 20, 14.103228) (1575131349830, 20, 16.250950) (1575131350830, 30, 16.236088) (1575131351830, 22, 18.305340) (1575131352830, 25, 17.360685) (1575131353830, 25, 14.978681) (1575131354830, 33, 14.096183) (1575131355830, 26, 10.019039) (1575131356830, 19, 19.158213) (1575131357830, 22, 15.593924) (1575131358830, 26, 18.780119) (1575131359830, 21, 16.001656) (1575131360830, 16, 18.458328) (1575131361830, 21, 16.417843) (1575131362830, 28, 11.736558) (1575131363830, 34, 18.143946) (1575131364830, 27, 10.303225) (1575131365830, 20, 19.756748) (1575131366830, 22, 12.940063) (1575131367830, 23, 11.509640) (1575131368830, 19, 18.319309) (1575131369830, 19, 16.278345) (1575131370830, 27, 10.898361) (1575131371830, 31, 13.922162) (1575131372830, 15, 19.296116) (1575131373830, 26, 15.885763) (1575131374830, 15, 15.525804) (1575131375830, 19, 19.579538) (1575131376830, 20, 11.073811) (1575131377830, 16, 13.932510) (1575131378830, 17, 11.900328) (1575131379830, 22, 16.540415) (1575131380830, 33, 15.203803) (1575131381830, 17, 11.518434) (1575131382830, 17, 13.152081) (1575131383830, 18, 11.378041) (1575131384830, 21, 15.390745) (1575131385830, 30, 15.127818) (1575131386830, 19, 16.530401) (1575131387830, 32, 16.542702) (1575131388830, 26, 16.366442) (1575131389830, 25, 10.306822) (1575131390830, 15, 13.691117) (1575131391830, 15, 13.476817) (1575131392830, 25, 12.529998) (1575131393830, 22, 15.550021) (1575131394830, 20, 15.064971) (1575131395830, 24, 13.313683) (1575131396830, 23, 17.002878) (1575131397830, 30, 19.991594) (1575131398830, 15, 11.116746) (1575131399820, 16, 19.405090) (1575131400820, 22, 14.377142) (1575131401820, 16, 16.868231) (1575131402820, 20, 11.565193) (1575131403820, 31, 13.009119) (1575131404820, 29, 18.136400) (1575131405820, 17, 13.806572) (1575131406820, 23, 14.688898) (1575131407820, 26, 12.931019) (1575131408820, 32, 12.185531) (1575131409820, 30, 13.608714) (1575131410820, 23, 18.624914) (1575131411820, 22, 12.970826) (1575131412820, 22, 12.065827) (1575131413820, 25, 16.967192) (1575131414820, 16, 10.283031) (1575131415820, 22, 16.072535) (1575131416820, 24, 10.794536) (1575131417820, 32, 10.591207) (1575131418820, 20, 13.015227) (1575131419820, 28, 15.410999) (1575131420820, 29, 12.785076) (1575131421820, 28, 15.305857) (1575131422820, 33, 12.820810) (1575131423820, 34, 13.618055) (1575131424820, 32, 12.971123) (1575131425820, 24, 10.974546) (1575131426820, 15, 10.742910) (1575131427820, 23, 16.810783) (1575131428820, 18, 13.115224) (1575131429820, 26, 17.418489) (1575131430820, 20, 17.302315) (1575131431820, 21, 14.283571) (1575131432820, 16, 16.826534) (1575131433820, 18, 19.222122) (1575131434820, 18, 14.931420) (1575131435820, 17, 19.549454) (1575131436820, 22, 16.908388) (1575131437820, 32, 15.637796) (1575131438820, 31, 15.517650) (1575131439820, 18, 14.038033) (1575131440820, 32, 19.859648) (1575131441820, 16, 13.220840) (1575131442820, 28, 16.445398) (1575131443820, 26, 16.695753) (1575131444820, 33, 13.696928) (1575131445820, 21, 15.352819) (1575131446820, 15, 12.388407) (1575131447820, 27, 11.267529) (1575131448820, 20, 14.103228) (1575131449820, 20, 16.250950) (1575131450820, 30, 16.236088) (1575131451820, 22, 18.305340) (1575131452820, 25, 17.360685) (1575131453820, 25, 14.978681) (1575131454820, 33, 14.096183) (1575131455820, 26, 10.019039) (1575131456820, 19, 19.158213) (1575131457820, 22, 15.593924) (1575131458820, 26, 18.780119) (1575131459820, 21, 16.001656) (1575131460820, 16, 18.458328) (1575131461820, 21, 16.417843) (1575131462820, 28, 11.736558) (1575131463820, 34, 18.143946) (1575131464820, 27, 10.303225) (1575131465820, 20, 19.756748) (1575131466820, 22, 12.940063) (1575131467820, 23, 11.509640) (1575131468820, 19, 18.319309) (1575131469820, 19, 16.278345) (1575131470820, 27, 10.898361) (1575131471820, 31, 13.922162) (1575131472820, 15, 19.296116) (1575131473820, 26, 15.885763) (1575131474820, 15, 15.525804) (1575131475820, 19, 19.579538) (1575131476820, 20, 11.073811) (1575131477820, 16, 13.932510) (1575131478820, 17, 11.900328) (1575131479820, 22, 16.540415) (1575131480820, 33, 15.203803) (1575131481820, 17, 11.518434) (1575131482820, 17, 13.152081) (1575131483820, 18, 11.378041) (1575131484820, 21, 15.390745) (1575131485820, 30, 15.127818) (1575131486820, 19, 16.530401) (1575131487820, 32, 16.542702) (1575131488820, 26, 16.366442) (1575131489820, 25, 10.306822) (1575131490820, 15, 13.691117) (1575131491820, 15, 13.476817) (1575131492820, 25, 12.529998) (1575131493820, 22, 15.550021) (1575131494820, 20, 15.064971) (1575131495820, 24, 13.313683) (1575131496820, 23, 17.002878) (1575131497820, 30, 19.991594) (1575131498820, 15, 11.116746) (1575131499810, 16, 19.405090) (1575131500810, 22, 14.377142) (1575131501810, 16, 16.868231) (1575131502810, 20, 11.565193) (1575131503810, 31, 13.009119) (1575131504810, 29, 18.136400) (1575131505810, 17, 13.806572) (1575131506810, 23, 14.688898) (1575131507810, 26, 12.931019) (1575131508810, 32, 12.185531) (1575131509810, 30, 13.608714) (1575131510810, 23, 18.624914) (1575131511810, 22, 12.970826) (1575131512810, 22, 12.065827) (1575131513810, 25, 16.967192) (1575131514810, 16, 10.283031) (1575131515810, 22, 16.072535) (1575131516810, 24, 10.794536) (1575131517810, 32, 10.591207) (1575131518810, 20, 13.015227) (1575131519810, 28, 15.410999) (1575131520810, 29, 12.785076) (1575131521810, 28, 15.305857) (1575131522810, 33, 12.820810) (1575131523810, 34, 13.618055) (1575131524810, 32, 12.971123) (1575131525810, 24, 10.974546) (1575131526810, 15, 10.742910) (1575131527810, 23, 16.810783) (1575131528810, 18, 13.115224) (1575131529810, 26, 17.418489) (1575131530810, 20, 17.302315) (1575131531810, 21, 14.283571) (1575131532810, 16, 16.826534) (1575131533810, 18, 19.222122) (1575131534810, 18, 14.931420) (1575131535810, 17, 19.549454) (1575131536810, 22, 16.908388) (1575131537810, 32, 15.637796) (1575131538810, 31, 15.517650) (1575131539810, 18, 14.038033) (1575131540810, 32, 19.859648) (1575131541810, 16, 13.220840) (1575131542810, 28, 16.445398) (1575131543810, 26, 16.695753) (1575131544810, 33, 13.696928) (1575131545810, 21, 15.352819) (1575131546810, 15, 12.388407) (1575131547810, 27, 11.267529) (1575131548810, 20, 14.103228) (1575131549810, 20, 16.250950) (1575131550810, 30, 16.236088) (1575131551810, 22, 18.305340) (1575131552810, 25, 17.360685) (1575131553810, 25, 14.978681) (1575131554810, 33, 14.096183) (1575131555810, 26, 10.019039) (1575131556810, 19, 19.158213) (1575131557810, 22, 15.593924) (1575131558810, 26, 18.780119) (1575131559810, 21, 16.001656) (1575131560810, 16, 18.458328) (1575131561810, 21, 16.417843) (1575131562810, 28, 11.736558) (1575131563810, 34, 18.143946) (1575131564810, 27, 10.303225) (1575131565810, 20, 19.756748) (1575131566810, 22, 12.940063) (1575131567810, 23, 11.509640) (1575131568810, 19, 18.319309) (1575131569810, 19, 16.278345) (1575131570810, 27, 10.898361) (1575131571810, 31, 13.922162) (1575131572810, 15, 19.296116) (1575131573810, 26, 15.885763) (1575131574810, 15, 15.525804) (1575131575810, 19, 19.579538) (1575131576810, 20, 11.073811) (1575131577810, 16, 13.932510) (1575131578810, 17, 11.900328) (1575131579810, 22, 16.540415) (1575131580810, 33, 15.203803) (1575131581810, 17, 11.518434) (1575131582810, 17, 13.152081) (1575131583810, 18, 11.378041) (1575131584810, 21, 15.390745) (1575131585810, 30, 15.127818) (1575131586810, 19, 16.530401) (1575131587810, 32, 16.542702) (1575131588810, 26, 16.366442) (1575131589810, 25, 10.306822) (1575131590810, 15, 13.691117) (1575131591810, 15, 13.476817) (1575131592810, 25, 12.529998) (1575131593810, 22, 15.550021) (1575131594810, 20, 15.064971) (1575131595810, 24, 13.313683) (1575131596810, 23, 17.002878) (1575131597810, 30, 19.991594) (1575131598810, 15, 11.116746) (1575131599800, 16, 19.405090) (1575131600800, 22, 14.377142) (1575131601800, 16, 16.868231) (1575131602800, 20, 11.565193) (1575131603800, 31, 13.009119) (1575131604800, 29, 18.136400) (1575131605800, 17, 13.806572) (1575131606800, 23, 14.688898) (1575131607800, 26, 12.931019) (1575131608800, 32, 12.185531) (1575131609800, 30, 13.608714) (1575131610800, 23, 18.624914) (1575131611800, 22, 12.970826) (1575131612800, 22, 12.065827) (1575131613800, 25, 16.967192) (1575131614800, 16, 10.283031) (1575131615800, 22, 16.072535) (1575131616800, 24, 10.794536) (1575131617800, 32, 10.591207) (1575131618800, 20, 13.015227) (1575131619800, 28, 15.410999) (1575131620800, 29, 12.785076) (1575131621800, 28, 15.305857) (1575131622800, 33, 12.820810) (1575131623800, 34, 13.618055) (1575131624800, 32, 12.971123) (1575131625800, 24, 10.974546) (1575131626800, 15, 10.742910) (1575131627800, 23, 16.810783) (1575131628800, 18, 13.115224) (1575131629800, 26, 17.418489) (1575131630800, 20, 17.302315) (1575131631800, 21, 14.283571) (1575131632800, 16, 16.826534) (1575131633800, 18, 19.222122) (1575131634800, 18, 14.931420) (1575131635800, 17, 19.549454) (1575131636800, 22, 16.908388) (1575131637800, 32, 15.637796) (1575131638800, 31, 15.517650) (1575131639800, 18, 14.038033) (1575131640800, 32, 19.859648) (1575131641800, 16, 13.220840) (1575131642800, 28, 16.445398) (1575131643800, 26, 16.695753) (1575131644800, 33, 13.696928) (1575131645800, 21, 15.352819) (1575131646800, 15, 12.388407) (1575131647800, 27, 11.267529) (1575131648800, 20, 14.103228) (1575131649800, 20, 16.250950) (1575131650800, 30, 16.236088) (1575131651800, 22, 18.305340) (1575131652800, 25, 17.360685) (1575131653800, 25, 14.978681) (1575131654800, 33, 14.096183) (1575131655800, 26, 10.019039) (1575131656800, 19, 19.158213) (1575131657800, 22, 15.593924) (1575131658800, 26, 18.780119) (1575131659800, 21, 16.001656) (1575131660800, 16, 18.458328) (1575131661800, 21, 16.417843) (1575131662800, 28, 11.736558) (1575131663800, 34, 18.143946) (1575131664800, 27, 10.303225) (1575131665800, 20, 19.756748) (1575131666800, 22, 12.940063) (1575131667800, 23, 11.509640) (1575131668800, 19, 18.319309) (1575131669800, 19, 16.278345) (1575131670800, 27, 10.898361) (1575131671800, 31, 13.922162) (1575131672800, 15, 19.296116) (1575131673800, 26, 15.885763) (1575131674800, 15, 15.525804) (1575131675800, 19, 19.579538) (1575131676800, 20, 11.073811) (1575131677800, 16, 13.932510) (1575131678800, 17, 11.900328) (1575131679800, 22, 16.540415) (1575131680800, 33, 15.203803) (1575131681800, 17, 11.518434) (1575131682800, 17, 13.152081) (1575131683800, 18, 11.378041) (1575131684800, 21, 15.390745) (1575131685800, 30, 15.127818) (1575131686800, 19, 16.530401) (1575131687800, 32, 16.542702) (1575131688800, 26, 16.366442) (1575131689800, 25, 10.306822) (1575131690800, 15, 13.691117) (1575131691800, 15, 13.476817) (1575131692800, 25, 12.529998) (1575131693800, 22, 15.550021)") + + end_time = int(round(time.time() * 1000)) + tdLog.info("Execute time with compress: %dms" % (end_time - start_time)) + + simLogPath = tdDnodes.getSimLogPath() + grepCmd = "grep -a 'compress rpc msg, before:' -r %s | head -2" % simLogPath + output = subprocess.check_output(grepCmd, shell=True).decode("utf-8") + + if output != "": + tdLog.info("Find %s in log file." % output) + tdLog.success("%s successfully executed! Compress works as expected." % __file__) + else: + tdLog.exit("%s failed! Compress does NOT works." % __file__) + + conn.close() diff --git a/tests/pytest/testNoCompress.py b/tests/pytest/testNoCompress.py new file mode 100644 index 0000000000..e3b40b4426 --- /dev/null +++ b/tests/pytest/testNoCompress.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/linux/python2/ + +# -*- coding: utf-8 -*- +import sys +import getopt +import subprocess +from distutils.log import warn as printf + +from util.log import * +from util.dnodes import * +from util.cases import * +from util.sql import * + +import taos + + +if __name__ == "__main__": + fileName = "all" + deployPath = "" + masterIp = "" + testCluster = False + valgrind = 0 + logSql = True + stop = 0 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'l:sgh', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help']) + for key, value in opts: + if key in ['-h', '--help']: + tdLog.printNoPrefix( + 'A collection of test cases written using Python') + tdLog.printNoPrefix('-l logSql Flag') + tdLog.printNoPrefix('-s stop All dnodes') + tdLog.printNoPrefix('-g valgrind Test Flag') + sys.exit(0) + + if key in ['-l', '--logSql']: + if (value.upper() == "TRUE"): + logSql = True + elif (value.upper() == "FALSE"): + logSql = False + else: + tdLog.printNoPrefix("logSql value %s is invalid" % logSql) + sys.exit(0) + + if key in ['-g', '--valgrind']: + valgrind = 1 + + if key in ['-s', '--stop']: + stop = 1 + + if (stop != 0): + if (valgrind == 0): + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP > /dev/null 2>&1" % toBeKilled + + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output(psCmd, shell=True) + + while(processID): + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output(psCmd, shell=True) + + for port in range(6030, 6041): + usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port + processID = subprocess.check_output(usePortPID, shell=True) + + if processID: + killCmd = "kill -9 %s" % processID + os.system(killCmd) + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if valgrind: + time.sleep(2) + + tdLog.info('stop All dnodes') + sys.exit(0) + + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + + tdDnodes.stopAll() + tdDnodes.addSimExtraCfg("compressMsgSize", "-1") + tdDnodes.deploy(1) + tdDnodes.start(1) + + host = '127.0.0.1' + + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + + tdCases.logSql(logSql) + + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + + tdSql.init(conn.cursor(), logSql) + + tdSql.execute("CREATE DATABASE IF NOT EXISTS t10b") + tdSql.execute("USE t10b") + tdSql.execute( + "CREATE TABLE IF NOT EXISTS s_sensor_info (ts TIMESTAMP, temperature INT, humidity FLOAT)") + + start_time = int(round(time.time() * 1000)) + for i in range(1, 1000): + tdSql.execute("IMPORT INTO s_sensor_info VALUES (1575129600000, 16, 19.405090) (1575129601000, 22, 14.377142) (1575129602000, 16, 16.868231) (1575129603000, 20, 11.565193) (1575129604000, 31, 13.009119) (1575129605000, 29, 18.136400) (1575129606000, 17, 13.806572) (1575129607000, 23, 14.688898) (1575129608000, 26, 12.931019) (1575129609000, 32, 12.185531) (1575129610000, 30, 13.608714) (1575129611000, 23, 18.624914) (1575129612000, 22, 12.970826) (1575129613000, 22, 12.065827) (1575129614000, 25, 16.967192) (1575129615000, 16, 10.283031) (1575129616000, 22, 16.072535) (1575129617000, 24, 10.794536) (1575129618000, 32, 10.591207) (1575129619000, 20, 13.015227) (1575129620000, 28, 15.410999) (1575129621000, 29, 12.785076) (1575129622000, 28, 15.305857) (1575129623000, 33, 12.820810) (1575129624000, 34, 13.618055) (1575129625000, 32, 12.971123) (1575129626000, 24, 10.974546) (1575129627000, 15, 10.742910) (1575129628000, 23, 16.810783) (1575129629000, 18, 13.115224) (1575129630000, 26, 17.418489) (1575129631000, 20, 17.302315) (1575129632000, 21, 14.283571) (1575129633000, 16, 16.826534) (1575129634000, 18, 19.222122) (1575129635000, 18, 14.931420) (1575129636000, 17, 19.549454) (1575129637000, 22, 16.908388) (1575129638000, 32, 15.637796) (1575129639000, 31, 15.517650) (1575129640000, 18, 14.038033) (1575129641000, 32, 19.859648) (1575129642000, 16, 13.220840) (1575129643000, 28, 16.445398) (1575129644000, 26, 16.695753) (1575129645000, 33, 13.696928) (1575129646000, 21, 15.352819) (1575129647000, 15, 12.388407) (1575129648000, 27, 11.267529) (1575129649000, 20, 14.103228) (1575129650000, 20, 16.250950) (1575129651000, 30, 16.236088) (1575129652000, 22, 18.305340) (1575129653000, 25, 17.360685) (1575129654000, 25, 14.978681) (1575129655000, 33, 14.096183) (1575129656000, 26, 10.019039) (1575129657000, 19, 19.158213) (1575129658000, 22, 15.593924) (1575129659000, 26, 18.780119) (1575129660000, 21, 16.001656) (1575129661000, 16, 18.458328) (1575129662000, 21, 16.417843) (1575129663000, 28, 11.736558) (1575129664000, 34, 18.143946) (1575129665000, 27, 10.303225) (1575129666000, 20, 19.756748) (1575129667000, 22, 12.940063) (1575129668000, 23, 11.509640) (1575129669000, 19, 18.319309) (1575129670000, 19, 16.278345) (1575129671000, 27, 10.898361) (1575129672000, 31, 13.922162) (1575129673000, 15, 19.296116) (1575129674000, 26, 15.885763) (1575129675000, 15, 15.525804) (1575129676000, 19, 19.579538) (1575129677000, 20, 11.073811) (1575129678000, 16, 13.932510) (1575129679000, 17, 11.900328) (1575129680000, 22, 16.540415) (1575129681000, 33, 15.203803) (1575129682000, 17, 11.518434) (1575129683000, 17, 13.152081) (1575129684000, 18, 11.378041) (1575129685000, 21, 15.390745) (1575129686000, 30, 15.127818) (1575129687000, 19, 16.530401) (1575129688000, 32, 16.542702) (1575129689000, 26, 16.366442) (1575129690000, 25, 10.306822) (1575129691000, 15, 13.691117) (1575129692000, 15, 13.476817) (1575129693000, 25, 12.529998) (1575129694000, 22, 15.550021) (1575129695000, 20, 15.064971) (1575129696000, 24, 13.313683) (1575129697000, 23, 17.002878) (1575129698000, 30, 19.991594) (1575129699000, 15, 11.116746) (1575129699990, 16, 19.405090) (1575129700990, 22, 14.377142) (1575129701990, 16, 16.868231) (1575129702990, 20, 11.565193) (1575129703990, 31, 13.009119) (1575129704990, 29, 18.136400) (1575129705990, 17, 13.806572) (1575129706990, 23, 14.688898) (1575129707990, 26, 12.931019) (1575129708990, 32, 12.185531) (1575129709990, 30, 13.608714) (1575129710990, 23, 18.624914) (1575129711990, 22, 12.970826) (1575129712990, 22, 12.065827) (1575129713990, 25, 16.967192) (1575129714990, 16, 10.283031) (1575129715990, 22, 16.072535) (1575129716990, 24, 10.794536) (1575129717990, 32, 10.591207) (1575129718990, 20, 13.015227) (1575129719990, 28, 15.410999) (1575129720990, 29, 12.785076) (1575129721990, 28, 15.305857) (1575129722990, 33, 12.820810) (1575129723990, 34, 13.618055) (1575129724990, 32, 12.971123) (1575129725990, 24, 10.974546) (1575129726990, 15, 10.742910) (1575129727990, 23, 16.810783) (1575129728990, 18, 13.115224) (1575129729990, 26, 17.418489) (1575129730990, 20, 17.302315) (1575129731990, 21, 14.283571) (1575129732990, 16, 16.826534) (1575129733990, 18, 19.222122) (1575129734990, 18, 14.931420) (1575129735990, 17, 19.549454) (1575129736990, 22, 16.908388) (1575129737990, 32, 15.637796) (1575129738990, 31, 15.517650) (1575129739990, 18, 14.038033) (1575129740990, 32, 19.859648) (1575129741990, 16, 13.220840) (1575129742990, 28, 16.445398) (1575129743990, 26, 16.695753) (1575129744990, 33, 13.696928) (1575129745990, 21, 15.352819) (1575129746990, 15, 12.388407) (1575129747990, 27, 11.267529) (1575129748990, 20, 14.103228) (1575129749990, 20, 16.250950) (1575129750990, 30, 16.236088) (1575129751990, 22, 18.305340) (1575129752990, 25, 17.360685) (1575129753990, 25, 14.978681) (1575129754990, 33, 14.096183) (1575129755990, 26, 10.019039) (1575129756990, 19, 19.158213) (1575129757990, 22, 15.593924) (1575129758990, 26, 18.780119) (1575129759990, 21, 16.001656) (1575129760990, 16, 18.458328) (1575129761990, 21, 16.417843) (1575129762990, 28, 11.736558) (1575129763990, 34, 18.143946) (1575129764990, 27, 10.303225) (1575129765990, 20, 19.756748) (1575129766990, 22, 12.940063) (1575129767990, 23, 11.509640) (1575129768990, 19, 18.319309) (1575129769990, 19, 16.278345) (1575129770990, 27, 10.898361) (1575129771990, 31, 13.922162) (1575129772990, 15, 19.296116) (1575129773990, 26, 15.885763) (1575129774990, 15, 15.525804) (1575129775990, 19, 19.579538) (1575129776990, 20, 11.073811) (1575129777990, 16, 13.932510) (1575129778990, 17, 11.900328) (1575129779990, 22, 16.540415) (1575129780990, 33, 15.203803) (1575129781990, 17, 11.518434) (1575129782990, 17, 13.152081) (1575129783990, 18, 11.378041) (1575129784990, 21, 15.390745) (1575129785990, 30, 15.127818) (1575129786990, 19, 16.530401) (1575129787990, 32, 16.542702) (1575129788990, 26, 16.366442) (1575129789990, 25, 10.306822) (1575129790990, 15, 13.691117) (1575129791990, 15, 13.476817) (1575129792990, 25, 12.529998) (1575129793990, 22, 15.550021) (1575129794990, 20, 15.064971) (1575129795990, 24, 13.313683) (1575129796990, 23, 17.002878) (1575129797990, 30, 19.991594) (1575129798990, 15, 11.116746) (1575129799980, 16, 19.405090) (1575129800980, 22, 14.377142) (1575129801980, 16, 16.868231) (1575129802980, 20, 11.565193) (1575129803980, 31, 13.009119) (1575129804980, 29, 18.136400) (1575129805980, 17, 13.806572) (1575129806980, 23, 14.688898) (1575129807980, 26, 12.931019) (1575129808980, 32, 12.185531) (1575129809980, 30, 13.608714) (1575129810980, 23, 18.624914) (1575129811980, 22, 12.970826) (1575129812980, 22, 12.065827) (1575129813980, 25, 16.967192) (1575129814980, 16, 10.283031) (1575129815980, 22, 16.072535) (1575129816980, 24, 10.794536) (1575129817980, 32, 10.591207) (1575129818980, 20, 13.015227) (1575129819980, 28, 15.410999) (1575129820980, 29, 12.785076) (1575129821980, 28, 15.305857) (1575129822980, 33, 12.820810) (1575129823980, 34, 13.618055) (1575129824980, 32, 12.971123) (1575129825980, 24, 10.974546) (1575129826980, 15, 10.742910) (1575129827980, 23, 16.810783) (1575129828980, 18, 13.115224) (1575129829980, 26, 17.418489) (1575129830980, 20, 17.302315) (1575129831980, 21, 14.283571) (1575129832980, 16, 16.826534) (1575129833980, 18, 19.222122) (1575129834980, 18, 14.931420) (1575129835980, 17, 19.549454) (1575129836980, 22, 16.908388) (1575129837980, 32, 15.637796) (1575129838980, 31, 15.517650) (1575129839980, 18, 14.038033) (1575129840980, 32, 19.859648) (1575129841980, 16, 13.220840) (1575129842980, 28, 16.445398) (1575129843980, 26, 16.695753) (1575129844980, 33, 13.696928) (1575129845980, 21, 15.352819) (1575129846980, 15, 12.388407) (1575129847980, 27, 11.267529) (1575129848980, 20, 14.103228) (1575129849980, 20, 16.250950) (1575129850980, 30, 16.236088) (1575129851980, 22, 18.305340) (1575129852980, 25, 17.360685) (1575129853980, 25, 14.978681) (1575129854980, 33, 14.096183) (1575129855980, 26, 10.019039) (1575129856980, 19, 19.158213) (1575129857980, 22, 15.593924) (1575129858980, 26, 18.780119) (1575129859980, 21, 16.001656) (1575129860980, 16, 18.458328) (1575129861980, 21, 16.417843) (1575129862980, 28, 11.736558) (1575129863980, 34, 18.143946) (1575129864980, 27, 10.303225) (1575129865980, 20, 19.756748) (1575129866980, 22, 12.940063) (1575129867980, 23, 11.509640) (1575129868980, 19, 18.319309) (1575129869980, 19, 16.278345) (1575129870980, 27, 10.898361) (1575129871980, 31, 13.922162) (1575129872980, 15, 19.296116) (1575129873980, 26, 15.885763) (1575129874980, 15, 15.525804) (1575129875980, 19, 19.579538) (1575129876980, 20, 11.073811) (1575129877980, 16, 13.932510) (1575129878980, 17, 11.900328) (1575129879980, 22, 16.540415) (1575129880980, 33, 15.203803) (1575129881980, 17, 11.518434) (1575129882980, 17, 13.152081) (1575129883980, 18, 11.378041) (1575129884980, 21, 15.390745) (1575129885980, 30, 15.127818) (1575129886980, 19, 16.530401) (1575129887980, 32, 16.542702) (1575129888980, 26, 16.366442) (1575129889980, 25, 10.306822) (1575129890980, 15, 13.691117) (1575129891980, 15, 13.476817) (1575129892980, 25, 12.529998) (1575129893980, 22, 15.550021) (1575129894980, 20, 15.064971) (1575129895980, 24, 13.313683) (1575129896980, 23, 17.002878) (1575129897980, 30, 19.991594) (1575129898980, 15, 11.116746) (1575129899970, 16, 19.405090) (1575129900970, 22, 14.377142) (1575129901970, 16, 16.868231) (1575129902970, 20, 11.565193) (1575129903970, 31, 13.009119) (1575129904970, 29, 18.136400) (1575129905970, 17, 13.806572) (1575129906970, 23, 14.688898) (1575129907970, 26, 12.931019) (1575129908970, 32, 12.185531) (1575129909970, 30, 13.608714) (1575129910970, 23, 18.624914) (1575129911970, 22, 12.970826) (1575129912970, 22, 12.065827) (1575129913970, 25, 16.967192) (1575129914970, 16, 10.283031) (1575129915970, 22, 16.072535) (1575129916970, 24, 10.794536) (1575129917970, 32, 10.591207) (1575129918970, 20, 13.015227) (1575129919970, 28, 15.410999) (1575129920970, 29, 12.785076) (1575129921970, 28, 15.305857) (1575129922970, 33, 12.820810) (1575129923970, 34, 13.618055) (1575129924970, 32, 12.971123) (1575129925970, 24, 10.974546) (1575129926970, 15, 10.742910) (1575129927970, 23, 16.810783) (1575129928970, 18, 13.115224) (1575129929970, 26, 17.418489) (1575129930970, 20, 17.302315) (1575129931970, 21, 14.283571) (1575129932970, 16, 16.826534) (1575129933970, 18, 19.222122) (1575129934970, 18, 14.931420) (1575129935970, 17, 19.549454) (1575129936970, 22, 16.908388) (1575129937970, 32, 15.637796) (1575129938970, 31, 15.517650) (1575129939970, 18, 14.038033) (1575129940970, 32, 19.859648) (1575129941970, 16, 13.220840) (1575129942970, 28, 16.445398) (1575129943970, 26, 16.695753) (1575129944970, 33, 13.696928) (1575129945970, 21, 15.352819) (1575129946970, 15, 12.388407) (1575129947970, 27, 11.267529) (1575129948970, 20, 14.103228) (1575129949970, 20, 16.250950) (1575129950970, 30, 16.236088) (1575129951970, 22, 18.305340) (1575129952970, 25, 17.360685) (1575129953970, 25, 14.978681) (1575129954970, 33, 14.096183) (1575129955970, 26, 10.019039) (1575129956970, 19, 19.158213) (1575129957970, 22, 15.593924) (1575129958970, 26, 18.780119) (1575129959970, 21, 16.001656) (1575129960970, 16, 18.458328) (1575129961970, 21, 16.417843) (1575129962970, 28, 11.736558) (1575129963970, 34, 18.143946) (1575129964970, 27, 10.303225) (1575129965970, 20, 19.756748) (1575129966970, 22, 12.940063) (1575129967970, 23, 11.509640) (1575129968970, 19, 18.319309) (1575129969970, 19, 16.278345) (1575129970970, 27, 10.898361) (1575129971970, 31, 13.922162) (1575129972970, 15, 19.296116) (1575129973970, 26, 15.885763) (1575129974970, 15, 15.525804) (1575129975970, 19, 19.579538) (1575129976970, 20, 11.073811) (1575129977970, 16, 13.932510) (1575129978970, 17, 11.900328) (1575129979970, 22, 16.540415) (1575129980970, 33, 15.203803) (1575129981970, 17, 11.518434) (1575129982970, 17, 13.152081) (1575129983970, 18, 11.378041) (1575129984970, 21, 15.390745) (1575129985970, 30, 15.127818) (1575129986970, 19, 16.530401) (1575129987970, 32, 16.542702) (1575129988970, 26, 16.366442) (1575129989970, 25, 10.306822) (1575129990970, 15, 13.691117) (1575129991970, 15, 13.476817) (1575129992970, 25, 12.529998) (1575129993970, 22, 15.550021) (1575129994970, 20, 15.064971) (1575129995970, 24, 13.313683) (1575129996970, 23, 17.002878) (1575129997970, 30, 19.991594) (1575129998970, 15, 11.116746) (1575129999960, 16, 19.405090) (1575130000960, 22, 14.377142) (1575130001960, 16, 16.868231) (1575130002960, 20, 11.565193) (1575130003960, 31, 13.009119) (1575130004960, 29, 18.136400) (1575130005960, 17, 13.806572) (1575130006960, 23, 14.688898) (1575130007960, 26, 12.931019) (1575130008960, 32, 12.185531) (1575130009960, 30, 13.608714) (1575130010960, 23, 18.624914) (1575130011960, 22, 12.970826) (1575130012960, 22, 12.065827) (1575130013960, 25, 16.967192) (1575130014960, 16, 10.283031) (1575130015960, 22, 16.072535) (1575130016960, 24, 10.794536) (1575130017960, 32, 10.591207) (1575130018960, 20, 13.015227) (1575130019960, 28, 15.410999) (1575130020960, 29, 12.785076) (1575130021960, 28, 15.305857) (1575130022960, 33, 12.820810) (1575130023960, 34, 13.618055) (1575130024960, 32, 12.971123) (1575130025960, 24, 10.974546) (1575130026960, 15, 10.742910) (1575130027960, 23, 16.810783) (1575130028960, 18, 13.115224) (1575130029960, 26, 17.418489) (1575130030960, 20, 17.302315) (1575130031960, 21, 14.283571) (1575130032960, 16, 16.826534) (1575130033960, 18, 19.222122) (1575130034960, 18, 14.931420) (1575130035960, 17, 19.549454) (1575130036960, 22, 16.908388) (1575130037960, 32, 15.637796) (1575130038960, 31, 15.517650) (1575130039960, 18, 14.038033) (1575130040960, 32, 19.859648) (1575130041960, 16, 13.220840) (1575130042960, 28, 16.445398) (1575130043960, 26, 16.695753) (1575130044960, 33, 13.696928) (1575130045960, 21, 15.352819) (1575130046960, 15, 12.388407) (1575130047960, 27, 11.267529) (1575130048960, 20, 14.103228) (1575130049960, 20, 16.250950) (1575130050960, 30, 16.236088) (1575130051960, 22, 18.305340) (1575130052960, 25, 17.360685) (1575130053960, 25, 14.978681) (1575130054960, 33, 14.096183) (1575130055960, 26, 10.019039) (1575130056960, 19, 19.158213) (1575130057960, 22, 15.593924) (1575130058960, 26, 18.780119) (1575130059960, 21, 16.001656) (1575130060960, 16, 18.458328) (1575130061960, 21, 16.417843) (1575130062960, 28, 11.736558) (1575130063960, 34, 18.143946) (1575130064960, 27, 10.303225) (1575130065960, 20, 19.756748) (1575130066960, 22, 12.940063) (1575130067960, 23, 11.509640) (1575130068960, 19, 18.319309) (1575130069960, 19, 16.278345) (1575130070960, 27, 10.898361) (1575130071960, 31, 13.922162) (1575130072960, 15, 19.296116) (1575130073960, 26, 15.885763) (1575130074960, 15, 15.525804) (1575130075960, 19, 19.579538) (1575130076960, 20, 11.073811) (1575130077960, 16, 13.932510) (1575130078960, 17, 11.900328) (1575130079960, 22, 16.540415) (1575130080960, 33, 15.203803) (1575130081960, 17, 11.518434) (1575130082960, 17, 13.152081) (1575130083960, 18, 11.378041) (1575130084960, 21, 15.390745) (1575130085960, 30, 15.127818) (1575130086960, 19, 16.530401) (1575130087960, 32, 16.542702) (1575130088960, 26, 16.366442) (1575130089960, 25, 10.306822) (1575130090960, 15, 13.691117) (1575130091960, 15, 13.476817) (1575130092960, 25, 12.529998) (1575130093960, 22, 15.550021) (1575130094960, 20, 15.064971) (1575130095960, 24, 13.313683) (1575130096960, 23, 17.002878) (1575130097960, 30, 19.991594) (1575130098960, 15, 11.116746) (1575130099950, 16, 19.405090) (1575130100950, 22, 14.377142) (1575130101950, 16, 16.868231) (1575130102950, 20, 11.565193) (1575130103950, 31, 13.009119) (1575130104950, 29, 18.136400) (1575130105950, 17, 13.806572) (1575130106950, 23, 14.688898) (1575130107950, 26, 12.931019) (1575130108950, 32, 12.185531) (1575130109950, 30, 13.608714) (1575130110950, 23, 18.624914) (1575130111950, 22, 12.970826) (1575130112950, 22, 12.065827) (1575130113950, 25, 16.967192) (1575130114950, 16, 10.283031) (1575130115950, 22, 16.072535) (1575130116950, 24, 10.794536) (1575130117950, 32, 10.591207) (1575130118950, 20, 13.015227) (1575130119950, 28, 15.410999) (1575130120950, 29, 12.785076) (1575130121950, 28, 15.305857) (1575130122950, 33, 12.820810) (1575130123950, 34, 13.618055) (1575130124950, 32, 12.971123) (1575130125950, 24, 10.974546) (1575130126950, 15, 10.742910) (1575130127950, 23, 16.810783) (1575130128950, 18, 13.115224) (1575130129950, 26, 17.418489) (1575130130950, 20, 17.302315) (1575130131950, 21, 14.283571) (1575130132950, 16, 16.826534) (1575130133950, 18, 19.222122) (1575130134950, 18, 14.931420) (1575130135950, 17, 19.549454) (1575130136950, 22, 16.908388) (1575130137950, 32, 15.637796) (1575130138950, 31, 15.517650) (1575130139950, 18, 14.038033) (1575130140950, 32, 19.859648) (1575130141950, 16, 13.220840) (1575130142950, 28, 16.445398) (1575130143950, 26, 16.695753) (1575130144950, 33, 13.696928) (1575130145950, 21, 15.352819) (1575130146950, 15, 12.388407) (1575130147950, 27, 11.267529) (1575130148950, 20, 14.103228) (1575130149950, 20, 16.250950) (1575130150950, 30, 16.236088) (1575130151950, 22, 18.305340) (1575130152950, 25, 17.360685) (1575130153950, 25, 14.978681) (1575130154950, 33, 14.096183) (1575130155950, 26, 10.019039) (1575130156950, 19, 19.158213) (1575130157950, 22, 15.593924) (1575130158950, 26, 18.780119) (1575130159950, 21, 16.001656) (1575130160950, 16, 18.458328) (1575130161950, 21, 16.417843) (1575130162950, 28, 11.736558) (1575130163950, 34, 18.143946) (1575130164950, 27, 10.303225) (1575130165950, 20, 19.756748) (1575130166950, 22, 12.940063) (1575130167950, 23, 11.509640) (1575130168950, 19, 18.319309) (1575130169950, 19, 16.278345) (1575130170950, 27, 10.898361) (1575130171950, 31, 13.922162) (1575130172950, 15, 19.296116) (1575130173950, 26, 15.885763) (1575130174950, 15, 15.525804) (1575130175950, 19, 19.579538) (1575130176950, 20, 11.073811) (1575130177950, 16, 13.932510) (1575130178950, 17, 11.900328) (1575130179950, 22, 16.540415) (1575130180950, 33, 15.203803) (1575130181950, 17, 11.518434) (1575130182950, 17, 13.152081) (1575130183950, 18, 11.378041) (1575130184950, 21, 15.390745) (1575130185950, 30, 15.127818) (1575130186950, 19, 16.530401) (1575130187950, 32, 16.542702) (1575130188950, 26, 16.366442) (1575130189950, 25, 10.306822) (1575130190950, 15, 13.691117) (1575130191950, 15, 13.476817) (1575130192950, 25, 12.529998) (1575130193950, 22, 15.550021) (1575130194950, 20, 15.064971) (1575130195950, 24, 13.313683) (1575130196950, 23, 17.002878) (1575130197950, 30, 19.991594) (1575130198950, 15, 11.116746) (1575130199940, 16, 19.405090) (1575130200940, 22, 14.377142) (1575130201940, 16, 16.868231) (1575130202940, 20, 11.565193) (1575130203940, 31, 13.009119) (1575130204940, 29, 18.136400) (1575130205940, 17, 13.806572) (1575130206940, 23, 14.688898) (1575130207940, 26, 12.931019) (1575130208940, 32, 12.185531) (1575130209940, 30, 13.608714) (1575130210940, 23, 18.624914) (1575130211940, 22, 12.970826) (1575130212940, 22, 12.065827) (1575130213940, 25, 16.967192) (1575130214940, 16, 10.283031) (1575130215940, 22, 16.072535) (1575130216940, 24, 10.794536) (1575130217940, 32, 10.591207) (1575130218940, 20, 13.015227) (1575130219940, 28, 15.410999) (1575130220940, 29, 12.785076) (1575130221940, 28, 15.305857) (1575130222940, 33, 12.820810) (1575130223940, 34, 13.618055) (1575130224940, 32, 12.971123) (1575130225940, 24, 10.974546) (1575130226940, 15, 10.742910) (1575130227940, 23, 16.810783) (1575130228940, 18, 13.115224) (1575130229940, 26, 17.418489) (1575130230940, 20, 17.302315) (1575130231940, 21, 14.283571) (1575130232940, 16, 16.826534) (1575130233940, 18, 19.222122) (1575130234940, 18, 14.931420) (1575130235940, 17, 19.549454) (1575130236940, 22, 16.908388) (1575130237940, 32, 15.637796) (1575130238940, 31, 15.517650) (1575130239940, 18, 14.038033) (1575130240940, 32, 19.859648) (1575130241940, 16, 13.220840) (1575130242940, 28, 16.445398) (1575130243940, 26, 16.695753) (1575130244940, 33, 13.696928) (1575130245940, 21, 15.352819) (1575130246940, 15, 12.388407) (1575130247940, 27, 11.267529) (1575130248940, 20, 14.103228) (1575130249940, 20, 16.250950) (1575130250940, 30, 16.236088) (1575130251940, 22, 18.305340) (1575130252940, 25, 17.360685) (1575130253940, 25, 14.978681) (1575130254940, 33, 14.096183) (1575130255940, 26, 10.019039) (1575130256940, 19, 19.158213) (1575130257940, 22, 15.593924) (1575130258940, 26, 18.780119) (1575130259940, 21, 16.001656) (1575130260940, 16, 18.458328) (1575130261940, 21, 16.417843) (1575130262940, 28, 11.736558) (1575130263940, 34, 18.143946) (1575130264940, 27, 10.303225) (1575130265940, 20, 19.756748) (1575130266940, 22, 12.940063) (1575130267940, 23, 11.509640) (1575130268940, 19, 18.319309) (1575130269940, 19, 16.278345) (1575130270940, 27, 10.898361) (1575130271940, 31, 13.922162) (1575130272940, 15, 19.296116) (1575130273940, 26, 15.885763) (1575130274940, 15, 15.525804) (1575130275940, 19, 19.579538) (1575130276940, 20, 11.073811) (1575130277940, 16, 13.932510) (1575130278940, 17, 11.900328) (1575130279940, 22, 16.540415) (1575130280940, 33, 15.203803) (1575130281940, 17, 11.518434) (1575130282940, 17, 13.152081) (1575130283940, 18, 11.378041) (1575130284940, 21, 15.390745) (1575130285940, 30, 15.127818) (1575130286940, 19, 16.530401) (1575130287940, 32, 16.542702) (1575130288940, 26, 16.366442) (1575130289940, 25, 10.306822) (1575130290940, 15, 13.691117) (1575130291940, 15, 13.476817) (1575130292940, 25, 12.529998) (1575130293940, 22, 15.550021) (1575130294940, 20, 15.064971) (1575130295940, 24, 13.313683) (1575130296940, 23, 17.002878) (1575130297940, 30, 19.991594) (1575130298940, 15, 11.116746) (1575130299930, 16, 19.405090) (1575130300930, 22, 14.377142) (1575130301930, 16, 16.868231) (1575130302930, 20, 11.565193) (1575130303930, 31, 13.009119) (1575130304930, 29, 18.136400) (1575130305930, 17, 13.806572) (1575130306930, 23, 14.688898) (1575130307930, 26, 12.931019) (1575130308930, 32, 12.185531) (1575130309930, 30, 13.608714) (1575130310930, 23, 18.624914) (1575130311930, 22, 12.970826) (1575130312930, 22, 12.065827) (1575130313930, 25, 16.967192) (1575130314930, 16, 10.283031) (1575130315930, 22, 16.072535) (1575130316930, 24, 10.794536) (1575130317930, 32, 10.591207) (1575130318930, 20, 13.015227) (1575130319930, 28, 15.410999) (1575130320930, 29, 12.785076) (1575130321930, 28, 15.305857) (1575130322930, 33, 12.820810) (1575130323930, 34, 13.618055) (1575130324930, 32, 12.971123) (1575130325930, 24, 10.974546) (1575130326930, 15, 10.742910) (1575130327930, 23, 16.810783) (1575130328930, 18, 13.115224) (1575130329930, 26, 17.418489) (1575130330930, 20, 17.302315) (1575130331930, 21, 14.283571) (1575130332930, 16, 16.826534) (1575130333930, 18, 19.222122) (1575130334930, 18, 14.931420) (1575130335930, 17, 19.549454) (1575130336930, 22, 16.908388) (1575130337930, 32, 15.637796) (1575130338930, 31, 15.517650) (1575130339930, 18, 14.038033) (1575130340930, 32, 19.859648) (1575130341930, 16, 13.220840) (1575130342930, 28, 16.445398) (1575130343930, 26, 16.695753) (1575130344930, 33, 13.696928) (1575130345930, 21, 15.352819) (1575130346930, 15, 12.388407) (1575130347930, 27, 11.267529) (1575130348930, 20, 14.103228) (1575130349930, 20, 16.250950) (1575130350930, 30, 16.236088) (1575130351930, 22, 18.305340) (1575130352930, 25, 17.360685) (1575130353930, 25, 14.978681) (1575130354930, 33, 14.096183) (1575130355930, 26, 10.019039) (1575130356930, 19, 19.158213) (1575130357930, 22, 15.593924) (1575130358930, 26, 18.780119) (1575130359930, 21, 16.001656) (1575130360930, 16, 18.458328) (1575130361930, 21, 16.417843) (1575130362930, 28, 11.736558) (1575130363930, 34, 18.143946) (1575130364930, 27, 10.303225) (1575130365930, 20, 19.756748) (1575130366930, 22, 12.940063) (1575130367930, 23, 11.509640) (1575130368930, 19, 18.319309) (1575130369930, 19, 16.278345) (1575130370930, 27, 10.898361) (1575130371930, 31, 13.922162) (1575130372930, 15, 19.296116) (1575130373930, 26, 15.885763) (1575130374930, 15, 15.525804) (1575130375930, 19, 19.579538) (1575130376930, 20, 11.073811) (1575130377930, 16, 13.932510) (1575130378930, 17, 11.900328) (1575130379930, 22, 16.540415) (1575130380930, 33, 15.203803) (1575130381930, 17, 11.518434) (1575130382930, 17, 13.152081) (1575130383930, 18, 11.378041) (1575130384930, 21, 15.390745) (1575130385930, 30, 15.127818) (1575130386930, 19, 16.530401) (1575130387930, 32, 16.542702) (1575130388930, 26, 16.366442) (1575130389930, 25, 10.306822) (1575130390930, 15, 13.691117) (1575130391930, 15, 13.476817) (1575130392930, 25, 12.529998) (1575130393930, 22, 15.550021) (1575130394930, 20, 15.064971) (1575130395930, 24, 13.313683) (1575130396930, 23, 17.002878) (1575130397930, 30, 19.991594) (1575130398930, 15, 11.116746) (1575130399920, 16, 19.405090) (1575130400920, 22, 14.377142) (1575130401920, 16, 16.868231) (1575130402920, 20, 11.565193) (1575130403920, 31, 13.009119) (1575130404920, 29, 18.136400) (1575130405920, 17, 13.806572) (1575130406920, 23, 14.688898) (1575130407920, 26, 12.931019) (1575130408920, 32, 12.185531) (1575130409920, 30, 13.608714) (1575130410920, 23, 18.624914) (1575130411920, 22, 12.970826) (1575130412920, 22, 12.065827) (1575130413920, 25, 16.967192) (1575130414920, 16, 10.283031) (1575130415920, 22, 16.072535) (1575130416920, 24, 10.794536) (1575130417920, 32, 10.591207) (1575130418920, 20, 13.015227) (1575130419920, 28, 15.410999) (1575130420920, 29, 12.785076) (1575130421920, 28, 15.305857) (1575130422920, 33, 12.820810) (1575130423920, 34, 13.618055) (1575130424920, 32, 12.971123) (1575130425920, 24, 10.974546) (1575130426920, 15, 10.742910) (1575130427920, 23, 16.810783) (1575130428920, 18, 13.115224) (1575130429920, 26, 17.418489) (1575130430920, 20, 17.302315) (1575130431920, 21, 14.283571) (1575130432920, 16, 16.826534) (1575130433920, 18, 19.222122) (1575130434920, 18, 14.931420) (1575130435920, 17, 19.549454) (1575130436920, 22, 16.908388) (1575130437920, 32, 15.637796) (1575130438920, 31, 15.517650) (1575130439920, 18, 14.038033) (1575130440920, 32, 19.859648) (1575130441920, 16, 13.220840) (1575130442920, 28, 16.445398) (1575130443920, 26, 16.695753) (1575130444920, 33, 13.696928) (1575130445920, 21, 15.352819) (1575130446920, 15, 12.388407) (1575130447920, 27, 11.267529) (1575130448920, 20, 14.103228) (1575130449920, 20, 16.250950) (1575130450920, 30, 16.236088) (1575130451920, 22, 18.305340) (1575130452920, 25, 17.360685) (1575130453920, 25, 14.978681) (1575130454920, 33, 14.096183) (1575130455920, 26, 10.019039) (1575130456920, 19, 19.158213) (1575130457920, 22, 15.593924) (1575130458920, 26, 18.780119) (1575130459920, 21, 16.001656) (1575130460920, 16, 18.458328) (1575130461920, 21, 16.417843) (1575130462920, 28, 11.736558) (1575130463920, 34, 18.143946) (1575130464920, 27, 10.303225) (1575130465920, 20, 19.756748) (1575130466920, 22, 12.940063) (1575130467920, 23, 11.509640) (1575130468920, 19, 18.319309) (1575130469920, 19, 16.278345) (1575130470920, 27, 10.898361) (1575130471920, 31, 13.922162) (1575130472920, 15, 19.296116) (1575130473920, 26, 15.885763) (1575130474920, 15, 15.525804) (1575130475920, 19, 19.579538) (1575130476920, 20, 11.073811) (1575130477920, 16, 13.932510) (1575130478920, 17, 11.900328) (1575130479920, 22, 16.540415) (1575130480920, 33, 15.203803) (1575130481920, 17, 11.518434) (1575130482920, 17, 13.152081) (1575130483920, 18, 11.378041) (1575130484920, 21, 15.390745) (1575130485920, 30, 15.127818) (1575130486920, 19, 16.530401) (1575130487920, 32, 16.542702) (1575130488920, 26, 16.366442) (1575130489920, 25, 10.306822) (1575130490920, 15, 13.691117) (1575130491920, 15, 13.476817) (1575130492920, 25, 12.529998) (1575130493920, 22, 15.550021) (1575130494920, 20, 15.064971) (1575130495920, 24, 13.313683) (1575130496920, 23, 17.002878) (1575130497920, 30, 19.991594) (1575130498920, 15, 11.116746) (1575130499910, 16, 19.405090) (1575130500910, 22, 14.377142) (1575130501910, 16, 16.868231) (1575130502910, 20, 11.565193) (1575130503910, 31, 13.009119) (1575130504910, 29, 18.136400) (1575130505910, 17, 13.806572) (1575130506910, 23, 14.688898) (1575130507910, 26, 12.931019) (1575130508910, 32, 12.185531) (1575130509910, 30, 13.608714) (1575130510910, 23, 18.624914) (1575130511910, 22, 12.970826) (1575130512910, 22, 12.065827) (1575130513910, 25, 16.967192) (1575130514910, 16, 10.283031) (1575130515910, 22, 16.072535) (1575130516910, 24, 10.794536) (1575130517910, 32, 10.591207) (1575130518910, 20, 13.015227) (1575130519910, 28, 15.410999) (1575130520910, 29, 12.785076) (1575130521910, 28, 15.305857) (1575130522910, 33, 12.820810) (1575130523910, 34, 13.618055) (1575130524910, 32, 12.971123) (1575130525910, 24, 10.974546) (1575130526910, 15, 10.742910) (1575130527910, 23, 16.810783) (1575130528910, 18, 13.115224) (1575130529910, 26, 17.418489) (1575130530910, 20, 17.302315) (1575130531910, 21, 14.283571) (1575130532910, 16, 16.826534) (1575130533910, 18, 19.222122) (1575130534910, 18, 14.931420) (1575130535910, 17, 19.549454) (1575130536910, 22, 16.908388) (1575130537910, 32, 15.637796) (1575130538910, 31, 15.517650) (1575130539910, 18, 14.038033) (1575130540910, 32, 19.859648) (1575130541910, 16, 13.220840) (1575130542910, 28, 16.445398) (1575130543910, 26, 16.695753) (1575130544910, 33, 13.696928) (1575130545910, 21, 15.352819) (1575130546910, 15, 12.388407) (1575130547910, 27, 11.267529) (1575130548910, 20, 14.103228) (1575130549910, 20, 16.250950) (1575130550910, 30, 16.236088) (1575130551910, 22, 18.305340) (1575130552910, 25, 17.360685) (1575130553910, 25, 14.978681) (1575130554910, 33, 14.096183) (1575130555910, 26, 10.019039) (1575130556910, 19, 19.158213) (1575130557910, 22, 15.593924) (1575130558910, 26, 18.780119) (1575130559910, 21, 16.001656) (1575130560910, 16, 18.458328) (1575130561910, 21, 16.417843) (1575130562910, 28, 11.736558) (1575130563910, 34, 18.143946) (1575130564910, 27, 10.303225) (1575130565910, 20, 19.756748) (1575130566910, 22, 12.940063) (1575130567910, 23, 11.509640) (1575130568910, 19, 18.319309) (1575130569910, 19, 16.278345) (1575130570910, 27, 10.898361) (1575130571910, 31, 13.922162) (1575130572910, 15, 19.296116) (1575130573910, 26, 15.885763) (1575130574910, 15, 15.525804) (1575130575910, 19, 19.579538) (1575130576910, 20, 11.073811) (1575130577910, 16, 13.932510) (1575130578910, 17, 11.900328) (1575130579910, 22, 16.540415) (1575130580910, 33, 15.203803) (1575130581910, 17, 11.518434) (1575130582910, 17, 13.152081) (1575130583910, 18, 11.378041) (1575130584910, 21, 15.390745) (1575130585910, 30, 15.127818) (1575130586910, 19, 16.530401) (1575130587910, 32, 16.542702) (1575130588910, 26, 16.366442) (1575130589910, 25, 10.306822) (1575130590910, 15, 13.691117) (1575130591910, 15, 13.476817) (1575130592910, 25, 12.529998) (1575130593910, 22, 15.550021) (1575130594910, 20, 15.064971) (1575130595910, 24, 13.313683) (1575130596910, 23, 17.002878) (1575130597910, 30, 19.991594) (1575130598910, 15, 11.116746) (1575130599900, 16, 19.405090) (1575130600900, 22, 14.377142) (1575130601900, 16, 16.868231) (1575130602900, 20, 11.565193) (1575130603900, 31, 13.009119) (1575130604900, 29, 18.136400) (1575130605900, 17, 13.806572) (1575130606900, 23, 14.688898) (1575130607900, 26, 12.931019) (1575130608900, 32, 12.185531) (1575130609900, 30, 13.608714) (1575130610900, 23, 18.624914) (1575130611900, 22, 12.970826) (1575130612900, 22, 12.065827) (1575130613900, 25, 16.967192) (1575130614900, 16, 10.283031) (1575130615900, 22, 16.072535) (1575130616900, 24, 10.794536) (1575130617900, 32, 10.591207) (1575130618900, 20, 13.015227) (1575130619900, 28, 15.410999) (1575130620900, 29, 12.785076) (1575130621900, 28, 15.305857) (1575130622900, 33, 12.820810) (1575130623900, 34, 13.618055) (1575130624900, 32, 12.971123) (1575130625900, 24, 10.974546) (1575130626900, 15, 10.742910) (1575130627900, 23, 16.810783) (1575130628900, 18, 13.115224) (1575130629900, 26, 17.418489) (1575130630900, 20, 17.302315) (1575130631900, 21, 14.283571) (1575130632900, 16, 16.826534) (1575130633900, 18, 19.222122) (1575130634900, 18, 14.931420) (1575130635900, 17, 19.549454) (1575130636900, 22, 16.908388) (1575130637900, 32, 15.637796) (1575130638900, 31, 15.517650) (1575130639900, 18, 14.038033) (1575130640900, 32, 19.859648) (1575130641900, 16, 13.220840) (1575130642900, 28, 16.445398) (1575130643900, 26, 16.695753) (1575130644900, 33, 13.696928) (1575130645900, 21, 15.352819) (1575130646900, 15, 12.388407) (1575130647900, 27, 11.267529) (1575130648900, 20, 14.103228) (1575130649900, 20, 16.250950) (1575130650900, 30, 16.236088) (1575130651900, 22, 18.305340) (1575130652900, 25, 17.360685) (1575130653900, 25, 14.978681) (1575130654900, 33, 14.096183) (1575130655900, 26, 10.019039) (1575130656900, 19, 19.158213) (1575130657900, 22, 15.593924) (1575130658900, 26, 18.780119) (1575130659900, 21, 16.001656) (1575130660900, 16, 18.458328) (1575130661900, 21, 16.417843) (1575130662900, 28, 11.736558) (1575130663900, 34, 18.143946) (1575130664900, 27, 10.303225) (1575130665900, 20, 19.756748) (1575130666900, 22, 12.940063) (1575130667900, 23, 11.509640) (1575130668900, 19, 18.319309) (1575130669900, 19, 16.278345) (1575130670900, 27, 10.898361) (1575130671900, 31, 13.922162) (1575130672900, 15, 19.296116) (1575130673900, 26, 15.885763) (1575130674900, 15, 15.525804) (1575130675900, 19, 19.579538) (1575130676900, 20, 11.073811) (1575130677900, 16, 13.932510) (1575130678900, 17, 11.900328) (1575130679900, 22, 16.540415) (1575130680900, 33, 15.203803) (1575130681900, 17, 11.518434) (1575130682900, 17, 13.152081) (1575130683900, 18, 11.378041) (1575130684900, 21, 15.390745) (1575130685900, 30, 15.127818) (1575130686900, 19, 16.530401) (1575130687900, 32, 16.542702) (1575130688900, 26, 16.366442) (1575130689900, 25, 10.306822) (1575130690900, 15, 13.691117) (1575130691900, 15, 13.476817) (1575130692900, 25, 12.529998) (1575130693900, 22, 15.550021) (1575130694900, 20, 15.064971) (1575130695900, 24, 13.313683) (1575130696900, 23, 17.002878) (1575130697900, 30, 19.991594) (1575130698900, 15, 11.116746) (1575130699890, 16, 19.405090) (1575130700890, 22, 14.377142) (1575130701890, 16, 16.868231) (1575130702890, 20, 11.565193) (1575130703890, 31, 13.009119) (1575130704890, 29, 18.136400) (1575130705890, 17, 13.806572) (1575130706890, 23, 14.688898) (1575130707890, 26, 12.931019) (1575130708890, 32, 12.185531) (1575130709890, 30, 13.608714) (1575130710890, 23, 18.624914) (1575130711890, 22, 12.970826) (1575130712890, 22, 12.065827) (1575130713890, 25, 16.967192) (1575130714890, 16, 10.283031) (1575130715890, 22, 16.072535) (1575130716890, 24, 10.794536) (1575130717890, 32, 10.591207) (1575130718890, 20, 13.015227) (1575130719890, 28, 15.410999) (1575130720890, 29, 12.785076) (1575130721890, 28, 15.305857) (1575130722890, 33, 12.820810) (1575130723890, 34, 13.618055) (1575130724890, 32, 12.971123) (1575130725890, 24, 10.974546) (1575130726890, 15, 10.742910) (1575130727890, 23, 16.810783) (1575130728890, 18, 13.115224) (1575130729890, 26, 17.418489) (1575130730890, 20, 17.302315) (1575130731890, 21, 14.283571) (1575130732890, 16, 16.826534) (1575130733890, 18, 19.222122) (1575130734890, 18, 14.931420) (1575130735890, 17, 19.549454) (1575130736890, 22, 16.908388) (1575130737890, 32, 15.637796) (1575130738890, 31, 15.517650) (1575130739890, 18, 14.038033) (1575130740890, 32, 19.859648) (1575130741890, 16, 13.220840) (1575130742890, 28, 16.445398) (1575130743890, 26, 16.695753) (1575130744890, 33, 13.696928) (1575130745890, 21, 15.352819) (1575130746890, 15, 12.388407) (1575130747890, 27, 11.267529) (1575130748890, 20, 14.103228) (1575130749890, 20, 16.250950) (1575130750890, 30, 16.236088) (1575130751890, 22, 18.305340) (1575130752890, 25, 17.360685) (1575130753890, 25, 14.978681) (1575130754890, 33, 14.096183) (1575130755890, 26, 10.019039) (1575130756890, 19, 19.158213) (1575130757890, 22, 15.593924) (1575130758890, 26, 18.780119) (1575130759890, 21, 16.001656) (1575130760890, 16, 18.458328) (1575130761890, 21, 16.417843) (1575130762890, 28, 11.736558) (1575130763890, 34, 18.143946) (1575130764890, 27, 10.303225) (1575130765890, 20, 19.756748) (1575130766890, 22, 12.940063) (1575130767890, 23, 11.509640) (1575130768890, 19, 18.319309) (1575130769890, 19, 16.278345) (1575130770890, 27, 10.898361) (1575130771890, 31, 13.922162) (1575130772890, 15, 19.296116) (1575130773890, 26, 15.885763) (1575130774890, 15, 15.525804) (1575130775890, 19, 19.579538) (1575130776890, 20, 11.073811) (1575130777890, 16, 13.932510) (1575130778890, 17, 11.900328) (1575130779890, 22, 16.540415) (1575130780890, 33, 15.203803) (1575130781890, 17, 11.518434) (1575130782890, 17, 13.152081) (1575130783890, 18, 11.378041) (1575130784890, 21, 15.390745) (1575130785890, 30, 15.127818) (1575130786890, 19, 16.530401) (1575130787890, 32, 16.542702) (1575130788890, 26, 16.366442) (1575130789890, 25, 10.306822) (1575130790890, 15, 13.691117) (1575130791890, 15, 13.476817) (1575130792890, 25, 12.529998) (1575130793890, 22, 15.550021) (1575130794890, 20, 15.064971) (1575130795890, 24, 13.313683) (1575130796890, 23, 17.002878) (1575130797890, 30, 19.991594) (1575130798890, 15, 11.116746) (1575130799880, 16, 19.405090) (1575130800880, 22, 14.377142) (1575130801880, 16, 16.868231) (1575130802880, 20, 11.565193) (1575130803880, 31, 13.009119) (1575130804880, 29, 18.136400) (1575130805880, 17, 13.806572) (1575130806880, 23, 14.688898) (1575130807880, 26, 12.931019) (1575130808880, 32, 12.185531) (1575130809880, 30, 13.608714) (1575130810880, 23, 18.624914) (1575130811880, 22, 12.970826) (1575130812880, 22, 12.065827) (1575130813880, 25, 16.967192) (1575130814880, 16, 10.283031) (1575130815880, 22, 16.072535) (1575130816880, 24, 10.794536) (1575130817880, 32, 10.591207) (1575130818880, 20, 13.015227) (1575130819880, 28, 15.410999) (1575130820880, 29, 12.785076) (1575130821880, 28, 15.305857) (1575130822880, 33, 12.820810) (1575130823880, 34, 13.618055) (1575130824880, 32, 12.971123) (1575130825880, 24, 10.974546) (1575130826880, 15, 10.742910) (1575130827880, 23, 16.810783) (1575130828880, 18, 13.115224) (1575130829880, 26, 17.418489) (1575130830880, 20, 17.302315) (1575130831880, 21, 14.283571) (1575130832880, 16, 16.826534) (1575130833880, 18, 19.222122) (1575130834880, 18, 14.931420) (1575130835880, 17, 19.549454) (1575130836880, 22, 16.908388) (1575130837880, 32, 15.637796) (1575130838880, 31, 15.517650) (1575130839880, 18, 14.038033) (1575130840880, 32, 19.859648) (1575130841880, 16, 13.220840) (1575130842880, 28, 16.445398) (1575130843880, 26, 16.695753) (1575130844880, 33, 13.696928) (1575130845880, 21, 15.352819) (1575130846880, 15, 12.388407) (1575130847880, 27, 11.267529) (1575130848880, 20, 14.103228) (1575130849880, 20, 16.250950) (1575130850880, 30, 16.236088) (1575130851880, 22, 18.305340) (1575130852880, 25, 17.360685) (1575130853880, 25, 14.978681) (1575130854880, 33, 14.096183) (1575130855880, 26, 10.019039) (1575130856880, 19, 19.158213) (1575130857880, 22, 15.593924) (1575130858880, 26, 18.780119) (1575130859880, 21, 16.001656) (1575130860880, 16, 18.458328) (1575130861880, 21, 16.417843) (1575130862880, 28, 11.736558) (1575130863880, 34, 18.143946) (1575130864880, 27, 10.303225) (1575130865880, 20, 19.756748) (1575130866880, 22, 12.940063) (1575130867880, 23, 11.509640) (1575130868880, 19, 18.319309) (1575130869880, 19, 16.278345) (1575130870880, 27, 10.898361) (1575130871880, 31, 13.922162) (1575130872880, 15, 19.296116) (1575130873880, 26, 15.885763) (1575130874880, 15, 15.525804) (1575130875880, 19, 19.579538) (1575130876880, 20, 11.073811) (1575130877880, 16, 13.932510) (1575130878880, 17, 11.900328) (1575130879880, 22, 16.540415) (1575130880880, 33, 15.203803) (1575130881880, 17, 11.518434) (1575130882880, 17, 13.152081) (1575130883880, 18, 11.378041) (1575130884880, 21, 15.390745) (1575130885880, 30, 15.127818) (1575130886880, 19, 16.530401) (1575130887880, 32, 16.542702) (1575130888880, 26, 16.366442) (1575130889880, 25, 10.306822) (1575130890880, 15, 13.691117) (1575130891880, 15, 13.476817) (1575130892880, 25, 12.529998) (1575130893880, 22, 15.550021) (1575130894880, 20, 15.064971) (1575130895880, 24, 13.313683) (1575130896880, 23, 17.002878) (1575130897880, 30, 19.991594) (1575130898880, 15, 11.116746) (1575130899870, 16, 19.405090) (1575130900870, 22, 14.377142) (1575130901870, 16, 16.868231) (1575130902870, 20, 11.565193) (1575130903870, 31, 13.009119) (1575130904870, 29, 18.136400) (1575130905870, 17, 13.806572) (1575130906870, 23, 14.688898) (1575130907870, 26, 12.931019) (1575130908870, 32, 12.185531) (1575130909870, 30, 13.608714) (1575130910870, 23, 18.624914) (1575130911870, 22, 12.970826) (1575130912870, 22, 12.065827) (1575130913870, 25, 16.967192) (1575130914870, 16, 10.283031) (1575130915870, 22, 16.072535) (1575130916870, 24, 10.794536) (1575130917870, 32, 10.591207) (1575130918870, 20, 13.015227) (1575130919870, 28, 15.410999) (1575130920870, 29, 12.785076) (1575130921870, 28, 15.305857) (1575130922870, 33, 12.820810) (1575130923870, 34, 13.618055) (1575130924870, 32, 12.971123) (1575130925870, 24, 10.974546) (1575130926870, 15, 10.742910) (1575130927870, 23, 16.810783) (1575130928870, 18, 13.115224) (1575130929870, 26, 17.418489) (1575130930870, 20, 17.302315) (1575130931870, 21, 14.283571) (1575130932870, 16, 16.826534) (1575130933870, 18, 19.222122) (1575130934870, 18, 14.931420) (1575130935870, 17, 19.549454) (1575130936870, 22, 16.908388) (1575130937870, 32, 15.637796) (1575130938870, 31, 15.517650) (1575130939870, 18, 14.038033) (1575130940870, 32, 19.859648) (1575130941870, 16, 13.220840) (1575130942870, 28, 16.445398) (1575130943870, 26, 16.695753) (1575130944870, 33, 13.696928) (1575130945870, 21, 15.352819) (1575130946870, 15, 12.388407) (1575130947870, 27, 11.267529) (1575130948870, 20, 14.103228) (1575130949870, 20, 16.250950) (1575130950870, 30, 16.236088) (1575130951870, 22, 18.305340) (1575130952870, 25, 17.360685) (1575130953870, 25, 14.978681) (1575130954870, 33, 14.096183) (1575130955870, 26, 10.019039) (1575130956870, 19, 19.158213) (1575130957870, 22, 15.593924) (1575130958870, 26, 18.780119) (1575130959870, 21, 16.001656) (1575130960870, 16, 18.458328) (1575130961870, 21, 16.417843) (1575130962870, 28, 11.736558) (1575130963870, 34, 18.143946) (1575130964870, 27, 10.303225) (1575130965870, 20, 19.756748) (1575130966870, 22, 12.940063) (1575130967870, 23, 11.509640) (1575130968870, 19, 18.319309) (1575130969870, 19, 16.278345) (1575130970870, 27, 10.898361) (1575130971870, 31, 13.922162) (1575130972870, 15, 19.296116) (1575130973870, 26, 15.885763) (1575130974870, 15, 15.525804) (1575130975870, 19, 19.579538) (1575130976870, 20, 11.073811) (1575130977870, 16, 13.932510) (1575130978870, 17, 11.900328) (1575130979870, 22, 16.540415) (1575130980870, 33, 15.203803) (1575130981870, 17, 11.518434) (1575130982870, 17, 13.152081) (1575130983870, 18, 11.378041) (1575130984870, 21, 15.390745) (1575130985870, 30, 15.127818) (1575130986870, 19, 16.530401) (1575130987870, 32, 16.542702) (1575130988870, 26, 16.366442) (1575130989870, 25, 10.306822) (1575130990870, 15, 13.691117) (1575130991870, 15, 13.476817) (1575130992870, 25, 12.529998) (1575130993870, 22, 15.550021) (1575130994870, 20, 15.064971) (1575130995870, 24, 13.313683) (1575130996870, 23, 17.002878) (1575130997870, 30, 19.991594) (1575130998870, 15, 11.116746) (1575130999860, 16, 19.405090) (1575131000860, 22, 14.377142) (1575131001860, 16, 16.868231) (1575131002860, 20, 11.565193) (1575131003860, 31, 13.009119) (1575131004860, 29, 18.136400) (1575131005860, 17, 13.806572) (1575131006860, 23, 14.688898) (1575131007860, 26, 12.931019) (1575131008860, 32, 12.185531) (1575131009860, 30, 13.608714) (1575131010860, 23, 18.624914) (1575131011860, 22, 12.970826) (1575131012860, 22, 12.065827) (1575131013860, 25, 16.967192) (1575131014860, 16, 10.283031) (1575131015860, 22, 16.072535) (1575131016860, 24, 10.794536) (1575131017860, 32, 10.591207) (1575131018860, 20, 13.015227) (1575131019860, 28, 15.410999) (1575131020860, 29, 12.785076) (1575131021860, 28, 15.305857) (1575131022860, 33, 12.820810) (1575131023860, 34, 13.618055) (1575131024860, 32, 12.971123) (1575131025860, 24, 10.974546) (1575131026860, 15, 10.742910) (1575131027860, 23, 16.810783) (1575131028860, 18, 13.115224) (1575131029860, 26, 17.418489) (1575131030860, 20, 17.302315) (1575131031860, 21, 14.283571) (1575131032860, 16, 16.826534) (1575131033860, 18, 19.222122) (1575131034860, 18, 14.931420) (1575131035860, 17, 19.549454) (1575131036860, 22, 16.908388) (1575131037860, 32, 15.637796) (1575131038860, 31, 15.517650) (1575131039860, 18, 14.038033) (1575131040860, 32, 19.859648) (1575131041860, 16, 13.220840) (1575131042860, 28, 16.445398) (1575131043860, 26, 16.695753) (1575131044860, 33, 13.696928) (1575131045860, 21, 15.352819) (1575131046860, 15, 12.388407) (1575131047860, 27, 11.267529) (1575131048860, 20, 14.103228) (1575131049860, 20, 16.250950) (1575131050860, 30, 16.236088) (1575131051860, 22, 18.305340) (1575131052860, 25, 17.360685) (1575131053860, 25, 14.978681) (1575131054860, 33, 14.096183) (1575131055860, 26, 10.019039) (1575131056860, 19, 19.158213) (1575131057860, 22, 15.593924) (1575131058860, 26, 18.780119) (1575131059860, 21, 16.001656) (1575131060860, 16, 18.458328) (1575131061860, 21, 16.417843) (1575131062860, 28, 11.736558) (1575131063860, 34, 18.143946) (1575131064860, 27, 10.303225) (1575131065860, 20, 19.756748) (1575131066860, 22, 12.940063) (1575131067860, 23, 11.509640) (1575131068860, 19, 18.319309) (1575131069860, 19, 16.278345) (1575131070860, 27, 10.898361) (1575131071860, 31, 13.922162) (1575131072860, 15, 19.296116) (1575131073860, 26, 15.885763) (1575131074860, 15, 15.525804) (1575131075860, 19, 19.579538) (1575131076860, 20, 11.073811) (1575131077860, 16, 13.932510) (1575131078860, 17, 11.900328) (1575131079860, 22, 16.540415) (1575131080860, 33, 15.203803) (1575131081860, 17, 11.518434) (1575131082860, 17, 13.152081) (1575131083860, 18, 11.378041) (1575131084860, 21, 15.390745) (1575131085860, 30, 15.127818) (1575131086860, 19, 16.530401) (1575131087860, 32, 16.542702) (1575131088860, 26, 16.366442) (1575131089860, 25, 10.306822) (1575131090860, 15, 13.691117) (1575131091860, 15, 13.476817) (1575131092860, 25, 12.529998) (1575131093860, 22, 15.550021) (1575131094860, 20, 15.064971) (1575131095860, 24, 13.313683) (1575131096860, 23, 17.002878) (1575131097860, 30, 19.991594) (1575131098860, 15, 11.116746) (1575131099850, 16, 19.405090) (1575131100850, 22, 14.377142) (1575131101850, 16, 16.868231) (1575131102850, 20, 11.565193) (1575131103850, 31, 13.009119) (1575131104850, 29, 18.136400) (1575131105850, 17, 13.806572) (1575131106850, 23, 14.688898) (1575131107850, 26, 12.931019) (1575131108850, 32, 12.185531) (1575131109850, 30, 13.608714) (1575131110850, 23, 18.624914) (1575131111850, 22, 12.970826) (1575131112850, 22, 12.065827) (1575131113850, 25, 16.967192) (1575131114850, 16, 10.283031) (1575131115850, 22, 16.072535) (1575131116850, 24, 10.794536) (1575131117850, 32, 10.591207) (1575131118850, 20, 13.015227) (1575131119850, 28, 15.410999) (1575131120850, 29, 12.785076) (1575131121850, 28, 15.305857) (1575131122850, 33, 12.820810) (1575131123850, 34, 13.618055) (1575131124850, 32, 12.971123) (1575131125850, 24, 10.974546) (1575131126850, 15, 10.742910) (1575131127850, 23, 16.810783) (1575131128850, 18, 13.115224) (1575131129850, 26, 17.418489) (1575131130850, 20, 17.302315) (1575131131850, 21, 14.283571) (1575131132850, 16, 16.826534) (1575131133850, 18, 19.222122) (1575131134850, 18, 14.931420) (1575131135850, 17, 19.549454) (1575131136850, 22, 16.908388) (1575131137850, 32, 15.637796) (1575131138850, 31, 15.517650) (1575131139850, 18, 14.038033) (1575131140850, 32, 19.859648) (1575131141850, 16, 13.220840) (1575131142850, 28, 16.445398) (1575131143850, 26, 16.695753) (1575131144850, 33, 13.696928) (1575131145850, 21, 15.352819) (1575131146850, 15, 12.388407) (1575131147850, 27, 11.267529) (1575131148850, 20, 14.103228) (1575131149850, 20, 16.250950) (1575131150850, 30, 16.236088) (1575131151850, 22, 18.305340) (1575131152850, 25, 17.360685) (1575131153850, 25, 14.978681) (1575131154850, 33, 14.096183) (1575131155850, 26, 10.019039) (1575131156850, 19, 19.158213) (1575131157850, 22, 15.593924) (1575131158850, 26, 18.780119) (1575131159850, 21, 16.001656) (1575131160850, 16, 18.458328) (1575131161850, 21, 16.417843) (1575131162850, 28, 11.736558) (1575131163850, 34, 18.143946) (1575131164850, 27, 10.303225) (1575131165850, 20, 19.756748) (1575131166850, 22, 12.940063) (1575131167850, 23, 11.509640) (1575131168850, 19, 18.319309) (1575131169850, 19, 16.278345) (1575131170850, 27, 10.898361) (1575131171850, 31, 13.922162) (1575131172850, 15, 19.296116) (1575131173850, 26, 15.885763) (1575131174850, 15, 15.525804) (1575131175850, 19, 19.579538) (1575131176850, 20, 11.073811) (1575131177850, 16, 13.932510) (1575131178850, 17, 11.900328) (1575131179850, 22, 16.540415) (1575131180850, 33, 15.203803) (1575131181850, 17, 11.518434) (1575131182850, 17, 13.152081) (1575131183850, 18, 11.378041) (1575131184850, 21, 15.390745) (1575131185850, 30, 15.127818) (1575131186850, 19, 16.530401) (1575131187850, 32, 16.542702) (1575131188850, 26, 16.366442) (1575131189850, 25, 10.306822) (1575131190850, 15, 13.691117) (1575131191850, 15, 13.476817) (1575131192850, 25, 12.529998) (1575131193850, 22, 15.550021) (1575131194850, 20, 15.064971) (1575131195850, 24, 13.313683) (1575131196850, 23, 17.002878) (1575131197850, 30, 19.991594) (1575131198850, 15, 11.116746) (1575131199840, 16, 19.405090) (1575131200840, 22, 14.377142) (1575131201840, 16, 16.868231) (1575131202840, 20, 11.565193) (1575131203840, 31, 13.009119) (1575131204840, 29, 18.136400) (1575131205840, 17, 13.806572) (1575131206840, 23, 14.688898) (1575131207840, 26, 12.931019) (1575131208840, 32, 12.185531) (1575131209840, 30, 13.608714) (1575131210840, 23, 18.624914) (1575131211840, 22, 12.970826) (1575131212840, 22, 12.065827) (1575131213840, 25, 16.967192) (1575131214840, 16, 10.283031) (1575131215840, 22, 16.072535) (1575131216840, 24, 10.794536) (1575131217840, 32, 10.591207) (1575131218840, 20, 13.015227) (1575131219840, 28, 15.410999) (1575131220840, 29, 12.785076) (1575131221840, 28, 15.305857) (1575131222840, 33, 12.820810) (1575131223840, 34, 13.618055) (1575131224840, 32, 12.971123) (1575131225840, 24, 10.974546) (1575131226840, 15, 10.742910) (1575131227840, 23, 16.810783) (1575131228840, 18, 13.115224) (1575131229840, 26, 17.418489) (1575131230840, 20, 17.302315) (1575131231840, 21, 14.283571) (1575131232840, 16, 16.826534) (1575131233840, 18, 19.222122) (1575131234840, 18, 14.931420) (1575131235840, 17, 19.549454) (1575131236840, 22, 16.908388) (1575131237840, 32, 15.637796) (1575131238840, 31, 15.517650) (1575131239840, 18, 14.038033) (1575131240840, 32, 19.859648) (1575131241840, 16, 13.220840) (1575131242840, 28, 16.445398) (1575131243840, 26, 16.695753) (1575131244840, 33, 13.696928) (1575131245840, 21, 15.352819) (1575131246840, 15, 12.388407) (1575131247840, 27, 11.267529) (1575131248840, 20, 14.103228) (1575131249840, 20, 16.250950) (1575131250840, 30, 16.236088) (1575131251840, 22, 18.305340) (1575131252840, 25, 17.360685) (1575131253840, 25, 14.978681) (1575131254840, 33, 14.096183) (1575131255840, 26, 10.019039) (1575131256840, 19, 19.158213) (1575131257840, 22, 15.593924) (1575131258840, 26, 18.780119) (1575131259840, 21, 16.001656) (1575131260840, 16, 18.458328) (1575131261840, 21, 16.417843) (1575131262840, 28, 11.736558) (1575131263840, 34, 18.143946) (1575131264840, 27, 10.303225) (1575131265840, 20, 19.756748) (1575131266840, 22, 12.940063) (1575131267840, 23, 11.509640) (1575131268840, 19, 18.319309) (1575131269840, 19, 16.278345) (1575131270840, 27, 10.898361) (1575131271840, 31, 13.922162) (1575131272840, 15, 19.296116) (1575131273840, 26, 15.885763) (1575131274840, 15, 15.525804) (1575131275840, 19, 19.579538) (1575131276840, 20, 11.073811) (1575131277840, 16, 13.932510) (1575131278840, 17, 11.900328) (1575131279840, 22, 16.540415) (1575131280840, 33, 15.203803) (1575131281840, 17, 11.518434) (1575131282840, 17, 13.152081) (1575131283840, 18, 11.378041) (1575131284840, 21, 15.390745) (1575131285840, 30, 15.127818) (1575131286840, 19, 16.530401) (1575131287840, 32, 16.542702) (1575131288840, 26, 16.366442) (1575131289840, 25, 10.306822) (1575131290840, 15, 13.691117) (1575131291840, 15, 13.476817) (1575131292840, 25, 12.529998) (1575131293840, 22, 15.550021) (1575131294840, 20, 15.064971) (1575131295840, 24, 13.313683) (1575131296840, 23, 17.002878) (1575131297840, 30, 19.991594) (1575131298840, 15, 11.116746) (1575131299830, 16, 19.405090) (1575131300830, 22, 14.377142) (1575131301830, 16, 16.868231) (1575131302830, 20, 11.565193) (1575131303830, 31, 13.009119) (1575131304830, 29, 18.136400) (1575131305830, 17, 13.806572) (1575131306830, 23, 14.688898) (1575131307830, 26, 12.931019) (1575131308830, 32, 12.185531) (1575131309830, 30, 13.608714) (1575131310830, 23, 18.624914) (1575131311830, 22, 12.970826) (1575131312830, 22, 12.065827) (1575131313830, 25, 16.967192) (1575131314830, 16, 10.283031) (1575131315830, 22, 16.072535) (1575131316830, 24, 10.794536) (1575131317830, 32, 10.591207) (1575131318830, 20, 13.015227) (1575131319830, 28, 15.410999) (1575131320830, 29, 12.785076) (1575131321830, 28, 15.305857) (1575131322830, 33, 12.820810) (1575131323830, 34, 13.618055) (1575131324830, 32, 12.971123) (1575131325830, 24, 10.974546) (1575131326830, 15, 10.742910) (1575131327830, 23, 16.810783) (1575131328830, 18, 13.115224) (1575131329830, 26, 17.418489) (1575131330830, 20, 17.302315) (1575131331830, 21, 14.283571) (1575131332830, 16, 16.826534) (1575131333830, 18, 19.222122) (1575131334830, 18, 14.931420) (1575131335830, 17, 19.549454) (1575131336830, 22, 16.908388) (1575131337830, 32, 15.637796) (1575131338830, 31, 15.517650) (1575131339830, 18, 14.038033) (1575131340830, 32, 19.859648) (1575131341830, 16, 13.220840) (1575131342830, 28, 16.445398) (1575131343830, 26, 16.695753) (1575131344830, 33, 13.696928) (1575131345830, 21, 15.352819) (1575131346830, 15, 12.388407) (1575131347830, 27, 11.267529) (1575131348830, 20, 14.103228) (1575131349830, 20, 16.250950) (1575131350830, 30, 16.236088) (1575131351830, 22, 18.305340) (1575131352830, 25, 17.360685) (1575131353830, 25, 14.978681) (1575131354830, 33, 14.096183) (1575131355830, 26, 10.019039) (1575131356830, 19, 19.158213) (1575131357830, 22, 15.593924) (1575131358830, 26, 18.780119) (1575131359830, 21, 16.001656) (1575131360830, 16, 18.458328) (1575131361830, 21, 16.417843) (1575131362830, 28, 11.736558) (1575131363830, 34, 18.143946) (1575131364830, 27, 10.303225) (1575131365830, 20, 19.756748) (1575131366830, 22, 12.940063) (1575131367830, 23, 11.509640) (1575131368830, 19, 18.319309) (1575131369830, 19, 16.278345) (1575131370830, 27, 10.898361) (1575131371830, 31, 13.922162) (1575131372830, 15, 19.296116) (1575131373830, 26, 15.885763) (1575131374830, 15, 15.525804) (1575131375830, 19, 19.579538) (1575131376830, 20, 11.073811) (1575131377830, 16, 13.932510) (1575131378830, 17, 11.900328) (1575131379830, 22, 16.540415) (1575131380830, 33, 15.203803) (1575131381830, 17, 11.518434) (1575131382830, 17, 13.152081) (1575131383830, 18, 11.378041) (1575131384830, 21, 15.390745) (1575131385830, 30, 15.127818) (1575131386830, 19, 16.530401) (1575131387830, 32, 16.542702) (1575131388830, 26, 16.366442) (1575131389830, 25, 10.306822) (1575131390830, 15, 13.691117) (1575131391830, 15, 13.476817) (1575131392830, 25, 12.529998) (1575131393830, 22, 15.550021) (1575131394830, 20, 15.064971) (1575131395830, 24, 13.313683) (1575131396830, 23, 17.002878) (1575131397830, 30, 19.991594) (1575131398830, 15, 11.116746) (1575131399820, 16, 19.405090) (1575131400820, 22, 14.377142) (1575131401820, 16, 16.868231) (1575131402820, 20, 11.565193) (1575131403820, 31, 13.009119) (1575131404820, 29, 18.136400) (1575131405820, 17, 13.806572) (1575131406820, 23, 14.688898) (1575131407820, 26, 12.931019) (1575131408820, 32, 12.185531) (1575131409820, 30, 13.608714) (1575131410820, 23, 18.624914) (1575131411820, 22, 12.970826) (1575131412820, 22, 12.065827) (1575131413820, 25, 16.967192) (1575131414820, 16, 10.283031) (1575131415820, 22, 16.072535) (1575131416820, 24, 10.794536) (1575131417820, 32, 10.591207) (1575131418820, 20, 13.015227) (1575131419820, 28, 15.410999) (1575131420820, 29, 12.785076) (1575131421820, 28, 15.305857) (1575131422820, 33, 12.820810) (1575131423820, 34, 13.618055) (1575131424820, 32, 12.971123) (1575131425820, 24, 10.974546) (1575131426820, 15, 10.742910) (1575131427820, 23, 16.810783) (1575131428820, 18, 13.115224) (1575131429820, 26, 17.418489) (1575131430820, 20, 17.302315) (1575131431820, 21, 14.283571) (1575131432820, 16, 16.826534) (1575131433820, 18, 19.222122) (1575131434820, 18, 14.931420) (1575131435820, 17, 19.549454) (1575131436820, 22, 16.908388) (1575131437820, 32, 15.637796) (1575131438820, 31, 15.517650) (1575131439820, 18, 14.038033) (1575131440820, 32, 19.859648) (1575131441820, 16, 13.220840) (1575131442820, 28, 16.445398) (1575131443820, 26, 16.695753) (1575131444820, 33, 13.696928) (1575131445820, 21, 15.352819) (1575131446820, 15, 12.388407) (1575131447820, 27, 11.267529) (1575131448820, 20, 14.103228) (1575131449820, 20, 16.250950) (1575131450820, 30, 16.236088) (1575131451820, 22, 18.305340) (1575131452820, 25, 17.360685) (1575131453820, 25, 14.978681) (1575131454820, 33, 14.096183) (1575131455820, 26, 10.019039) (1575131456820, 19, 19.158213) (1575131457820, 22, 15.593924) (1575131458820, 26, 18.780119) (1575131459820, 21, 16.001656) (1575131460820, 16, 18.458328) (1575131461820, 21, 16.417843) (1575131462820, 28, 11.736558) (1575131463820, 34, 18.143946) (1575131464820, 27, 10.303225) (1575131465820, 20, 19.756748) (1575131466820, 22, 12.940063) (1575131467820, 23, 11.509640) (1575131468820, 19, 18.319309) (1575131469820, 19, 16.278345) (1575131470820, 27, 10.898361) (1575131471820, 31, 13.922162) (1575131472820, 15, 19.296116) (1575131473820, 26, 15.885763) (1575131474820, 15, 15.525804) (1575131475820, 19, 19.579538) (1575131476820, 20, 11.073811) (1575131477820, 16, 13.932510) (1575131478820, 17, 11.900328) (1575131479820, 22, 16.540415) (1575131480820, 33, 15.203803) (1575131481820, 17, 11.518434) (1575131482820, 17, 13.152081) (1575131483820, 18, 11.378041) (1575131484820, 21, 15.390745) (1575131485820, 30, 15.127818) (1575131486820, 19, 16.530401) (1575131487820, 32, 16.542702) (1575131488820, 26, 16.366442) (1575131489820, 25, 10.306822) (1575131490820, 15, 13.691117) (1575131491820, 15, 13.476817) (1575131492820, 25, 12.529998) (1575131493820, 22, 15.550021) (1575131494820, 20, 15.064971) (1575131495820, 24, 13.313683) (1575131496820, 23, 17.002878) (1575131497820, 30, 19.991594) (1575131498820, 15, 11.116746) (1575131499810, 16, 19.405090) (1575131500810, 22, 14.377142) (1575131501810, 16, 16.868231) (1575131502810, 20, 11.565193) (1575131503810, 31, 13.009119) (1575131504810, 29, 18.136400) (1575131505810, 17, 13.806572) (1575131506810, 23, 14.688898) (1575131507810, 26, 12.931019) (1575131508810, 32, 12.185531) (1575131509810, 30, 13.608714) (1575131510810, 23, 18.624914) (1575131511810, 22, 12.970826) (1575131512810, 22, 12.065827) (1575131513810, 25, 16.967192) (1575131514810, 16, 10.283031) (1575131515810, 22, 16.072535) (1575131516810, 24, 10.794536) (1575131517810, 32, 10.591207) (1575131518810, 20, 13.015227) (1575131519810, 28, 15.410999) (1575131520810, 29, 12.785076) (1575131521810, 28, 15.305857) (1575131522810, 33, 12.820810) (1575131523810, 34, 13.618055) (1575131524810, 32, 12.971123) (1575131525810, 24, 10.974546) (1575131526810, 15, 10.742910) (1575131527810, 23, 16.810783) (1575131528810, 18, 13.115224) (1575131529810, 26, 17.418489) (1575131530810, 20, 17.302315) (1575131531810, 21, 14.283571) (1575131532810, 16, 16.826534) (1575131533810, 18, 19.222122) (1575131534810, 18, 14.931420) (1575131535810, 17, 19.549454) (1575131536810, 22, 16.908388) (1575131537810, 32, 15.637796) (1575131538810, 31, 15.517650) (1575131539810, 18, 14.038033) (1575131540810, 32, 19.859648) (1575131541810, 16, 13.220840) (1575131542810, 28, 16.445398) (1575131543810, 26, 16.695753) (1575131544810, 33, 13.696928) (1575131545810, 21, 15.352819) (1575131546810, 15, 12.388407) (1575131547810, 27, 11.267529) (1575131548810, 20, 14.103228) (1575131549810, 20, 16.250950) (1575131550810, 30, 16.236088) (1575131551810, 22, 18.305340) (1575131552810, 25, 17.360685) (1575131553810, 25, 14.978681) (1575131554810, 33, 14.096183) (1575131555810, 26, 10.019039) (1575131556810, 19, 19.158213) (1575131557810, 22, 15.593924) (1575131558810, 26, 18.780119) (1575131559810, 21, 16.001656) (1575131560810, 16, 18.458328) (1575131561810, 21, 16.417843) (1575131562810, 28, 11.736558) (1575131563810, 34, 18.143946) (1575131564810, 27, 10.303225) (1575131565810, 20, 19.756748) (1575131566810, 22, 12.940063) (1575131567810, 23, 11.509640) (1575131568810, 19, 18.319309) (1575131569810, 19, 16.278345) (1575131570810, 27, 10.898361) (1575131571810, 31, 13.922162) (1575131572810, 15, 19.296116) (1575131573810, 26, 15.885763) (1575131574810, 15, 15.525804) (1575131575810, 19, 19.579538) (1575131576810, 20, 11.073811) (1575131577810, 16, 13.932510) (1575131578810, 17, 11.900328) (1575131579810, 22, 16.540415) (1575131580810, 33, 15.203803) (1575131581810, 17, 11.518434) (1575131582810, 17, 13.152081) (1575131583810, 18, 11.378041) (1575131584810, 21, 15.390745) (1575131585810, 30, 15.127818) (1575131586810, 19, 16.530401) (1575131587810, 32, 16.542702) (1575131588810, 26, 16.366442) (1575131589810, 25, 10.306822) (1575131590810, 15, 13.691117) (1575131591810, 15, 13.476817) (1575131592810, 25, 12.529998) (1575131593810, 22, 15.550021) (1575131594810, 20, 15.064971) (1575131595810, 24, 13.313683) (1575131596810, 23, 17.002878) (1575131597810, 30, 19.991594) (1575131598810, 15, 11.116746) (1575131599800, 16, 19.405090) (1575131600800, 22, 14.377142) (1575131601800, 16, 16.868231) (1575131602800, 20, 11.565193) (1575131603800, 31, 13.009119) (1575131604800, 29, 18.136400) (1575131605800, 17, 13.806572) (1575131606800, 23, 14.688898) (1575131607800, 26, 12.931019) (1575131608800, 32, 12.185531) (1575131609800, 30, 13.608714) (1575131610800, 23, 18.624914) (1575131611800, 22, 12.970826) (1575131612800, 22, 12.065827) (1575131613800, 25, 16.967192) (1575131614800, 16, 10.283031) (1575131615800, 22, 16.072535) (1575131616800, 24, 10.794536) (1575131617800, 32, 10.591207) (1575131618800, 20, 13.015227) (1575131619800, 28, 15.410999) (1575131620800, 29, 12.785076) (1575131621800, 28, 15.305857) (1575131622800, 33, 12.820810) (1575131623800, 34, 13.618055) (1575131624800, 32, 12.971123) (1575131625800, 24, 10.974546) (1575131626800, 15, 10.742910) (1575131627800, 23, 16.810783) (1575131628800, 18, 13.115224) (1575131629800, 26, 17.418489) (1575131630800, 20, 17.302315) (1575131631800, 21, 14.283571) (1575131632800, 16, 16.826534) (1575131633800, 18, 19.222122) (1575131634800, 18, 14.931420) (1575131635800, 17, 19.549454) (1575131636800, 22, 16.908388) (1575131637800, 32, 15.637796) (1575131638800, 31, 15.517650) (1575131639800, 18, 14.038033) (1575131640800, 32, 19.859648) (1575131641800, 16, 13.220840) (1575131642800, 28, 16.445398) (1575131643800, 26, 16.695753) (1575131644800, 33, 13.696928) (1575131645800, 21, 15.352819) (1575131646800, 15, 12.388407) (1575131647800, 27, 11.267529) (1575131648800, 20, 14.103228) (1575131649800, 20, 16.250950) (1575131650800, 30, 16.236088) (1575131651800, 22, 18.305340) (1575131652800, 25, 17.360685) (1575131653800, 25, 14.978681) (1575131654800, 33, 14.096183) (1575131655800, 26, 10.019039) (1575131656800, 19, 19.158213) (1575131657800, 22, 15.593924) (1575131658800, 26, 18.780119) (1575131659800, 21, 16.001656) (1575131660800, 16, 18.458328) (1575131661800, 21, 16.417843) (1575131662800, 28, 11.736558) (1575131663800, 34, 18.143946) (1575131664800, 27, 10.303225) (1575131665800, 20, 19.756748) (1575131666800, 22, 12.940063) (1575131667800, 23, 11.509640) (1575131668800, 19, 18.319309) (1575131669800, 19, 16.278345) (1575131670800, 27, 10.898361) (1575131671800, 31, 13.922162) (1575131672800, 15, 19.296116) (1575131673800, 26, 15.885763) (1575131674800, 15, 15.525804) (1575131675800, 19, 19.579538) (1575131676800, 20, 11.073811) (1575131677800, 16, 13.932510) (1575131678800, 17, 11.900328) (1575131679800, 22, 16.540415) (1575131680800, 33, 15.203803) (1575131681800, 17, 11.518434) (1575131682800, 17, 13.152081) (1575131683800, 18, 11.378041) (1575131684800, 21, 15.390745) (1575131685800, 30, 15.127818) (1575131686800, 19, 16.530401) (1575131687800, 32, 16.542702) (1575131688800, 26, 16.366442) (1575131689800, 25, 10.306822) (1575131690800, 15, 13.691117) (1575131691800, 15, 13.476817) (1575131692800, 25, 12.529998) (1575131693800, 22, 15.550021)") + + end_time = int(round(time.time() * 1000)) + tdLog.info("Execute time without compress: %dms" % (end_time - start_time)) + + simLogPath = tdDnodes.getSimLogPath() + grepCmd = "grep -a 'compress rpc msg, before:' -r %s | head -2" % simLogPath + output = subprocess.check_output(grepCmd, shell=True).decode("utf-8") + + if output != "": + tdLog.info("Find %s in log file." % output) + tdLog.exit("%s successfully executed! Compress works but NOT expected." % __file__) + else: + tdLog.success("%s failed! Compress does NOT works as expected." % __file__) + + conn.close() diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py new file mode 100644 index 0000000000..db3a5fea93 --- /dev/null +++ b/tests/pytest/util/dnodes-random-fail.py @@ -0,0 +1,500 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import os.path +import subprocess +from util.log import * + + +class TDSimClient: + def __init__(self): + self.testCluster = False + + self.cfgDict = { + "numOfLogLines": "100000000", + "numOfThreadsPerCore": "2.0", + "locale": "en_US.UTF-8", + "charset": "UTF-8", + "asyncLog": "0", + "anyIp": "0", + "sdbDebugFlag": "135", + "rpcDebugFlag": "135", + "tmrDebugFlag": "131", + "cDebugFlag": "135", + "udebugFlag": "135", + "jnidebugFlag": "135", + "qdebugFlag": "135", + } + + def init(self, path): + self.__init__() + self.path = path + + def getLogDir(self): + self.logDir = "%s/sim/psim/log" % (self.path) + return self.logDir + + def getCfgDir(self): + self.cfgDir = "%s/sim/psim/cfg" % (self.path) + return self.cfgDir + + def setTestCluster(self, value): + self.testCluster = value + + def addExtraCfg(self, option, value): + self.cfgDict.update({option: value}) + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def deploy(self): + self.logDir = "%s/sim/psim/log" % (self.path) + self.cfgDir = "%s/sim/psim/cfg" % (self.path) + self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("logDir", self.logDir) + + for key, value in self.cfgDict.items(): + self.cfg(key, value) + + tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) + + +class TDDnode: + def __init__(self, index): + self.index = index + self.running = 0 + self.deployed = 0 + self.testCluster = False + self.valgrind = 0 + + def init(self, path): + self.path = path + + def setTestCluster(self, value): + self.testCluster = value + + def setValgrind(self, value): + self.valgrind = value + + def getDataSize(self): + totalSize = 0 + + if (self.deployed == 1): + for dirpath, dirnames, filenames in os.walk(self.dataDir): + for f in filenames: + fp = os.path.join(dirpath, f) + + if not os.path.islink(fp): + totalSize = totalSize + os.path.getsize(fp) + + return totalSize + + def deploy(self): + self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) + self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) + self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) + self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % ( + self.path, self.index) + + cmd = "rm -rf " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + if self.testCluster: + self.startIP() + + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("publicIp", "192.168.0.%d" % (self.index)) + self.cfg("internalIp", "192.168.0.%d" % (self.index)) + self.cfg("privateIp", "192.168.0.%d" % (self.index)) + self.cfg("dataDir", self.dataDir) + self.cfg("logDir", self.logDir) + self.cfg("numOfLogLines", "100000000") + self.cfg("mnodeEqualVnodeNum", "0") + self.cfg("walLevel", "1") + self.cfg("statusInterval", "1") + self.cfg("numOfTotalVnodes", "64") + self.cfg("numOfMnodes", "3") + self.cfg("numOfThreadsPerCore", "2.0") + self.cfg("monitor", "0") + self.cfg("maxVnodeConnections", "30000") + self.cfg("maxMgmtConnections", "30000") + self.cfg("maxMeterConnections", "30000") + self.cfg("maxShellConns", "30000") + self.cfg("locale", "en_US.UTF-8") + self.cfg("charset", "UTF-8") + self.cfg("asyncLog", "0") + self.cfg("anyIp", "0") + self.cfg("dDebugFlag", "135") + self.cfg("mDebugFlag", "135") + self.cfg("sdbDebugFlag", "135") + self.cfg("rpcDebugFlag", "135") + self.cfg("tmrDebugFlag", "131") + self.cfg("cDebugFlag", "135") + self.cfg("httpDebugFlag", "135") + self.cfg("monitorDebugFlag", "135") + self.cfg("udebugFlag", "135") + self.cfg("jnidebugFlag", "135") + self.cfg("qdebugFlag", "135") + self.deployed = 1 + tdLog.debug( + "dnode:%d is deployed and configured by %s" % + (self.index, self.cfgPath)) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def start(self): + buildPath = self.getBuildPath() + + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + binPath = buildPath + "/build/bin/taosd" + + if self.deployed == 0: + tdLog.exit("dnode:%d is not deployed" % (self.index)) + + if self.valgrind == 0: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) + else: + valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + + cmd = "nohup %s %s -c %s --random-file-fail-factor 5 2>&1 & " % ( + valgrindCmdline, binPath, self.cfgDir) + + print(cmd) + + if os.system(cmd) != 0: + tdLog.exit(cmd) + self.running = 1 + tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) + + tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) + time.sleep(5) + + def stop(self): + if self.valgrind == 0: + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + if self.running != 0: + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + while(processID): + killCmd = "kill -INT %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) + + self.running = 0 + tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) + + def forcestop(self): + if self.valgrind == 0: + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + if self.running != 0: + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) + + self.running = 0 + tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) + + def startIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def stopIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( + self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def getDnodeRootDir(self, index): + dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + return dnodeRootDir + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim/psim" % (self.path) + return dnodesRootDir + + +class TDDnodes: + def __init__(self): + self.dnodes = [] + self.dnodes.append(TDDnode(1)) + self.dnodes.append(TDDnode(2)) + self.dnodes.append(TDDnode(3)) + self.dnodes.append(TDDnode(4)) + self.dnodes.append(TDDnode(5)) + self.dnodes.append(TDDnode(6)) + self.dnodes.append(TDDnode(7)) + self.dnodes.append(TDDnode(8)) + self.dnodes.append(TDDnode(9)) + self.dnodes.append(TDDnode(10)) + self.simDeployed = False + + def init(self, path): + psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug("binPath %s" % (binPath)) + binPath = os.path.realpath(binPath) + tdLog.debug("binPath real path %s" % (binPath)) + + # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) + # tdLog.debug(cmd) + # os.system(cmd) + + # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + if path == "": + # self.path = os.path.expanduser('~') + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + + for i in range(len(self.dnodes)): + self.dnodes[i].init(self.path) + + self.sim = TDSimClient() + self.sim.init(self.path) + + def setTestCluster(self, value): + self.testCluster = value + + def setValgrind(self, value): + self.valgrind = value + + def deploy(self, index): + self.sim.setTestCluster(self.testCluster) + + if (self.simDeployed == False): + self.sim.deploy() + self.simDeployed = True + + self.check(index) + self.dnodes[index - 1].setTestCluster(self.testCluster) + self.dnodes[index - 1].setValgrind(self.valgrind) + self.dnodes[index - 1].deploy() + + def cfg(self, index, option, value): + self.check(index) + self.dnodes[index - 1].cfg(option, value) + + def start(self, index): + self.check(index) + self.dnodes[index - 1].start() + + def stop(self, index): + self.check(index) + self.dnodes[index - 1].stop() + + def getDataSize(self, index): + self.check(index) + return self.dnodes[index - 1].getDataSize() + + def forcestop(self, index): + self.check(index) + self.dnodes[index - 1].forcestop() + + def startIP(self, index): + self.check(index) + + if self.testCluster: + self.dnodes[index - 1].startIP() + + def stopIP(self, index): + self.check(index) + + if self.dnodes[index - 1].testCluster: + self.dnodes[index - 1].stopIP() + + def check(self, index): + if index < 1 or index > 10: + tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) + + def stopAll(self): + tdLog.info("stop all dnodes") + for i in range(len(self.dnodes)): + self.dnodes[i].stop() + + psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + if processID: + cmd = "sudo systemctl stop taosd" + os.system(cmd) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim" % (self.path) + return dnodesRootDir + + def getSimCfgPath(self): + return self.sim.getCfgDir() + + def getSimLogPath(self): + return self.sim.getLogDir() + + def addSimExtraCfg(self, option, value): + self.sim.addExtraCfg(option, value) + + +tdDnodes = TDDnodes() diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 45d31b001e..0341b3f136 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -22,35 +22,59 @@ class TDSimClient: def __init__(self): self.testCluster = False + self.cfgDict = { + "numOfLogLines": "100000000", + "numOfThreadsPerCore": "2.0", + "locale": "en_US.UTF-8", + "charset": "UTF-8", + "asyncLog": "0", + "maxTablesPerVnode": "4", + "maxVgroupsPerDb": "1000", + "sdbDebugFlag": "143", + "rpcDebugFlag": "135", + "tmrDebugFlag": "131", + "cDebugFlag": "135", + "udebugFlag": "135", + "jnidebugFlag": "135", + "qdebugFlag": "135", + } def init(self, path): self.__init__() self.path = path + def getLogDir(self): + self.logDir = "%s/sim/psim/log" % (self.path) + return self.logDir + def getCfgDir(self): + self.cfgDir = "%s/sim/psim/cfg" % (self.path) return self.cfgDir def setTestCluster(self, value): self.testCluster = value + def addExtraCfg(self, option, value): + self.cfgDict.update({option: value}) + def cfg(self, option, value): cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) def deploy(self): - self.logDir = "%s/sim/psim/log" % (self.path,) + self.logDir = "%s/sim/psim/log" % (self.path) self.cfgDir = "%s/sim/psim/cfg" % (self.path) self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir + + cmd = "mkdir -p " + self.logDir if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir + cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) @@ -66,21 +90,10 @@ class TDSimClient: self.cfg("masterIp", "192.168.0.1") self.cfg("secondIp", "192.168.0.2") self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("maxVgroupsPerDb", "4") - self.cfg("maxTablesPerVnode", "1000") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") + + for key, value in self.cfgDict.items(): + self.cfg(key, value) + tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) @@ -380,6 +393,9 @@ class TDDnodes: for i in range(len(self.dnodes)): self.dnodes[i].init(self.path) + self.sim = TDSimClient() + self.sim.init(self.path) + def setTestCluster(self, value): self.testCluster = value @@ -387,8 +403,6 @@ class TDDnodes: self.valgrind = value def deploy(self, index): - self.sim = TDSimClient() - self.sim.init(self.path) self.sim.setTestCluster(self.testCluster) if (self.simDeployed == False): @@ -476,5 +490,11 @@ class TDDnodes: def getSimCfgPath(self): return self.sim.getCfgDir() + def getSimLogPath(self): + return self.sim.getLogDir() + + def addSimExtraCfg(self, option, value): + self.sim.addExtraCfg(option, value) + tdDnodes = TDDnodes() diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 96ea036adc..ec39ab61b9 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -71,7 +71,8 @@ class TDSql: except Exception as e: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.exit("%s(%d) failed: sql:%s, %s" % args) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) return self.queryRows def waitedQuery(self, sql, expectRows, timeout): @@ -89,7 +90,8 @@ class TDSql: except Exception as e: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.exit("%s(%d) failed: sql:%s, %s" % args) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) return (self.queryRows, timeout) def checkRows(self, expectRows): @@ -158,7 +160,8 @@ class TDSql: except Exception as e: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.exit("%s(%d) failed: sql:%s, %s" % args) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) return self.affectedRows def checkAffectedRows(self, expectAffectedRows): diff --git a/tests/script/general/parser/fill_us.sim b/tests/script/general/parser/fill_us.sim index 190b565b42..a66629c90b 100644 --- a/tests/script/general/parser/fill_us.sim +++ b/tests/script/general/parser/fill_us.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh -system sh/ip.sh -i 1 -s up -system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 31acaad143..26a6a39815 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,3 +1,4 @@ +sleep 2000 run general/parser/alter.sim sleep 2000 run general/parser/alter1.sim @@ -7,7 +8,6 @@ sleep 2000 run general/parser/auto_create_tb.sim sleep 2000 run general/parser/auto_create_tb_drop_tb.sim - sleep 2000 run general/parser/col_arithmetic_operation.sim sleep 2000 @@ -23,77 +23,81 @@ run general/parser/create_tb.sim sleep 2000 run general/parser/dbtbnameValidate.sim sleep 2000 +run general/parser/fill.sim +sleep 2000 +run general/parser/fill_stb.sim +sleep 2000 +#run general/parser/fill_us.sim # +sleep 2000 +run general/parser/first_last.sim +sleep 2000 run general/parser/import_commit1.sim sleep 2000 run general/parser/import_commit2.sim sleep 2000 run general/parser/import_commit3.sim sleep 2000 -run general/parser/insert_tb.sim -sleep 2000 -run general/parser/first_last.sim -sleep 2000 #run general/parser/import_file.sim sleep 2000 -run general/parser/lastrow.sim -sleep 2000 -run general/parser/nchar.sim -sleep 2000 -#run general/parser/null_char.sim -sleep 2000 -run general/parser/single_row_in_tb.sim -sleep 2000 -run general/parser/select_from_cache_disk.sim -sleep 2000 -run general/parser/selectResNum.sim -sleep 2000 -run general/parser/mixed_blocks.sim -sleep 2000 -run general/parser/limit1.sim -sleep 2000 -run general/parser/limit.sim -sleep 2000 -run general/parser/limit1_tblocks100.sim -sleep 2000 -run general/parser/select_across_vnodes.sim -sleep 2000 -run general/parser/slimit1.sim -sleep 2000 -run general/parser/tbnameIn.sim -sleep 2000 -run general/parser/projection_limit_offset.sim -sleep 2000 -run general/parser/limit2.sim -sleep 2000 -run general/parser/fill.sim -sleep 2000 -run general/parser/fill_stb.sim -sleep 2000 -run general/parser/where.sim -sleep 2000 -run general/parser/slimit.sim -sleep 2000 -run general/parser/select_with_tags.sim -sleep 2000 -run general/parser/interp.sim +run general/parser/insert_tb.sim sleep 2000 run general/parser/tags_dynamically_specifiy.sim sleep 2000 -run general/parser/groupby.sim +run general/parser/interp.sim +sleep 2000 +run general/parser/lastrow.sim +sleep 2000 +run general/parser/limit.sim +sleep 2000 +run general/parser/limit1.sim +sleep 2000 +run general/parser/limit1_tblocks100.sim +sleep 2000 +run general/parser/limit2.sim +sleep 2000 +run general/parser/mixed_blocks.sim +sleep 2000 +run general/parser/nchar.sim +sleep 2000 +run general/parser/null_char.sim +sleep 2000 +run general/parser/selectResNum.sim +sleep 2000 +run general/parser/select_across_vnodes.sim +sleep 2000 +run general/parser/select_from_cache_disk.sim sleep 2000 run general/parser/set_tag_vals.sim sleep 2000 +run general/parser/single_row_in_tb.sim +sleep 2000 +run general/parser/slimit.sim +sleep 2000 +run general/parser/slimit1.sim +sleep 2000 +run general/parser/slimit_alter_tags.sim +sleep 2000 +run general/parser/tbnameIn.sim +sleep 2000 run general/parser/slimit_alter_tags.sim # persistent failed sleep 2000 run general/parser/join.sim sleep 2000 run general/parser/join_multivnode.sim sleep 2000 -run general/parser/repeatAlter.sim +run general/parser/projection_limit_offset.sim sleep 2000 -run general/parser/binary_escapeCharacter.sim +run general/parser/select_with_tags.sim sleep 2000 -run general/parser/bug.sim +run general/parser/groupby.sim +sleep 2000 +run general/parser/union.sim +sleep 2000 +run general/parser/sliding.sim +sleep 2000 +run general/parser/fill_us.sim +sleep 2000 +run general/parser/tags_filter.sim #sleep 2000 #run general/parser/repeatStream.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index cb07093a89..99af6cf676 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -117,8 +117,6 @@ cd ../../../debug; make ./test.sh -f general/parser/import_commit3.sim ./test.sh -f general/parser/insert_tb.sim ./test.sh -f general/parser/first_last.sim -# dyh is processing this script -#./test.sh -f general/parser/import_file.sim ./test.sh -f general/parser/lastrow.sim ./test.sh -f general/parser/nchar.sim ./test.sh -f general/parser/null_char.sim @@ -145,7 +143,6 @@ cd ../../../debug; make ./test.sh -f general/parser/groupby.sim ./test.sh -f general/parser/set_tag_vals.sim #./test.sh -f general/parser/sliding.sim -./test.sh -f general/parser/tags_dynamically_specifiy.sim ./test.sh -f general/parser/tags_filter.sim ./test.sh -f general/parser/slimit_alter_tags.sim ./test.sh -f general/parser/join.sim diff --git a/tests/script/sh/exec-random-fail.sh b/tests/script/sh/exec-random-fail.sh new file mode 100755 index 0000000000..7ba301617c --- /dev/null +++ b/tests/script/sh/exec-random-fail.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# if [ $# != 4 || $# != 5 ]; then + # echo "argument list need input : " + # echo " -n nodeName" + # echo " -s start/stop" + # echo " -c clear" + # exit 1 +# fi + +NODE_NAME= +EXEC_OPTON= +CLEAR_OPTION="false" +while getopts "n:s:u:x:ct" arg +do + case $arg in + n) + NODE_NAME=$OPTARG + ;; + s) + EXEC_OPTON=$OPTARG + ;; + c) + CLEAR_OPTION="clear" + ;; + t) + SHELL_OPTION="true" + ;; + u) + USERS=$OPTARG + ;; + x) + SIGNAL=$OPTARG + ;; + ?) + echo "unkown argument" + ;; + esac +done + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` +fi + +BUILD_DIR=$TAOS_DIR/$BIN_DIR/build + +SIM_DIR=$TAOS_DIR/sim +NODE_DIR=$SIM_DIR/$NODE_NAME +EXE_DIR=$BUILD_DIR/bin +CFG_DIR=$NODE_DIR/cfg +LOG_DIR=$NODE_DIR/log +DATA_DIR=$NODE_DIR/data +MGMT_DIR=$NODE_DIR/data/mgmt +TSDB_DIR=$NODE_DIR/data/tsdb + +TAOS_CFG=$NODE_DIR/cfg/taos.cfg + +echo ------------ $EXEC_OPTON $NODE_NAME + +TAOS_FLAG=$SIM_DIR/tsim/flag +if [ -f "$TAOS_FLAG" ]; then + EXE_DIR=/usr/local/bin/taos +fi + +if [ "$CLEAR_OPTION" = "clear" ]; then + echo rm -rf $MGMT_DIR $TSDB_DIR + rm -rf $TSDB_DIR + rm -rf $MGMT_DIR +fi + +if [ "$EXEC_OPTON" = "start" ]; then + echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR + + if [ "$SHELL_OPTION" = "true" ]; then + nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + else + nohup $EXE_DIR/taosd -c $CFG_DIR --random-file-fail-factor 5 > /dev/null 2>&1 & + fi + +else + #relative path + RCFG_DIR=sim/$NODE_NAME/cfg + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + if [ "$SIGNAL" = "SIGINT" ]; then + echo try to kill by signal SIGINT + kill -SIGINT $PID + else + echo try to kill by signal SIGKILL + kill -9 $PID + fi + sleep 1 + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + done +fi + diff --git a/tests/script/unique/cluster/client1_0.sim b/tests/script/unique/cluster/client1_0.sim index 6516fe58e4..184f0263e3 100644 --- a/tests/script/unique/cluster/client1_0.sim +++ b/tests/script/unique/cluster/client1_0.sim @@ -1,5 +1,21 @@ +#system sh/stop_dnodes.sh +#system sh/deploy.sh -n dnode1 -i 1 +#system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 10000 +#system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256 +#system sh/exec.sh -n dnode1 -s start +#sql connect +#$db = db1 +#sql create database $db +#sql use $db +#$stb = stb1 +#sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) + + + + + $tblStart = 0 -$tblEnd = 10000 +$tblEnd = 1000 $tsStart = 1325347200000 # 2012-01-01 00:00:00.000 ############################################################### @@ -10,7 +26,6 @@ $stb = stb1 sql use $db - ######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) $tagPrex = ' . tag @@ -19,12 +34,15 @@ while $i < $tblEnd $tb = tb . $i $tagBinary = $tagPrex . $i $tagBinary = $tagBinary . ' - sql create table if not exists $tb using $stb tags ($i, $tagBinary) +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) $i = $i + 1 endw - print ====================== client1_0 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 $rowsPerLoop = 100 $ts = $tsStart @@ -54,3 +72,4 @@ while $i < $tblEnd print ====================== client1_0 insert data complete once ............ endi endw +print ====================== client1_0 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/cluster_main.sim b/tests/script/unique/cluster/cluster_main.sim index 21800da2d4..236f1aa59a 100644 --- a/tests/script/unique/cluster/cluster_main.sim +++ b/tests/script/unique/cluster/cluster_main.sim @@ -3,41 +3,49 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode3 -c walLevel -v 1 system sh/cfg.sh -n dnode4 -c walLevel -v 1 +system sh/cfg.sh -n dnode5 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 256 system sh/cfg.sh -n dnode1 -c alternativeRole -v 0 system sh/cfg.sh -n dnode2 -c alternativeRole -v 0 system sh/cfg.sh -n dnode3 -c alternativeRole -v 0 system sh/cfg.sh -n dnode4 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode5 -c alternativeRole -v 0 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 1000 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 1000 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 5000 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -64,407 +72,159 @@ $stb = stb1 sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) print ============== step4: start 10 client1/ 10 client2/ 10 client3/ 10 client4/ 1 client5 -#run_back unique/cluster/client_test.sim run_back unique/cluster/client1_0.sim -run_back unique/cluster/client1_1.sim -run_back unique/cluster/client1_2.sim -run_back unique/cluster/client1_3.sim -run_back unique/cluster/client2_0.sim -run_back unique/cluster/client2_1.sim -run_back unique/cluster/client2_2.sim -run_back unique/cluster/client2_3.sim -run_back unique/cluster/client3.sim -run_back unique/cluster/client4.sim +#run_back unique/cluster/client1_1.sim +#run_back unique/cluster/client1_2.sim +#run_back unique/cluster/client1_3.sim +#run_back unique/cluster/client2_0.sim +#run_back unique/cluster/client2_1.sim +#run_back unique/cluster/client2_2.sim +#run_back unique/cluster/client2_3.sim +#run_back unique/cluster/client3.sim +#run_back unique/cluster/client4.sim sleep 20000 + +wait_subsim_insert_complete_create_tables: +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 +if $data00 < 1000 then + sleep 3000 + goto wait_subsim_insert_complete_create_tables +endi + wait_subsim_insert_data: +print select count(*) from $stb sql select count(*) from $stb print data00 $data00 -if $data00 < 1 then +if $data00 < 1000 then sleep 3000 goto wait_subsim_insert_data endi - print wait for a while to let clients start insert data sleep 5000 $loop_cnt = 0 loop_cluster_do: print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** -print ============== step5: start dnode4 and add into cluster, then wait dnode4 ready +print ============== step5: start dnode4/dnode5 and add into cluster, then wait ready system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode5 -s start sql create dnode $hostname4 +sql create dnode $hostname5 -wait_dnode4_ready_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 4 then - sleep 2000 - goto wait_dnode4_ready_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 - -if $loop_cnt == 0 then - $dnode4Status = $data4_4 -elif $loop_cnt == 1 then - $dnode4Status = $data4_6 -elif $loop_cnt == 2 then - $dnode4Status = $data4_8 -else then - print **** **** **** END loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** - return -endi - -if $dnode4Status != ready then - sleep 2000 - goto wait_dnode4_ready_0 -endi +sleep 5000 print ============== step6: stop and drop dnode1, then remove data dir of dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT - -$cnt = 0 -wait_dnode1_offline_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 4 then - sleep 2000 - goto wait_dnode1_offline_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 - -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 - -if $loop_cnt == 0 then - $dnode1Status = $data4_1 -elif $loop_cnt == 1 then - $dnode1Status = $data4_5 -elif $loop_cnt == 2 then - $dnode1Status = $data4_7 -elif $loop_cnt == 3 then - $dnode1Status = $data4_9 -else then - print **** **** **** END loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** - return -endi - -if $dnode1Status != offline then - sleep 2000 - goto wait_dnode1_offline_0 -endi - - - -$cnt = 0 -wait_mnode1_offline_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -print show mnodes -sql show mnodes -if $rows != 3 then - sleep 2000 - goto wait_mnode1_offline_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -$mnode1Status = $data2_1 -$mnode2Status = $data2_2 -$mnode3Status = $data2_3 -$mnode4Status = $data2_4 - -if $loop_cnt == 0 then - $mnode1Status = $data2_1 -elif $loop_cnt == 1 then - $mnode1Status = $data2_5 -elif $loop_cnt == 2 then - $mnode1Status = $data2_7 -elif $loop_cnt == 3 then - $mnode1Status = $data2_9 -else then - print **** **** **** END loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** - return -endi - -if $mnode1Status != offline then - sleep 2000 - goto wait_mnode1_offline_0 -endi - - - +sleep 5000 sql drop dnode $hostname1 +sleep 5000 + system rm -rf ../../../sim/dnode1/data +sleep 20000 -$cnt = 0 -wait_mnode4_slave_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -print show mnodes sql show mnodes -if $rows != 3 then - sleep 2000 - goto wait_mnode4_slave_0 -endi +print show mnodes +print rows: $rows print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -$mnode1Status = $data2_1 -$mnode2Status = $data2_2 -$mnode3Status = $data2_3 -$mnode4Status = $data2_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 +return -1 -if $loop_cnt == 0 then - $mnode4Status = $data2_4 -elif $loop_cnt == 1 then - $mnode4Status = $data2_6 -elif $loop_cnt == 2 then - $mnode4Status = $data2_8 -else then - print **** **** **** END loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** - return -endi - -if $mnode4Status != slave then - sleep 2000 - goto wait_mnode4_slave_0 -endi - -print ============== step7: stop dnode2, waiting dnode4 +print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT +sleep 5000 -$cnt = 0 -wait_dnode2_offline_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_offline_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 - -if $dnode2Status != offline then - sleep 2000 - goto wait_dnode2_offline_0 -endi - -sleep 3000 -print show mnodes sql show mnodes +print show mnodes +print rows: $rows print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step8: restart dnode2, then wait sync end system sh/exec.sh -n dnode2 -s start - -$cnt = 0 -wait_dnode2_ready_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_ready_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode2_ready_0 -endi - -sleep 3000 -print show mnodes +sleep 20000 sql show mnodes +print show mnodes +print rows: $rows print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step9: stop dnode3, then wait sync end system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 3000 - -$cnt = 0 -wait_dnode3_offline_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode3_offline_0 -endi +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 - -if $dnode3Status != offline then - sleep 2000 - goto wait_dnode3_offline_0 -endi +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step10: restart dnode3, then wait sync end system sh/exec.sh -n dnode3 -s start -sleep 3000 - -$cnt = 0 -wait_dnode3_ready_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode3_ready_0 -endi +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 - -if $dnode3Status != ready then - sleep 2000 - goto wait_dnode3_ready_0 -endi +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step11: stop dnode4, then wait sync end system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 3000 - -$cnt = 0 -wait_dnode4_offline_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode4_offline_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 - -if $loop_cnt == 0 then - $dnode4Status = $data4_4 -elif $loop_cnt == 1 then - $dnode4Status = $data4_6 -elif $loop_cnt == 2 then - $dnode4Status = $data4_8 -else then - print **** **** **** END loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** - return -endi - -if $dnode4Status != offline then - sleep 2000 - goto wait_dnode4_offline_0 -endi +sleep 20000 print ============== step12: restart dnode4, then wait sync end system sh/exec.sh -n dnode4 -s start -sleep 3000 - -$cnt = 0 -wait_dnode4_ready_1: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode4_ready_1 -endi +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 - -if $loop_cnt == 0 then - $dnode4Status = $data4_4 -elif $loop_cnt == 1 then - $dnode4Status = $data4_6 -elif $loop_cnt == 2 then - $dnode4Status = $data4_8 -else then - print **** **** **** END loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** - return -endi - -if $dnode4Status != ready then - sleep 2000 - goto wait_dnode4_ready_1 -endi +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step13: alter replica 2 sql alter database $db replica 2 @@ -476,50 +236,14 @@ if $data04 != 2 then return -1 endi -print ============== step14: stop and drop dnode4, then remove data dir of dnode4 +print ============== step14: stop and drop dnode4/dnode5, then remove data dir of dnode4/dnode5 system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 3000 - -$cnt = 0 -wait_dnode4_offline_1: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode4_offline_1 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 - -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 - -if $loop_cnt == 0 then - $dnode4Status = $data4_4 -elif $loop_cnt == 1 then - $dnode4Status = $data4_6 -elif $loop_cnt == 2 then - $dnode4Status = $data4_8 -else then - print **** **** **** END loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** - return -endi - -if $dnode4Status != offline then - sleep 2000 - goto wait_dnode4_offline_1 -endi - -sleep 3000 +system sh/exec.sh -n dnode5 -s stop -x SIGINT +sleep 20000 sql drop dnode $hostname4 +sql drop dnode $hostname5 system rm -rf ../../../sim/dnode4/data - +system rm -rf ../../../sim/dnode5/data print ============== step15: alter replica 1 sql alter database $db replica 1 @@ -530,7 +254,6 @@ if $data04 != 1 then return -1 endi - print ============== step16: alter replica 2 sql alter database $db replica 2 sql show databases @@ -546,42 +269,7 @@ system sh/cfg.sh -n dnode1 -c second -v $hostname3 system sh/exec.sh -n dnode1 -s start sql create dnode $hostname1 - -wait_dnode1_ready_0: -$cnt = $cnt + 1 -if $cnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode1_ready_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 - -if $loop_cnt == 0 then - $dnode1Status = $data4_5 -elif $loop_cnt == 1 then - $dnode1Status = $data4_7 -elif $loop_cnt == 2 then - $dnode1Status = $data4_9 -else then - print **** **** **** END loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** - return -endi - -if $dnode1Status != ready then - sleep 2000 - goto wait_dnode1_ready_0 -endi +sleep 20000 print ============== step18: alter replica 3 sql alter database $db replica 3