Merge remote-tracking branch 'origin/develop' into feature/vnode

src/client/src/tscServer.c

@@ -106,13 +106,14 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
   SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;
 
   taosCorBeginWrite(&pVgroupInfo->version);
-  //TODO(dengyihao), dont care vgid 
+  tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse);
   pVgroupInfo->inUse = pEpSet->inUse;
   pVgroupInfo->numOfEps = pEpSet->numOfEps;
-  for (int32_t i = 0; pVgroupInfo->numOfEps; i++) {
+  for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) {
     strncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN);
     pVgroupInfo->epAddr[i].port = pEpSet->port[i];
   }
+  tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse);
   taosCorEndWrite(&pVgroupInfo->version);
 }
 void tscPrintMgmtEp() {
@@ -283,9 +284,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
   }
 
   if (pEpSet) { 
-    //SRpcEpSet dump; 
+    if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) {
-    tscEpSetHtons(pEpSet); 
-    if (tscEpSetIsEqual(&pSql->epSet, pEpSet)) {
       if(pCmd->command < TSDB_SQL_MGMT)  { 
         tscUpdateVgroupInfo(pSql, pEpSet); 
       } else {

tests/perftest-scripts/cassandraTestWriteLoop.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+DATA_DIR=/mnt/root/testdata
+NUM_LOOP=1
+NUM_OF_FILES=100
+OUT_FILE=cassandraWrite.out
+
+rowsPerRequest=(1 10 50 100 500 1000 2000)
+
+function printTo {
+  if $verbose ; then
+    echo $1
+  fi
+}
+
+function runTest {
+  for c in `seq 1 $clients`; do
+    avgRPR[$c]=0
+  done
+
+  printf "R/R, "
+  for c in `seq 1 $clients`; do
+    if [ "$c" == "1" ]; then
+      printf "$c client, "
+    else
+      printf "$c clients, "
+    fi
+  done
+  printf "\n"
+
+  for r in ${rowsPerRequest[@]}; do
+    printf "$r, "
+    for c in `seq 1 $clients`; do
+      totalRPR=0
+      for i in `seq 1 $NUM_LOOP`; do
+        printTo "loop i:$i, java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+            -datadir $DATA_DIR \
+            -numofFiles $NUM_OF_FILES \
+            -rowsperrequest $r \
+            -writeclients $c \
+            -conf $CAS_TEST_DIR/application.conf"
+        java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+            -datadir $DATA_DIR \
+            -numofFiles $NUM_OF_FILES \
+            -rowsperrequest $r \
+            -writeclients $c \
+            -conf $CAS_TEST_DIR/application.conf \
+            2>&1 > $OUT_FILE
+        RPR=`cat $OUT_FILE | grep "insertation speed:" | awk '{print $(NF-1)}'`
+        totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc`
+        printTo "rows:$r, clients:$c, i:$i RPR:$RPR"
+      done
+      avgRPR[$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
+    done
+    for c in `seq 1 $clients`; do
+      printf "${avgRPR[$c]}, "
+    done
+    printf "\n"
+  done
+}
+
+################ Main ################
+
+verbose=false
+clients=1
+
+while : ; do
+  case $1 in
+    -v)
+      verbose=true
+      shift ;;
+
+    -c)
+      clients=$2
+      shift 2;;
+    *)
+      break ;;
+  esac
+done
+
+printTo "Cassandra Test begin.."
+
+WORK_DIR=/mnt/root/TDengine
+CAS_TEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra
+
+runTest
+
+printTo "Cassandra Test done!"

tests/pytest/fulltest.sh

@@ -162,6 +162,7 @@ python3 ./test.py -f client/client.py
 # Misc
 python3 testCompress.py
 python3 testNoCompress.py
+python3 testMinTablesPerVnode.py
 
 # functions
 python3 ./test.py -f functions/function_avg.py

tests/pytest/test.sh

@@ -0,0 +1,23 @@
+EXEC_DIR=`dirname "$0"`
+if [[ $EXEC_DIR != "." ]]
+then
+    echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)"
+    exit -1
+fi
+CURR_DIR=`pwd`
+IN_TDINTERNAL="community"
+if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  TAOS_DIR=$CURR_DIR/../../..
+else
+  TAOS_DIR=$CURR_DIR/../..
+fi
+TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1`
+LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib
+export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR
+
+if [[ "$1" == *"test.py"* ]]; then
+  python3 ./test.py $@
+else
+  python3 $1 $@
+fi

tests/pytest/testMinTablesPerVnode.py

@@ -0,0 +1,131 @@
+#!/usr/bin/python
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+# install pip
+# pip install src/connector/python/linux/python2/
+
+# -*- coding: utf-8 -*-
+import sys
+import getopt
+import subprocess
+from distutils.log import warn as printf
+
+from util.log import *
+from util.dnodes import *
+from util.cases import *
+from util.sql import *
+
+import taos
+
+
+if __name__ == "__main__":
+    fileName = "all"
+    deployPath = ""
+    testCluster = False
+    valgrind = 0
+    logSql = True
+    stop = 0
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'l:sgh', [
+        'logSql', 'stop', 'valgrind', 'help'])
+    for key, value in opts:
+        if key in ['-h', '--help']:
+            tdLog.printNoPrefix(
+                'A collection of test cases written using Python')
+            tdLog.printNoPrefix('-l <True:False> logSql Flag')
+            tdLog.printNoPrefix('-s stop All dnodes')
+            tdLog.printNoPrefix('-g valgrind Test Flag')
+            sys.exit(0)
+
+        if key in ['-l', '--logSql']:
+            if (value.upper() == "TRUE"):
+                logSql = True
+            elif (value.upper() == "FALSE"):
+                logSql = False
+            else:
+                tdLog.printNoPrefix("logSql value %s is invalid" % logSql)
+                sys.exit(0)
+
+        if key in ['-g', '--valgrind']:
+            valgrind = 1
+
+        if key in ['-s', '--stop']:
+            stop = 1
+
+    if (stop != 0):
+        if (valgrind == 0):
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
+        killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP > /dev/null 2>&1" % toBeKilled
+
+        psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
+        processID = subprocess.check_output(psCmd, shell=True)
+
+        while(processID):
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(psCmd, shell=True)
+
+        for port in range(6030, 6041):
+            usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port
+            processID = subprocess.check_output(usePortPID, shell=True)
+
+            if processID:
+                killCmd = "kill -TERM %s" % processID
+                os.system(killCmd)
+            fuserCmd = "fuser -k -n tcp %d" % port
+            os.system(fuserCmd)
+        if valgrind:
+            time.sleep(2)
+
+        tdLog.info('stop All dnodes')
+        sys.exit(0)
+
+    tdDnodes.init(deployPath)
+    tdDnodes.setTestCluster(testCluster)
+    tdDnodes.setValgrind(valgrind)
+
+    tdDnodes.stopAll()
+    tdDnodes.addSimExtraCfg("minTablesPerVnode", "100")
+    tdDnodes.deploy(1)
+    tdDnodes.start(1)
+
+    host = '127.0.0.1'
+
+    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+
+    tdCases.logSql(logSql)
+
+    conn = taos.connect(
+        host,
+        config=tdDnodes.getSimCfgPath())
+
+    tdSql.init(conn.cursor(), True)
+
+    tdSql.execute("DROP DATABASE IF EXISTS db")
+    tdSql.execute("CREATE DATABASE IF NOT EXISTS db")
+    tdSql.execute("USE db")
+
+    for i in range(0, 100):
+        tdSql.execute(
+            "CREATE TABLE IF NOT EXISTS tb%d (ts TIMESTAMP, temperature INT, humidity FLOAT)" % i)
+
+    for i in range(1, 6):
+        tdSql.execute("INSERT INTO tb99 values (now + %da, %d, %f)" % (i, i, i * 1.0))
+
+    tdSql.execute("DROP TABLE tb99")
+    tdSql.execute(
+            "CREATE TABLE IF NOT EXISTS tb99 (ts TIMESTAMP, temperature INT, humidity FLOAT)")
+    tdSql.query("SELECT * FROM tb99")
+    tdSql.checkRows(0)
+    
+    conn.close()