From e11f0585ef0a0dcc0aa51c91126d223ea85d516b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 22 Mar 2021 10:00:55 +0800 Subject: [PATCH 01/22] [TD-3298]: fix coverity scan error --- src/tfs/src/tdisk.c | 2 +- src/tfs/src/tfs.c | 12 ++++++------ src/tsdb/inc/tsdbFile.h | 2 +- src/tsdb/src/tsdbCommit.c | 14 +++++++------- src/tsdb/src/tsdbFS.c | 14 +++++++------- src/tsdb/src/tsdbFile.c | 2 +- src/tsdb/src/tsdbReadImpl.c | 2 +- src/util/inc/tchecksum.h | 1 - 8 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/tfs/src/tdisk.c b/src/tfs/src/tdisk.c index 28c836e5d9..22601e48c3 100644 --- a/src/tfs/src/tdisk.c +++ b/src/tfs/src/tdisk.c @@ -27,7 +27,7 @@ SDisk *tfsNewDisk(int level, int id, const char *dir) { pDisk->level = level; pDisk->id = id; - strncpy(pDisk->dir, dir, TSDB_FILENAME_LEN); + tstrncpy(pDisk->dir, dir, TSDB_FILENAME_LEN); return pDisk; } diff --git a/src/tfs/src/tfs.c b/src/tfs/src/tfs.c index 63fc090f00..f78535b8ed 100644 --- a/src/tfs/src/tfs.c +++ b/src/tfs/src/tfs.c @@ -187,7 +187,7 @@ void tfsInitFile(TFILE *pf, int level, int id, const char *bname) { pf->level = level; pf->id = id; - strncpy(pf->rname, bname, TSDB_FILENAME_LEN); + tstrncpy(pf->rname, bname, TSDB_FILENAME_LEN); char tmpName[TMPNAME_LEN] = {0}; snprintf(tmpName, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), bname); @@ -230,15 +230,15 @@ void *tfsDecodeFile(void *buf, TFILE *pf) { void tfsbasename(const TFILE *pf, char *dest) { char tname[TSDB_FILENAME_LEN] = "\0"; - strncpy(tname, pf->aname, TSDB_FILENAME_LEN); - strncpy(dest, basename(tname), TSDB_FILENAME_LEN); + tstrncpy(tname, pf->aname, TSDB_FILENAME_LEN); + tstrncpy(dest, basename(tname), TSDB_FILENAME_LEN); } void tfsdirname(const TFILE *pf, char *dest) { char tname[TSDB_FILENAME_LEN] = "\0"; - strncpy(tname, pf->aname, TSDB_FILENAME_LEN); - strncpy(dest, dirname(tname), TSDB_FILENAME_LEN); + tstrncpy(tname, pf->aname, TSDB_FILENAME_LEN); + tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); } // DIR APIs ==================================== @@ -344,7 +344,7 @@ TDIR *tfsOpendir(const char *rname) { } tfsInitDiskIter(&(tdir->iter)); - strncpy(tdir->dirname, rname, TSDB_FILENAME_LEN); + tstrncpy(tdir->dirname, rname, TSDB_FILENAME_LEN); if (tfsOpendirImpl(tdir) < 0) { free(tdir); diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h index f1e2422e45..792efcdb2e 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/src/tsdb/inc/tsdbFile.h @@ -334,7 +334,7 @@ static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) { static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); } } diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index a1ed0796a6..02dd2d1ca4 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -164,7 +164,7 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, tstrerror(terrno)); tsdbCloseMFile(&mf); - tsdbApplyMFileChange(&mf, pOMFile); + (void)tsdbApplyMFileChange(&mf, pOMFile); // TODO: need to reload metaCache return -1; } @@ -304,7 +304,7 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) { SDFileSet *pSet = NULL; int fid; - memset(&commith, 0, sizeof(SMemTable *)); + memset(&commith, 0, sizeof(commith)); if (pMem->numOfRows <= 0) { // No memory data, just apply retention on each file on disk @@ -399,9 +399,9 @@ static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); SMemTable *pIMem = pRepo->imem; - tsdbLockRepo(pRepo); + (void)tsdbLockRepo(pRepo); pRepo->imem = NULL; - tsdbUnlockRepo(pRepo); + (void)tsdbUnlockRepo(pRepo); tsdbUnRefMemTable(pRepo, pIMem); tsem_post(&(pRepo->readyToCommit)); } @@ -1136,12 +1136,12 @@ static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { } static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) { - if (taosArrayPush(pCommith->aSupBlk, pSupBlock) < 0) { + if (taosArrayPush(pCommith->aSupBlk, pSupBlock) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } - if (pSubBlocks && taosArrayPushBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) < 0) { + if (pSubBlocks && taosArrayPushBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } @@ -1379,7 +1379,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tstrerror(terrno)); tsdbCloseDFileSet(pWSet); - tsdbRemoveDFile(pWHeadf); + (void)tsdbRemoveDFile(pWHeadf); if (pCommith->isRFileSet) { tsdbCloseAndUnsetFSet(&(pCommith->readh)); return -1; diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index cbff4fbeaa..a9e74cb229 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -380,7 +380,7 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) { if (taosWrite(fd, pBuf, fsheader.len) < fsheader.len) { terrno = TAOS_SYSTEM_ERROR(errno); close(fd); - remove(tfname); + (void)remove(tfname); taosTZfree(pBuf); return -1; } @@ -413,7 +413,7 @@ static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo) { sizeTo = taosArrayGetSize(pTo->df); // Apply meta file change - tsdbApplyMFileChange(pFrom->pmf, pTo->pmf); + (void)tsdbApplyMFileChange(pFrom->pmf, pTo->pmf); // Apply SDFileSet change if (ifrom >= sizeFrom) { @@ -853,7 +853,7 @@ static int tsdbScanRootDir(STsdbRepo *pRepo) { continue; } - tfsremove(pf); + (void)tfsremove(pf); tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); } @@ -879,7 +879,7 @@ static int tsdbScanDataDir(STsdbRepo *pRepo) { tfsbasename(pf, bname); if (!tsdbIsTFileInFS(pfs, pf)) { - tfsremove(pf); + (void)tfsremove(pf); tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); } } @@ -939,7 +939,7 @@ static int tsdbRestoreMeta(STsdbRepo *pRepo) { if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) { // Skip current.t file tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); - tfsremove(pf); + (void)tfsremove(pf); continue; } @@ -1045,7 +1045,7 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { int code = regexec(®ex, bname, 0, NULL, 0); if (code == 0) { - if (taosArrayPush(fArray, (void *)pf) < 0) { + if (taosArrayPush(fArray, (void *)pf) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tfsClosedir(tdir); taosArrayDestroy(fArray); @@ -1055,7 +1055,7 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { } else if (code == REG_NOMATCH) { // Not match tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); - tfsremove(pf); + (void)tfsremove(pf); continue; } else { // Has other error diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 8124a0e3b5..ac33096aae 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -523,7 +523,7 @@ static int tsdbApplyDFileChange(SDFile *from, SDFile *to) { tsdbRollBackDFile(to); } } else { - tsdbRemoveDFile(from); + (void)tsdbRemoveDFile(from); } } } diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index 572706d45e..7212ae1636 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -139,7 +139,7 @@ int tsdbLoadBlockIdx(SReadH *pReadh) { ptr = tsdbDecodeSBlockIdx(ptr, &blkIdx); ASSERT(ptr != NULL); - if (taosArrayPush(pReadh->aBlkIdx, (void *)(&blkIdx)) < 0) { + if (taosArrayPush(pReadh->aBlkIdx, (void *)(&blkIdx)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } diff --git a/src/util/inc/tchecksum.h b/src/util/inc/tchecksum.h index 495aaf33e8..12ca3a5443 100644 --- a/src/util/inc/tchecksum.h +++ b/src/util/inc/tchecksum.h @@ -47,7 +47,6 @@ static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uin } static FORCE_INLINE int taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) { - if (ssize < 0) return 0; return (checksum == (*crc32c)(0, stream, (size_t)ssize)); } From 8c8f3d0835ff09f07434926e21b2c009589f2b4f Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 22 Mar 2021 11:35:46 +0800 Subject: [PATCH 02/22] [TD-3406]support more dnodes in docker clu --- tests/pytest/cluster/clusterEnvSetup/basic.py | 4 +- .../clusterEnvSetup/buildClusterEnv.sh | 54 +++++++++++++------ .../clusterEnvSetup/cleanClusterEnv.sh | 2 +- .../clusterEnvSetup/docker-compose.yml | 5 ++ .../pytest/cluster/clusterEnvSetup/node3.yml | 7 ++- .../pytest/cluster/clusterEnvSetup/node4.yml | 5 ++ .../pytest/cluster/clusterEnvSetup/node5.yml | 5 ++ 7 files changed, 62 insertions(+), 20 deletions(-) diff --git a/tests/pytest/cluster/clusterEnvSetup/basic.py b/tests/pytest/cluster/clusterEnvSetup/basic.py index eb39051898..3f82a36497 100644 --- a/tests/pytest/cluster/clusterEnvSetup/basic.py +++ b/tests/pytest/cluster/clusterEnvSetup/basic.py @@ -45,8 +45,8 @@ class BuildDockerCluser: os.system("docker exec -d $(docker ps|grep tdnode1|awk '{print $1}') tarbitrator") def run(self): - if self.numOfNodes < 2 or self.numOfNodes > 5: - print("the number of nodes must be between 2 and 5") + if self.numOfNodes < 2 or self.numOfNodes > 10: + print("the number of nodes must be between 2 and 10") exit(0) print("remove Flag value %s" % self.removeFlag) if self.removeFlag == False: diff --git a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh index f6199649cd..47361db6a3 100755 --- a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh +++ b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh @@ -32,7 +32,7 @@ do done function addTaoscfg { - for i in {1..5} + for((i=1;i<=$NUM_OF_NODES;i++)) do touch $DOCKER_DIR/node$i/cfg/taos.cfg echo 'firstEp tdnode1:6030' > $DOCKER_DIR/node$i/cfg/taos.cfg @@ -42,7 +42,7 @@ function addTaoscfg { } function createDIR { - for i in {1..5} + for((i=1;i<=$NUM_OF_NODES;i++)) do mkdir -p $DOCKER_DIR/node$i/data mkdir -p $DOCKER_DIR/node$i/log @@ -53,7 +53,7 @@ function createDIR { function cleanEnv { echo "Clean up docker environment" - for i in {1..5} + for((i=1;i<=$NUM_OF_NODES;i++)) do rm -rf $DOCKER_DIR/node$i/data/* rm -rf $DOCKER_DIR/node$i/log/* @@ -99,22 +99,44 @@ function clusterUp { cd $DOCKER_DIR - if [ $NUM_OF_NODES -eq 2 ]; then - echo "create 2 dnodes" - PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose up -d + # if [ $NUM_OF_NODES -eq 2 ]; then + # echo "create 2 dnodes" + # PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose up -d + # fi + docker_run="PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " + if [ $NUM_OF_NODES -ge 2 ];then + echo "create $NUM_OF_NODES dnodes" + for((i=3;i<=$NUM_OF_NODES;i++)) + do + if [ ! -f node$i.yml ];then + echo "node$i.yml not exist" + # cp $CURR_DIR/node3.yml $CURR_DIR/node$i.yml + # sed -i "s/td2.0-node3/td2.0-node$i/g" $CURR_DIR/node$i.yml + # sed -i "s/'tdnode3'/'tdnode$i'/g" $CURR_DIR/node$i.yml + # sed -i "s#/node3/#/node$i/#g" $CURR_DIR/node$i.yml + # sed -i "s#ipv4_address: 172.27.0.9#ipv4_address: 172.27.0.`expr $i + 6`#g" $CURR_DIR/node$i.yml + cp node3.yml node$i.yml + sed -i "s/td2.0-node3/td2.0-node$i/g" node$i.yml + sed -i "s/'tdnode3'/'tdnode$i'/g" node$i.yml + sed -i "s#/node3/#/node$i/#g" node$i.yml + sed -i "s#ipv4_address: 172.27.0.9#ipv4_address: 172.27.0.`expr $i + 6`#g" node$i.yml + fi + docker_run=$docker_run" -f node$i.yml " + done + docker_run=$docker_run" up -d" fi + echo $docker_run |sh + # if [ $NUM_OF_NODES -eq 3 ]; then + # PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml -f node3.yml up -d + # fi - if [ $NUM_OF_NODES -eq 3 ]; then - PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml -f node3.yml up -d - fi + # if [ $NUM_OF_NODES -eq 4 ]; then + # PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d + # fi - if [ $NUM_OF_NODES -eq 4 ]; then - PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d - fi - - if [ $NUM_OF_NODES -eq 5 ]; then - PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d - fi + # if [ $NUM_OF_NODES -eq 5 ]; then + # PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d + # fi echo "docker compose finish" } diff --git a/tests/pytest/cluster/clusterEnvSetup/cleanClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/cleanClusterEnv.sh index 006db3f4eb..675cae5fef 100755 --- a/tests/pytest/cluster/clusterEnvSetup/cleanClusterEnv.sh +++ b/tests/pytest/cluster/clusterEnvSetup/cleanClusterEnv.sh @@ -28,7 +28,7 @@ function removeDockerContainers { function cleanEnv { echo "Clean up docker environment" - for i in {1..5} + for i in {1..10} do rm -rf $DOCKER_DIR/node$i/data/* rm -rf $DOCKER_DIR/node$i/log/* diff --git a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml index 80cd859cdf..9994d671cc 100644 --- a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml +++ b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml @@ -30,6 +30,11 @@ services: - "tdnode3:172.27.0.9" - "tdnode4:172.27.0.10" - "tdnode5:172.27.0.11" + - "tdnode6:172.27.0.12" + - "tdnode7:172.27.0.13" + - "tdnode8:172.27.0.14" + - "tdnode9:172.27.0.15" + - "tdnode9:172.27.0.16" volumes: # bind data directory - type: bind diff --git a/tests/pytest/cluster/clusterEnvSetup/node3.yml b/tests/pytest/cluster/clusterEnvSetup/node3.yml index 93044d24af..431719af0c 100644 --- a/tests/pytest/cluster/clusterEnvSetup/node3.yml +++ b/tests/pytest/cluster/clusterEnvSetup/node3.yml @@ -24,10 +24,15 @@ services: sysctl -p && exec my-main-application" extra_hosts: - - "tdnode1:172.27.0.7" - "tdnode2:172.27.0.8" + - "tdnode3:172.27.0.9" - "tdnode4:172.27.0.10" - "tdnode5:172.27.0.11" + - "tdnode6:172.27.0.12" + - "tdnode7:172.27.0.13" + - "tdnode8:172.27.0.14" + - "tdnode9:172.27.0.15" + - "tdnode9:172.27.0.16" volumes: # bind data directory - type: bind diff --git a/tests/pytest/cluster/clusterEnvSetup/node4.yml b/tests/pytest/cluster/clusterEnvSetup/node4.yml index 1fc603b3b5..ecc29bfeeb 100644 --- a/tests/pytest/cluster/clusterEnvSetup/node4.yml +++ b/tests/pytest/cluster/clusterEnvSetup/node4.yml @@ -28,6 +28,11 @@ services: - "tdnode3:172.27.0.9" - "tdnode4:172.27.0.10" - "tdnode5:172.27.0.11" + - "tdnode6:172.27.0.12" + - "tdnode7:172.27.0.13" + - "tdnode8:172.27.0.14" + - "tdnode9:172.27.0.15" + - "tdnode9:172.27.0.16" volumes: # bind data directory - type: bind diff --git a/tests/pytest/cluster/clusterEnvSetup/node5.yml b/tests/pytest/cluster/clusterEnvSetup/node5.yml index 5e3aba3b76..af209ff314 100644 --- a/tests/pytest/cluster/clusterEnvSetup/node5.yml +++ b/tests/pytest/cluster/clusterEnvSetup/node5.yml @@ -28,6 +28,11 @@ services: - "tdnode3:172.27.0.9" - "tdnode4:172.27.0.10" - "tdnode5:172.27.0.11" + - "tdnode6:172.27.0.12" + - "tdnode7:172.27.0.13" + - "tdnode8:172.27.0.14" + - "tdnode9:172.27.0.15" + - "tdnode9:172.27.0.16" volumes: # bind data directory - type: bind From a2a6655a6a20f9543affc54ad0a6682838e0f83a Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 22 Mar 2021 11:44:41 +0800 Subject: [PATCH 03/22] change --- .../clusterEnvSetup/buildClusterEnv.sh | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh index 47361db6a3..4cd305d6cd 100755 --- a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh +++ b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh @@ -99,10 +99,6 @@ function clusterUp { cd $DOCKER_DIR - # if [ $NUM_OF_NODES -eq 2 ]; then - # echo "create 2 dnodes" - # PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose up -d - # fi docker_run="PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " if [ $NUM_OF_NODES -ge 2 ];then echo "create $NUM_OF_NODES dnodes" @@ -110,11 +106,6 @@ function clusterUp { do if [ ! -f node$i.yml ];then echo "node$i.yml not exist" - # cp $CURR_DIR/node3.yml $CURR_DIR/node$i.yml - # sed -i "s/td2.0-node3/td2.0-node$i/g" $CURR_DIR/node$i.yml - # sed -i "s/'tdnode3'/'tdnode$i'/g" $CURR_DIR/node$i.yml - # sed -i "s#/node3/#/node$i/#g" $CURR_DIR/node$i.yml - # sed -i "s#ipv4_address: 172.27.0.9#ipv4_address: 172.27.0.`expr $i + 6`#g" $CURR_DIR/node$i.yml cp node3.yml node$i.yml sed -i "s/td2.0-node3/td2.0-node$i/g" node$i.yml sed -i "s/'tdnode3'/'tdnode$i'/g" node$i.yml @@ -126,18 +117,7 @@ function clusterUp { docker_run=$docker_run" up -d" fi echo $docker_run |sh - # if [ $NUM_OF_NODES -eq 3 ]; then - # PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml -f node3.yml up -d - # fi - - # if [ $NUM_OF_NODES -eq 4 ]; then - # PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d - # fi - - # if [ $NUM_OF_NODES -eq 5 ]; then - # PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d - # fi - + echo "docker compose finish" } From aa6b81892f5cd2118b89d21f99073e841b59e8e2 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 22 Mar 2021 13:56:40 +0800 Subject: [PATCH 04/22] [TD-3368]output log of failed case --- tests/test-all.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index dcc2b61f43..b5e23c1506 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -79,24 +79,26 @@ function runSimCaseOneByOnefq { date +%F\ %T | tee -a out.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then echo -n $case - ./test.sh -f $case > /dev/null 2>&1 && \ + ./test.sh -f $case > ../../../sim/case.log 2>&1 && \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ - echo -e "${RED} failed${NC}" | tee -a out.log + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat ../../../sim/case.log ) else echo -n $case - ./test.sh -f $case > /dev/null 2>&1 && \ + ./test.sh -f $case > ../../sim/case.log 2>&1 && \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ - echo -e "${RED} failed${NC}" | tee -a out.log + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat ../../sim/case.log ) fi out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` + rm -rf ../../../sim/case.log else cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + rm -rf ../../sim/case.log fi exit 8 fi @@ -105,6 +107,8 @@ function runSimCaseOneByOnefq { dohavecore $2 fi done + rm -rf ../../../sim/case.log + rm -rf ../../sim/case.log } function runPyCaseOneByOne { @@ -158,13 +162,16 @@ function runPyCaseOneByOnefq() { start_time=`date +%s` date +%F\ %T | tee -a pytest-out.log echo -n $case - $line > /dev/null 2>&1 && \ + $line > ../../sim/case.log 2>&1 && \ echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ - echo -e "${RED} failed${NC}" | tee -a pytest-out.log + echo -e "${RED} failed${NC}" | tee -a pytest-out.log end_time=`date +%s` out_log=`tail -1 pytest-out.log ` if [[ $out_log =~ 'failed' ]];then cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + echo '=====================log=====================' + cat ../../sim/case.log + rm -rf ../../sim/case.log exit 8 fi echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log @@ -174,6 +181,7 @@ function runPyCaseOneByOnefq() { dohavecore $2 fi done + rm -rf ../../sim/case.log } totalFailed=0 From de97080d8524a06f471a48a49925a47bc892b026 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Mar 2021 14:36:30 +0800 Subject: [PATCH 05/22] Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#5521) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 105 +++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 44 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4191896ccb..51c94a8fa1 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4212,7 +4212,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq) } } else { snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%d", - superTblInfo?superTblInfo->childTblPrefix:g_args.tb_prefix, tableSeq); + g_args.tb_prefix, tableSeq); } } @@ -4355,7 +4355,7 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, tableName); } else { len = snprintf(buffer, - (superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len), + superTblInfo->maxSqlLen, "insert into %s.%s values", pThreadInfo->db_name, tableName); @@ -5471,17 +5471,21 @@ static int queryTestProcess() { && g_queryInfo.superQueryInfo.concurrent > 0) { pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); + if (NULL == pids) { + taos_close(taos); + ERROR_EXIT("memory allocation failed\n"); + } infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); - if ((NULL == pids) || (NULL == infos)) { - printf("malloc failed for create threads\n"); + if (NULL == infos) { taos_close(taos); - exit(-1); + free(pids); + ERROR_EXIT("memory allocation failed for create threads\n"); } for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; - + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { t_info->taos = taos; @@ -5489,6 +5493,8 @@ static int queryTestProcess() { sprintf(sqlStr, "use %s", g_queryInfo.dbName); verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); if (0 != queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE)) { + free(infos); + free(pids); errorPrint( "use database %s failed!\n\n", g_queryInfo.dbName); return -1; @@ -5496,7 +5502,7 @@ static int queryTestProcess() { } else { t_info->taos = NULL; } - + pthread_create(pids + i, NULL, superQueryProcess, t_info); } }else { @@ -5509,11 +5515,21 @@ static int queryTestProcess() { if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); - if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - printf("malloc failed for create threads\n"); + if (NULL == pidsOfSub) { taos_close(taos); - exit(-1); + free(infos); + free(pids); + + ERROR_EXIT("memory allocation failed for create threads\n"); + } + + infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); + if (NULL == infosOfSub) { + taos_close(taos); + free(pidsOfSub); + free(infos); + free(pids); + ERROR_EXIT("memory allocation failed for create threads\n"); } int ntables = g_queryInfo.subQueryInfo.childTblCount; @@ -5544,62 +5560,63 @@ static int queryTestProcess() { } g_queryInfo.subQueryInfo.threadCnt = threads; - }else { + } else { g_queryInfo.subQueryInfo.threadCnt = 0; } - + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { pthread_join(pids[i], NULL); } tmfree((char*)pids); - tmfree((char*)infos); - + tmfree((char*)infos); + for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { pthread_join(pidsOfSub[i], NULL); } tmfree((char*)pidsOfSub); - tmfree((char*)infosOfSub); - + tmfree((char*)infosOfSub); + taos_close(taos); return 0; } -static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { +static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { if (res == NULL || taos_errno(res) != 0) { - printf("failed to subscribe result, code:%d, reason:%s\n", code, taos_errstr(res)); + errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + __func__, __LINE__, code, taos_errstr(res)); return; } - + getResult(res, (char*)param); taos_free_result(res); } static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) { - TAOS_SUB* tsub = NULL; + TAOS_SUB* tsub = NULL; if (g_queryInfo.superQueryInfo.subscribeMode) { - tsub = taos_subscribe(taos, - g_queryInfo.superQueryInfo.subscribeRestart, - topic, sql, subscribe_callback, (void*)resultFileName, + tsub = taos_subscribe(taos, + g_queryInfo.superQueryInfo.subscribeRestart, + topic, sql, subscribe_callback, (void*)resultFileName, g_queryInfo.superQueryInfo.subscribeInterval); } else { - tsub = taos_subscribe(taos, - g_queryInfo.superQueryInfo.subscribeRestart, + tsub = taos_subscribe(taos, + g_queryInfo.superQueryInfo.subscribeRestart, topic, sql, NULL, NULL, 0); } if (tsub == NULL) { printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql); return NULL; - } + } return tsub; } static void *subSubscribeProcess(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; + threadInfo *winfo = (threadInfo *)sarg; char subSqlstr[1024]; char sqlStr[MAX_TB_NAME_SIZE*2]; @@ -5608,7 +5625,7 @@ static void *subSubscribeProcess(void *sarg) { if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)){ return NULL; } - + //int64_t st = 0; //int64_t et = 0; do { @@ -5643,13 +5660,13 @@ static void *subSubscribeProcess(void *sarg) { if (1 == g_queryInfo.subQueryInfo.subscribeMode) { continue; } - + res = taos_consume(g_queryInfo.subQueryInfo.tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.subQueryInfo.result[i], + sprintf(tmpFile, "%s-%d", + g_queryInfo.subQueryInfo.result[i], winfo->threadID); } getResult(res, tmpFile); @@ -5657,16 +5674,16 @@ static void *subSubscribeProcess(void *sarg) { } } taos_free_result(res); - + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], + taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], g_queryInfo.subQueryInfo.subscribeKeepProgress); } return NULL; } static void *superSubscribeProcess(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; + threadInfo *winfo = (threadInfo *)sarg; char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); @@ -5674,7 +5691,7 @@ static void *superSubscribeProcess(void *sarg) { if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)) { return NULL; } - + //int64_t st = 0; //int64_t et = 0; do { @@ -5689,13 +5706,13 @@ static void *superSubscribeProcess(void *sarg) { sprintf(topic, "taosdemo-subscribe-%d", i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", + sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); } - g_queryInfo.superQueryInfo.tsub[i] = - subscribeImpl(winfo->taos, - g_queryInfo.superQueryInfo.sql[i], - topic, tmpFile); + g_queryInfo.superQueryInfo.tsub[i] = + subscribeImpl(winfo->taos, + g_queryInfo.superQueryInfo.sql[i], + topic, tmpFile); if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { return NULL; } @@ -5711,12 +5728,12 @@ static void *superSubscribeProcess(void *sarg) { if (1 == g_queryInfo.superQueryInfo.subscribeMode) { continue; } - + res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", + sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); } getResult(res, tmpFile); @@ -5726,7 +5743,7 @@ static void *superSubscribeProcess(void *sarg) { taos_free_result(res); for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], + taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); } return NULL; From 0916d24506b4aaf08315045508446bf1ab105cfa Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 22 Mar 2021 14:46:58 +0800 Subject: [PATCH 06/22] fix hosts error --- tests/pytest/cluster/clusterEnvSetup/docker-compose.yml | 2 +- tests/pytest/cluster/clusterEnvSetup/node3.yml | 2 +- tests/pytest/cluster/clusterEnvSetup/node4.yml | 2 +- tests/pytest/cluster/clusterEnvSetup/node5.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml index 9994d671cc..7b47b5e459 100644 --- a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml +++ b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml @@ -34,7 +34,7 @@ services: - "tdnode7:172.27.0.13" - "tdnode8:172.27.0.14" - "tdnode9:172.27.0.15" - - "tdnode9:172.27.0.16" + - "tdnode10:172.27.0.16" volumes: # bind data directory - type: bind diff --git a/tests/pytest/cluster/clusterEnvSetup/node3.yml b/tests/pytest/cluster/clusterEnvSetup/node3.yml index 431719af0c..56fba49bd7 100644 --- a/tests/pytest/cluster/clusterEnvSetup/node3.yml +++ b/tests/pytest/cluster/clusterEnvSetup/node3.yml @@ -32,7 +32,7 @@ services: - "tdnode7:172.27.0.13" - "tdnode8:172.27.0.14" - "tdnode9:172.27.0.15" - - "tdnode9:172.27.0.16" + - "tdnode10:172.27.0.16" volumes: # bind data directory - type: bind diff --git a/tests/pytest/cluster/clusterEnvSetup/node4.yml b/tests/pytest/cluster/clusterEnvSetup/node4.yml index ecc29bfeeb..2af9fc0293 100644 --- a/tests/pytest/cluster/clusterEnvSetup/node4.yml +++ b/tests/pytest/cluster/clusterEnvSetup/node4.yml @@ -32,7 +32,7 @@ services: - "tdnode7:172.27.0.13" - "tdnode8:172.27.0.14" - "tdnode9:172.27.0.15" - - "tdnode9:172.27.0.16" + - "tdnode10:172.27.0.16" volumes: # bind data directory - type: bind diff --git a/tests/pytest/cluster/clusterEnvSetup/node5.yml b/tests/pytest/cluster/clusterEnvSetup/node5.yml index af209ff314..76d5346a14 100644 --- a/tests/pytest/cluster/clusterEnvSetup/node5.yml +++ b/tests/pytest/cluster/clusterEnvSetup/node5.yml @@ -32,7 +32,7 @@ services: - "tdnode7:172.27.0.13" - "tdnode8:172.27.0.14" - "tdnode9:172.27.0.15" - - "tdnode9:172.27.0.16" + - "tdnode10:172.27.0.16" volumes: # bind data directory - type: bind From 4e0b9ced167bd229dd8cb385232977fc8733fa43 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 22 Mar 2021 14:49:20 +0800 Subject: [PATCH 07/22] [TD-2639] : clarify the length limitation of tags. --- documentation20/cn/12.taos-sql/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 2248b11987..b4fa2b160a 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -249,7 +249,7 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic 3) TAGS 列名不能为预留关键字; - 4) TAGS 最多允许128个,至少1个,总长度不超过16k个字符。 + 4) TAGS 最多允许 128 个,至少 1 个,总长度不超过 16 KB。 - **删除超级表** From 720661366a0792b483a98c3f16527de1c3cc88d3 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 22 Mar 2021 14:19:49 +0800 Subject: [PATCH 08/22] [TD-3413]: make docker cluster support enterprise --- tests/pytest/cluster/clusterEnvSetup/basic.py | 2 +- .../clusterEnvSetup/buildClusterEnv.sh | 55 +++++++++++++++---- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/tests/pytest/cluster/clusterEnvSetup/basic.py b/tests/pytest/cluster/clusterEnvSetup/basic.py index 3f82a36497..dc7e07fd5c 100644 --- a/tests/pytest/cluster/clusterEnvSetup/basic.py +++ b/tests/pytest/cluster/clusterEnvSetup/basic.py @@ -96,7 +96,7 @@ parser.add_argument( '-v', '--version', action='store', - default='2.0.17.1', + default='2.0.18.1', type=str, help='the version of the cluster to be build, Default is 2.0.17.1') parser.add_argument( diff --git a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh index 4cd305d6cd..88f92e1486 100755 --- a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh +++ b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh @@ -1,6 +1,7 @@ #!/bin/bash echo "Executing buildClusterEnv.sh" CURR_DIR=`pwd` +IN_TDINTERNAL="community" if [ $# != 6 ]; then echo "argument list need input : " @@ -68,23 +69,48 @@ function prepareBuild { fi if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz ] || [ ! -e $DOCKER_DIR/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then - cd $CURR_DIR/../../../../packaging + cd $CURR_DIR/../../../../packaging + echo $CURR_DIR + echo $IN_TDINTERNAL echo "generating TDeninger packages" - ./release.sh -v edge -n $VERSION >> /dev/null - - if [ ! -e $CURR_DIR/../../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then - echo "no TDengine install package found" - exit 1 + if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + pwd + ./release.sh -v cluster -n $VERSION >> /dev/null 2>&1 + else + pwd + ./release.sh -v edge -n $VERSION >> /dev/null 2>&1 fi - if [ ! -e $CURR_DIR/../../../../release/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then - echo "no arbitrator install package found" - exit 1 + if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + if [ ! -e $CURR_DIR/../../../../release/TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz ]; then + echo "no TDengine install package found" + exit 1 + fi + + if [ ! -e $CURR_DIR/../../../../release/TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz ]; then + echo "no arbitrator install package found" + exit 1 + fi + else + if [ ! -e $CURR_DIR/../../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then + echo "no TDengine install package found" + exit 1 + fi + + if [ ! -e $CURR_DIR/../../../../release/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then + echo "no arbitrator install package found" + exit 1 + fi fi cd $CURR_DIR/../../../../release - mv TDengine-server-$VERSION-Linux-x64.tar.gz $DOCKER_DIR - mv TDengine-arbitrator-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + mv TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + mv TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + else + mv TDengine-server-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + mv TDengine-arbitrator-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + fi fi rm -rf $DOCKER_DIR/*.yml @@ -99,7 +125,12 @@ function clusterUp { cd $DOCKER_DIR - docker_run="PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " + if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + docker_run="PACKAGE=TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine--enterprisearbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-enterprise-server-$VERSION DIR2=TDengine-enterprise-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " + else + docker_run="PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " + fi + if [ $NUM_OF_NODES -ge 2 ];then echo "create $NUM_OF_NODES dnodes" for((i=3;i<=$NUM_OF_NODES;i++)) From 81d62650e9385607af260aa61f72449ea937f497 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 22 Mar 2021 14:56:22 +0800 Subject: [PATCH 09/22] Fix script issue --- tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh index 88f92e1486..de6f9d8c17 100755 --- a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh +++ b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh @@ -126,7 +126,7 @@ function clusterUp { cd $DOCKER_DIR if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then - docker_run="PACKAGE=TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine--enterprisearbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-enterprise-server-$VERSION DIR2=TDengine-enterprise-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " + docker_run="PACKAGE=TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-enterprise-server-$VERSION DIR2=TDengine-enterprise-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " else docker_run="PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " fi From 5a3486541618f7847c0ecd199c72a9bacc12e50c Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Mon, 22 Mar 2021 15:25:45 +0800 Subject: [PATCH 10/22] Update docs.md TDengine-2.0.18.0 need using taos-jdbcdriver-2.0.22 --- documentation20/cn/08.connector/01.java/docs.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation20/cn/08.connector/01.java/docs.md b/documentation20/cn/08.connector/01.java/docs.md index c39f6ffb4c..3ed7343579 100644 --- a/documentation20/cn/08.connector/01.java/docs.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -451,7 +451,8 @@ Query OK, 1 row(s) in set (0.000141s) | taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | | -------------------- | ----------------- | -------- | -| 2.0.12 及以上 | 2.0.8.0 及以上 | 1.8.x | +| 2.0.22 | 2.0.18.0 及以上 | 1.8.x | +| 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.0 | 1.8.x | | 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | | 1.0.3 | 1.6.1.x 及以上 | 1.8.x | | 1.0.2 | 1.6.1.x 及以上 | 1.8.x | From 82ad11630dfce99c92986e06f3051ca17dc1a3d9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Mar 2021 15:44:06 +0800 Subject: [PATCH 11/22] Feature/sangshuduo/td 3416 taosdemo interlace testcase (#5524) * [TD-3316] : add testcase for taosdemo limit and offset. check offset 0. * [TD-3316] : add testcase for taosdemo limit and offset. fix sample file import bug. * [TD-3316] : add test case for limit and offset. fix sample data issue. * [TD-3327] : fix taosdemo segfault when import data from sample data file. * [TD-3317] : make taosdemo support interlace mode. json parameter rows_per_tbl support. * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode insertion. refactor. * [TD-3317] : support interlace mode insertion. change json file. * [TD-3317] : support interlace mode insertion. fix multithread create table regression. * [TD-3317] : support interlace mode insertion. working but not perfect. * [TD-3317] : support interlace mode insertion. rename lowaTest with taosdemoTestWithJson * [TD-3317] : support interlace mode insertion. perfect * [TD-3317] : support interlace mode insertion. cleanup. * [TD-3317] : support interlace mode insertion. adjust algorithm of loop times. * [TD-3317] : support interlace mode insertion. fix delay time bug. * [TD-3317] : support interlace mode insertion. fix progressive timestamp bug. * [TD-3317] : support interlace mode insertion. add an option for performance print. * [TD-3317] : support interlace mode insertion. change json test case with less table for acceleration. * [TD-3317] : support interlace mode insertion. change progressive mode timestamp step and testcase. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3317] : support interlace mode insertion. add time shift for no sleep time. * [TD-3317] : support interlace insert. rework timestamp. * [TD-3416] : add taosdemo interlace testcase. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 18 +++++++++++------- tests/pytest/fulltest.sh | 1 + tests/pytest/tools/taosdemoTestInterlace.py | 20 +++++++++++++++----- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 51c94a8fa1..0943e0b3a9 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4287,13 +4287,14 @@ static int generateDataTail(char *tableName, int32_t tableSeq, if ((g_args.disorderRatio != 0) && (rand_num < g_args.disorderRange)) { - int64_t d = startTime - taosRandom() % 1000000 + rand_num; + int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k + - taosRandom() % 1000000 + rand_num; len = generateData(data, data_type, ncols_per_record, d, lenOfBinary); } else { len = generateData(data, data_type, ncols_per_record, - startTime + DEFAULT_TIMESTAMP_STEP * startFrom, + startTime + DEFAULT_TIMESTAMP_STEP * k, lenOfBinary); } @@ -4402,7 +4403,8 @@ static int generateDataBuffer(char *pTblName, int k; int dataLen; k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo, - g_args.num_of_RPR, pstr, insertRows, startFrom, startTime, + g_args.num_of_RPR, pstr, insertRows, startFrom, + startTime, pSamplePos, &dataLen); return k; } @@ -4475,7 +4477,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int generatedRecPerTbl = 0; bool flagSleep = true; int sleepTimeTotal = 0; - int timeShift = 0; while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampUs(); @@ -4513,16 +4514,18 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { generateDataTail( tableName, tableSeq, pThreadInfo, superTblInfo, batchPerTbl, pstr, insertRows, 0, - startTime + timeShift + sleepTimeTotal, + startTime, &(pThreadInfo->samplePos), &dataLen); + pstr += dataLen; recOfBatch += batchPerTbl; + startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; + verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); - timeShift ++; tableSeq ++; if (insertMode == INTERLACE_INSERT_MODE) { if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { @@ -4668,13 +4671,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int generated = generateDataBuffer( tableName, tableSeq, pThreadInfo, buffer, insertRows, - i, start_time + pThreadInfo->totalInsertRows * timeStampStep, + i, start_time, &(pThreadInfo->samplePos)); if (generated > 0) i += generated; else goto free_and_statistics_2; + start_time += generated * timeStampStep; pThreadInfo->totalInsertRows += generated; startTs = taosGetTimestampUs(); diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 46a1abf12c..5037f2c399 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -238,6 +238,7 @@ python3 test.py -f tools/taosdemoTestLimitOffset.py python3 test.py -f tools/taosdumpTest.py python3 test.py -f tools/taosdemoTest2.py python3 test.py -f tools/taosdemoTestSampleData.py +python3 test.py -f tools/taosdemoTestInterlace.py # subscribe python3 test.py -f subscribe/singlemeter.py diff --git a/tests/pytest/tools/taosdemoTestInterlace.py b/tests/pytest/tools/taosdemoTestInterlace.py index 9ceb8c5cbb..953bfff90f 100644 --- a/tests/pytest/tools/taosdemoTestInterlace.py +++ b/tests/pytest/tools/taosdemoTestInterlace.py @@ -17,6 +17,7 @@ from util.log import * from util.cases import * from util.sql import * from util.dnodes import * +import subprocess class TDTestCase: @@ -39,7 +40,7 @@ class TDTestCase: if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + buildPath = root[:len(root) - len("/build/bin")] break return buildPath @@ -50,14 +51,23 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("%staosdemo -f tools/insert-interlace.json" % binPath) + binPath = buildPath + "/build/bin/" + taosdemoCmd = "%staosdemo -f tools/insert-interlace.json -pp 2>&1 | grep sleep | wc -l" % binPath + sleepTimes = subprocess.check_output( + taosdemoCmd, shell=True).decode("utf-8") + print("sleep times: %d" % int(sleepTimes)) + + if (int(sleepTimes) != 16): + caller = inspect.getframeinfo(inspect.stack()[0][0]) + tdLog.exit( + "%s(%d) failed: expected sleep times 16, actual %d" % + (caller.filename, caller.lineno, int(sleepTimes))) tdSql.execute("use db") tdSql.query("select count(tbname) from db.stb") - tdSql.checkData(0, 0, 100) + tdSql.checkData(0, 0, 9) tdSql.query("select count(*) from db.stb") - tdSql.checkData(0, 0, 33000) + tdSql.checkData(0, 0, 2250) def stop(self): tdSql.close() From fb7599b7f4ac97ff1b79231ab400987e4709ba8e Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 22 Mar 2021 16:11:49 +0800 Subject: [PATCH 12/22] fix docker compose error --- tests/pytest/cluster/clusterEnvSetup/Dockerfile | 2 ++ tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh | 1 + tests/pytest/cluster/clusterEnvSetup/docker-compose.yml | 2 ++ tests/pytest/cluster/clusterEnvSetup/node3.yml | 2 ++ tests/pytest/cluster/clusterEnvSetup/node4.yml | 2 ++ tests/pytest/cluster/clusterEnvSetup/node5.yml | 2 ++ 6 files changed, 11 insertions(+) diff --git a/tests/pytest/cluster/clusterEnvSetup/Dockerfile b/tests/pytest/cluster/clusterEnvSetup/Dockerfile index c9c4d79be9..437dbc65e6 100644 --- a/tests/pytest/cluster/clusterEnvSetup/Dockerfile +++ b/tests/pytest/cluster/clusterEnvSetup/Dockerfile @@ -28,6 +28,8 @@ RUN ulimit -c unlimited COPY --from=builder /root/bin/taosd /usr/bin COPY --from=builder /root/bin/tarbitrator /usr/bin +COPY --from=builder /root/bin/taosdemo /usr/bin +COPY --from=builder /root/bin/taosdump /usr/bin COPY --from=builder /root/bin/taos /usr/bin COPY --from=builder /root/cfg/taos.cfg /etc/taos/ COPY --from=builder /root/lib/libtaos.so.* /usr/lib/libtaos.so.1 diff --git a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh index 4cd305d6cd..52ead0675b 100755 --- a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh +++ b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh @@ -110,6 +110,7 @@ function clusterUp { sed -i "s/td2.0-node3/td2.0-node$i/g" node$i.yml sed -i "s/'tdnode3'/'tdnode$i'/g" node$i.yml sed -i "s#/node3/#/node$i/#g" node$i.yml + sed -i "s#/hostname: tdnode3/#/hostname: tdnode$i/#g" node$i.yml sed -i "s#ipv4_address: 172.27.0.9#ipv4_address: 172.27.0.`expr $i + 6`#g" node$i.yml fi docker_run=$docker_run" -f node$i.yml " diff --git a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml index 7b47b5e459..d241062a2d 100644 --- a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml +++ b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml @@ -66,7 +66,9 @@ services: context: . args: - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} - DATADIR=${DATADIR} image: 'tdengine:${VERSION}' container_name: 'tdnode2' diff --git a/tests/pytest/cluster/clusterEnvSetup/node3.yml b/tests/pytest/cluster/clusterEnvSetup/node3.yml index 56fba49bd7..18f1b37c1c 100644 --- a/tests/pytest/cluster/clusterEnvSetup/node3.yml +++ b/tests/pytest/cluster/clusterEnvSetup/node3.yml @@ -6,7 +6,9 @@ services: context: . args: - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} - DATADIR=${DATADIR} image: 'tdengine:${VERSION}' container_name: 'tdnode3' diff --git a/tests/pytest/cluster/clusterEnvSetup/node4.yml b/tests/pytest/cluster/clusterEnvSetup/node4.yml index 2af9fc0293..f542c22c45 100644 --- a/tests/pytest/cluster/clusterEnvSetup/node4.yml +++ b/tests/pytest/cluster/clusterEnvSetup/node4.yml @@ -6,7 +6,9 @@ services: context: . args: - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} - DATADIR=${DATADIR} image: 'tdengine:${VERSION}' container_name: 'tdnode4' diff --git a/tests/pytest/cluster/clusterEnvSetup/node5.yml b/tests/pytest/cluster/clusterEnvSetup/node5.yml index 76d5346a14..dd5941ac76 100644 --- a/tests/pytest/cluster/clusterEnvSetup/node5.yml +++ b/tests/pytest/cluster/clusterEnvSetup/node5.yml @@ -6,7 +6,9 @@ services: context: . args: - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} - DATADIR=${DATADIR} image: 'tdengine:${VERSION}' container_name: 'tdnode5' From d354ce2e55adc07b5415f693350b4a819b718b8b Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 22 Mar 2021 16:18:25 +0800 Subject: [PATCH 13/22] fix --- tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh index 52ead0675b..36fd4197b6 100755 --- a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh +++ b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh @@ -110,7 +110,7 @@ function clusterUp { sed -i "s/td2.0-node3/td2.0-node$i/g" node$i.yml sed -i "s/'tdnode3'/'tdnode$i'/g" node$i.yml sed -i "s#/node3/#/node$i/#g" node$i.yml - sed -i "s#/hostname: tdnode3/#/hostname: tdnode$i/#g" node$i.yml + sed -i "s#hostname: tdnode3#hostname: tdnode$i#g" node$i.yml sed -i "s#ipv4_address: 172.27.0.9#ipv4_address: 172.27.0.`expr $i + 6`#g" node$i.yml fi docker_run=$docker_run" -f node$i.yml " From 21773500795b3cd9e7dd06216c73344252e4163d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Mar 2021 17:32:49 +0800 Subject: [PATCH 14/22] [TD-3430] : cmake minimal version is 2.8.12.x. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0749f3982e..79c140c741 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ For user manual, system design and architecture, engineering blogs, refer to [TD # Building At the moment, TDengine only supports building and running on Linux systems. You can choose to [install from packages](https://www.taosdata.com/en/getting-started/#Install-from-Package) or from the source code. This quick guide is for installation from the source only. -To build TDengine, use [CMake](https://cmake.org/) 3.5 or higher versions in the project directory. +To build TDengine, use [CMake](https://cmake.org/) 2.8.12.x or higher versions in the project directory. ## Install tools From 835886d11e0337078631e7bec476dd89a84c08e4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Mar 2021 18:44:06 +0800 Subject: [PATCH 15/22] Hotfix/sangshuduo/td 3393 taosdemo timestamp rework (#5531) * [TD-3393] : taosdemo timestamp rework. change default timestamp step to 1ms. * [TD-3393] : taosdemo timestamp rework. adjust shift timestamp logic to table sequence turn to start. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0943e0b3a9..b390ee76cd 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -92,7 +92,7 @@ enum TEST_MODE { #define MAX_DATABASE_COUNT 256 #define INPUT_BUF_LEN 256 -#define DEFAULT_TIMESTAMP_STEP 10 +#define DEFAULT_TIMESTAMP_STEP 1 typedef enum CREATE_SUB_TALBE_MOD_EN { PRE_CREATE_SUBTBL, @@ -4519,7 +4519,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pstr += dataLen; recOfBatch += batchPerTbl; - startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", @@ -4530,6 +4529,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (insertMode == INTERLACE_INSERT_MODE) { if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { // turn to first table + startTime += batchPerTbl * superTblInfo->timeStampStep; tableSeq = pThreadInfo->start_table_from; generatedRecPerTbl += batchPerTbl; flagSleep = true; From dfdeb7e0cc1578da9482fcb436672c1021b011f5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 23 Mar 2021 11:15:11 +0800 Subject: [PATCH 16/22] [TD-3431] : fix taosdemo cmdline incorrect arguments. (#5536) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 43 +++++++++---------- tests/pytest/tools/taosdemoTest.py | 2 +- tests/pytest/tools/taosdemoTest2.py | 2 +- .../pytest/tools/taosdemoTestWithoutMetric.py | 2 +- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index b390ee76cd..6534196603 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -523,8 +523,8 @@ SArguments g_args = { 1, // replica "t", // tb_prefix NULL, // sqlFile - false, // use_metric - false, // insert_only + true, // use_metric + true, // insert_only false, // debug_print false, // verbose_print false, // performance statistic print @@ -614,7 +614,7 @@ static void printHelp() { printf("%s%s%s%s\n", indent, "-m", indent, "Table prefix name. Default is 't'."); printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file."); - printf("%s%s%s%s\n", indent, "-M", indent, "Use metric flag."); + printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag."); printf("%s%s%s%s\n", indent, "-o", indent, "Direct output to the named file. Default is './output.txt'."); printf("%s%s%s%s\n", indent, "-q", indent, @@ -742,10 +742,10 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->len_of_binary = atoi(argv[++i]); } else if (strcmp(argv[i], "-m") == 0) { arguments->tb_prefix = argv[++i]; - } else if (strcmp(argv[i], "-M") == 0) { - arguments->use_metric = true; + } else if (strcmp(argv[i], "-N") == 0) { + arguments->use_metric = false; } else if (strcmp(argv[i], "-x") == 0) { - arguments->insert_only = true; + arguments->insert_only = false; } else if (strcmp(argv[i], "-y") == 0) { arguments->answer_yes = true; } else if (strcmp(argv[i], "-g") == 0) { @@ -2576,12 +2576,12 @@ static int startMultiThreadCreateChildTable( t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; startFrom = t_info->end_table_to + 1; - t_info->use_metric = 1; + t_info->use_metric = true; t_info->cols = cols; t_info->minDelay = INT16_MAX; pthread_create(pids + i, NULL, createTable, t_info); } - + for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); } @@ -2592,12 +2592,11 @@ static int startMultiThreadCreateChildTable( } free(pids); - free(infos); + free(infos); return 0; } - static void createChildTables() { char tblColsBuf[MAX_SQL_SIZE]; int len; @@ -6150,13 +6149,13 @@ static void queryResult() { static void testCmdLine() { - g_args.test_mode = INSERT_TEST; - insertTestProcess(); + g_args.test_mode = INSERT_TEST; + insertTestProcess(); - if (g_Dbs.insert_only) - return; - else - queryResult(); + if (g_Dbs.insert_only) + return; + else + queryResult(); } int main(int argc, char *argv[]) { @@ -6166,7 +6165,7 @@ int main(int argc, char *argv[]) { if (g_args.metaFile) { initOfInsertMeta(); - initOfQueryMeta(); + initOfQueryMeta(); if (false == getInfoFromJsonFile(g_args.metaFile)) { printf("Failed to read %s\n", g_args.metaFile); @@ -6180,12 +6179,12 @@ int main(int argc, char *argv[]) { if (NULL != g_args.sqlFile) { TAOS* qtaos = taos_connect( - g_Dbs.host, - g_Dbs.user, - g_Dbs.password, - g_Dbs.db[0].dbName, + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, g_Dbs.port); - querySqlFile(qtaos, g_args.sqlFile); + querySqlFile(qtaos, g_args.sqlFile); taos_close(qtaos); } else { diff --git a/tests/pytest/tools/taosdemoTest.py b/tests/pytest/tools/taosdemoTest.py index 2ce9228c54..ff5921be60 100644 --- a/tests/pytest/tools/taosdemoTest.py +++ b/tests/pytest/tools/taosdemoTest.py @@ -51,7 +51,7 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath + "/build/bin/" - os.system("%staosdemo -y -M -t %d -n %d -x" % + os.system("%staosdemo -y -t %d -n %d" % (binPath, self.numberOfTables, self.numberOfRecords)) tdSql.execute("use test") diff --git a/tests/pytest/tools/taosdemoTest2.py b/tests/pytest/tools/taosdemoTest2.py index 75a79d0585..92f6fa1265 100644 --- a/tests/pytest/tools/taosdemoTest2.py +++ b/tests/pytest/tools/taosdemoTest2.py @@ -31,7 +31,7 @@ class TDTestCase: def insertDataAndAlterTable(self, threadID): if(threadID == 0): - os.system("taosdemo -M -y -t %d -n %d -x" % + os.system("taosdemo -y -t %d -n %d" % (self.numberOfTables, self.numberOfRecords)) if(threadID == 1): time.sleep(2) diff --git a/tests/pytest/tools/taosdemoTestWithoutMetric.py b/tests/pytest/tools/taosdemoTestWithoutMetric.py index 647d6a37cb..9687600563 100644 --- a/tests/pytest/tools/taosdemoTestWithoutMetric.py +++ b/tests/pytest/tools/taosdemoTestWithoutMetric.py @@ -50,7 +50,7 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath + "/build/bin/" - os.system("%staosdemo -y -t %d -n %d -x" % + os.system("%staosdemo -N -y -t %d -n %d" % (binPath, self.numberOfTables, self.numberOfRecords)) tdSql.query("show databases") From 3b97bd716df4efa805367be8e020999c8c9e4657 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 23 Mar 2021 15:14:50 +0800 Subject: [PATCH 17/22] Feature/sangshuduo/td 3317 taosdemo interlace (#5537) * [TD-3316] : add testcase for taosdemo limit and offset. check offset 0. * [TD-3316] : add testcase for taosdemo limit and offset. fix sample file import bug. * [TD-3316] : add test case for limit and offset. fix sample data issue. * [TD-3327] : fix taosdemo segfault when import data from sample data file. * [TD-3317] : make taosdemo support interlace mode. json parameter rows_per_tbl support. * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode insertion. refactor. * [TD-3317] : support interlace mode insertion. change json file. * [TD-3317] : support interlace mode insertion. fix multithread create table regression. * [TD-3317] : support interlace mode insertion. working but not perfect. * [TD-3317] : support interlace mode insertion. rename lowaTest with taosdemoTestWithJson * [TD-3317] : support interlace mode insertion. perfect * [TD-3317] : support interlace mode insertion. cleanup. * [TD-3317] : support interlace mode insertion. adjust algorithm of loop times. * [TD-3317] : support interlace mode insertion. fix delay time bug. * [TD-3317] : support interlace mode insertion. fix progressive timestamp bug. * [TD-3317] : support interlace mode insertion. add an option for performance print. * [TD-3317] : support interlace mode insertion. change json test case with less table for acceleration. * [TD-3317] : support interlace mode insertion. change progressive mode timestamp step and testcase. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3317] : support interlace mode insertion. add time shift for no sleep time. * [TD-3317] : support interlace insert. rework timestamp. * [TD-3317] : support interlace mode insertion. change rows_per_tbl to interlace_rows. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/insert-interlace.json | 2 +- src/kit/taosdemo/insert.json | 2 +- src/kit/taosdemo/taosdemo.c | 21 ++++++++++--------- .../taosdemo/src/main/resources/insert.json | 2 +- .../cluster/clusterEnvSetup/insert.json | 2 +- tests/pytest/tools/insert-interlace.json | 4 ++-- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/kit/taosdemo/insert-interlace.json b/src/kit/taosdemo/insert-interlace.json index 0f54f008fb..344db4fd00 100644 --- a/src/kit/taosdemo/insert-interlace.json +++ b/src/kit/taosdemo/insert-interlace.json @@ -41,7 +41,7 @@ "insert_mode": "taosc", "insert_rows": 1000, "multi_thread_write_one_tbl": "no", - "rows_per_tbl": 20, + "interlace_rows": 20, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/src/kit/taosdemo/insert.json b/src/kit/taosdemo/insert.json index e6b1895043..f0e3ab1d50 100644 --- a/src/kit/taosdemo/insert.json +++ b/src/kit/taosdemo/insert.json @@ -41,7 +41,7 @@ "insert_mode": "taosc", "insert_rows": 100000, "multi_thread_write_one_tbl": "no", - "rows_per_tbl": 0, + "interlace_rows": 0, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6534196603..581405efcb 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -199,7 +199,7 @@ typedef struct SArguments_S { int num_of_CPR; int num_of_threads; int insert_interval; - int rows_per_tbl; + int interlace_rows; int num_of_RPR; int max_sql_len; int num_of_tables; @@ -547,7 +547,7 @@ SArguments g_args = { 10, // num_of_CPR 10, // num_of_connections/thread 0, // insert_interval - 0, // rows_per_tbl; + 0, // interlace_rows; 100, // num_of_RPR TSDB_PAYLOAD_SIZE, // max_sql_len 10000, // num_of_tables @@ -682,7 +682,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } else if (strcmp(argv[i], "-i") == 0) { arguments->insert_interval = atoi(argv[++i]); } else if (strcmp(argv[i], "-B") == 0) { - arguments->rows_per_tbl = atoi(argv[++i]); + arguments->interlace_rows = atoi(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { arguments->num_of_RPR = atoi(argv[++i]); } else if (strcmp(argv[i], "-t") == 0) { @@ -3008,13 +3008,13 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* rowsPerTbl = cJSON_GetObjectItem(root, "rows_per_tbl"); + cJSON* rowsPerTbl = cJSON_GetObjectItem(root, "interlace_rows"); if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) { - g_args.rows_per_tbl = rowsPerTbl->valueint; + g_args.interlace_rows = rowsPerTbl->valueint; } else if (!rowsPerTbl) { - g_args.rows_per_tbl = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req + g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { - errorPrint("%s() LN%d, failed to read json, rows_per_tbl input mistake\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", __func__, __LINE__); goto PARSE_OVER; } @@ -3498,7 +3498,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "rows_per_tbl"); + cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "interlace_rows"); if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].rowsPerTbl = rowsPerTbl->valueint; } else if (!rowsPerTbl) { @@ -4425,7 +4425,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int insertMode; char tableName[TSDB_TABLE_NAME_LEN]; - int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.rows_per_tbl; + int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.interlace_rows; if (rowsPerTbl > 0) { insertMode = INTERLACE_INSERT_MODE; @@ -4518,6 +4518,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pstr += dataLen; recOfBatch += batchPerTbl; + startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", @@ -4746,7 +4747,7 @@ static void* syncWrite(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; SSuperTable* superTblInfo = winfo->superTblInfo; - int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.rows_per_tbl; + int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.interlace_rows; if (rowsPerTbl > 0) { // interlace mode diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/insert.json b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json index a7bd87e6d3..35c7773175 100644 --- a/tests/examples/JDBC/taosdemo/src/main/resources/insert.json +++ b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json @@ -38,7 +38,7 @@ "insert_rows": 100, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 0, - "rows_per_tbl": 3, + "interlace_rows": 3, "max_sql_len": 1024, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/tests/pytest/cluster/clusterEnvSetup/insert.json b/tests/pytest/cluster/clusterEnvSetup/insert.json index 56a64b7b85..4548ef74e6 100644 --- a/tests/pytest/cluster/clusterEnvSetup/insert.json +++ b/tests/pytest/cluster/clusterEnvSetup/insert.json @@ -39,7 +39,7 @@ "insert_rows": 100000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 1, - "rows_per_tbl": 100, + "interlace_rows": 100, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/tests/pytest/tools/insert-interlace.json b/tests/pytest/tools/insert-interlace.json index a2ff2c001c..d4767ad064 100644 --- a/tests/pytest/tools/insert-interlace.json +++ b/tests/pytest/tools/insert-interlace.json @@ -10,7 +10,7 @@ "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 5000, - "rows_per_tbl": 50, + "interlace_rows": 50, "num_of_records_per_req": 100, "max_sql_len": 1024000, "databases": [{ @@ -42,7 +42,7 @@ "insert_mode": "taosc", "insert_rows": 250, "multi_thread_write_one_tbl": "no", - "rows_per_tbl": 80, + "interlace_rows": 80, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, From 03b282e92febceb9cc324576f36406f01094334a Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 23 Mar 2021 15:22:34 +0800 Subject: [PATCH 18/22] [TD-2639] : update memory size estimation. --- documentation20/cn/11.administrator/docs.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index 027828d903..c2c2927387 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -6,19 +6,27 @@ ### 内存需求 -每个 DB 可以创建固定数目的 vgroup,默认与 CPU 核数相同,可通过 maxVgroupsPerDb 配置;vgroup 中的每个副本会是一个 vnode;每个 vnode 会占用固定大小的内存(大小与数据库的配置参数 blocks 和 cache 有关);每个 Table 会占用与标签总长度有关的内存;此外,系统会有一些固定的内存开销。因此,每个 DB 需要的系统内存可通过如下公式计算: +每个 Database 可以创建固定数目的 vgroup,默认与 CPU 核数相同,可通过 maxVgroupsPerDb 配置;vgroup 中的每个副本会是一个 vnode;每个 vnode 会占用固定大小的内存(大小与数据库的配置参数 blocks 和 cache 有关);每个 Table 会占用与标签总长度有关的内存;此外,系统会有一些固定的内存开销。因此,每个 DB 需要的系统内存可通过如下公式计算: ``` -Memory Size = maxVgroupsPerDb * (blocks * cache + 10MB) + numOfTables * (tagSizePerTable + 0.5KB) +Database Memory Size = maxVgroupsPerDb * (blocks * cache + 10MB) + numOfTables * (tagSizePerTable + 0.5KB) ``` -示例:假设是 4 核机器,cache 是缺省大小 16M, blocks 是缺省值 6,假设有 10 万张表,标签总长度是 256 字节,则总的内存需求为:4 \* (16 \* 6 + 10) + 100000 \* (0.25 + 0.5) / 1000 = 499M。 +示例:假设是 4 核机器,cache 是缺省大小 16M, blocks 是缺省值 6,并且一个 DB 中有 10 万张表,标签总长度是 256 字节,则这个 DB 总的内存需求为:4 \* (16 \* 6 + 10) + 100000 \* (0.25 + 0.5) / 1000 = 499M。 -注意:从这个公式计算得到的内存容量,应理解为系统的“必要需求”,而不是“内存总数”。在实际运行的生产系统中,由于操作系统缓存、资源管理调度等方面的需要,内存规划应当在计算结果的基础上保留一定冗余,以维持系统状态和系统性能的稳定性。 +在实际的系统运维中,我们通常会更关心 TDengine 服务进程(taosd)会占用的内存量。 +``` +taosd 内存总量 = vnode 内存 + mnode 内存 + 查询内存 +``` -实际运行的系统往往会根据数据特点的不同,将数据存放在不同的 DB 里。因此做规划时,也需要考虑。 +其中: +1. “vnode 内存”指的是集群中所有的 Database 存储分摊到当前 taosd 节点上所占用的内存资源。可以按上文“Database Memory Size”计算公式估算每个 DB 的内存占用量进行加总,再按集群中总共的 TDengine 节点数做平均(如果设置为多副本,则还需要乘以对应的副本倍数)。 +2. “mnode 内存”指的是集群中管理节点所占用的资源。如果一个 taosd 节点上分布有 mnode 管理节点,则内存消耗还需要增加“0.2KB * 集群中数据表总数”。 +3. “查询内存”指的是服务端处理查询请求时所需要占用的内存。单条查询语句至少会占用“0.2KB * 查询涉及的数据表总数”的内存量。 -如果内存充裕,可以加大 Blocks 的配置,这样更多数据将保存在内存里,提高查询速度。 +注意:以上内存估算方法,主要讲解了系统的“必须内存需求”,而不是“内存总数上限”。在实际运行的生产环境中,由于操作系统缓存、资源管理调度等方面的原因,内存规划应当在估算结果的基础上保留一定冗余,以维持系统状态和系统性能的稳定性。并且,生产环境通常会配置系统资源的监控工具,以便及时发现硬件资源的紧缺情况。 + +最后,如果内存充裕,可以考虑加大 Blocks 的配置,这样更多数据将保存在内存里,提高查询速度。 ### CPU 需求 From 94bb5fe80d58d4f063616d800ff015a1960bc322 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 23 Mar 2021 15:28:25 +0800 Subject: [PATCH 19/22] [TD-3407] : print more info to result file. (#5538) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 581405efcb..5a65e3dc5e 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1064,6 +1064,7 @@ static int printfInsertMeta() { printf("max sql length: \033[33m%d\033[0m\n", g_args.max_sql_len); printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { printf("database[\033[33m%d\033[0m]:\n", i); printf(" database[%d] name: \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName); @@ -1220,16 +1221,19 @@ static int printfInsertMeta() { } static void printfInsertMetaToFile(FILE* fp) { - SHOW_PARSE_RESULT_START_TO_FILE(fp); + + SHOW_PARSE_RESULT_START_TO_FILE(fp); fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); fprintf(fp, "user: %s\n", g_Dbs.user); - fprintf(fp, "password: %s\n", g_Dbs.password); fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); - + fprintf(fp, "insert interval: %d\n", g_args.insert_interval); + fprintf(fp, "number of records per req: %d\n", g_args.num_of_RPR); + fprintf(fp, "max sql length: %d\n", g_args.max_sql_len); fprintf(fp, "database count: %d\n", g_Dbs.dbCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { fprintf(fp, "database[%d]:\n", i); fprintf(fp, " database[%d] name: %s\n", i, g_Dbs.db[i].dbName); @@ -1364,11 +1368,14 @@ static void printfInsertMetaToFile(FILE* fp) { } fprintf(fp, "\n"); } + SHOW_PARSE_RESULT_END_TO_FILE(fp); } static void printfQueryMeta() { + SHOW_PARSE_RESULT_START(); + printf("host: \033[33m%s:%u\033[0m\n", g_queryInfo.host, g_queryInfo.port); printf("user: \033[33m%s\033[0m\n", g_queryInfo.user); @@ -1411,11 +1418,11 @@ static void printfQueryMeta() { } printf("\n"); - SHOW_PARSE_RESULT_END(); + SHOW_PARSE_RESULT_END(); } -static char* xFormatTimestamp(char* buf, int64_t val, int precision) { +static char* formatTimestamp(char* buf, int64_t val, int precision) { time_t tt; if (precision == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)(val / 1000000); @@ -1447,7 +1454,9 @@ static char* xFormatTimestamp(char* buf, int64_t val, int precision) { return buf; } -static void xDumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) { +static void xDumpFieldToFile(FILE* fp, const char* val, + TAOS_FIELD* field, int32_t length, int precision) { + if (val == NULL) { fprintf(fp, "%s", TSDB_DATA_NULL_STR); return; @@ -1483,7 +1492,7 @@ static void xDumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32 fprintf(fp, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: - xFormatTimestamp(buf, *(int64_t*)val, precision); + formatTimestamp(buf, *(int64_t*)val, precision); fprintf(fp, "'%s'", buf); break; default: @@ -1562,7 +1571,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); - xFormatTimestamp(dbInfos[count]->create_time, + formatTimestamp(dbInfos[count]->create_time, *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI); dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); From 64b8620bb393233e5d862064e9ad6a8f8d3d198b Mon Sep 17 00:00:00 2001 From: Xiaxin Li Date: Tue, 23 Mar 2021 15:28:33 +0800 Subject: [PATCH 20/22] Update README.md Add User List --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 79c140c741..b4c37f2624 100644 --- a/README.md +++ b/README.md @@ -250,3 +250,6 @@ Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to th Add WeChat “tdengine” to join the group,you can communicate with other users. +# [User List](https://github.com/taosdata/TDengine/issues/2432) + +If you are using TDengine and think that it helps you or want do some contributions to it, please add your company to to the [user list](https://github.com/taosdata/TDengine/issues/2432) to let us know your needs. From 89016ef4a49bf3abed9527859eb2fd867abec288 Mon Sep 17 00:00:00 2001 From: Xiaxin Li Date: Tue, 23 Mar 2021 15:34:39 +0800 Subject: [PATCH 21/22] Update README-CN.md Add User List --- README-CN.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README-CN.md b/README-CN.md index 9601cde3af..112ba0e4ea 100644 --- a/README-CN.md +++ b/README-CN.md @@ -263,5 +263,8 @@ TDengine 的测试框架和所有测试例全部开源。 # 成为社区贡献者 点击[这里](https://www.taosdata.com/cn/contributor/),了解如何成为 TDengine 的贡献者。 -#加入技术交流群 +# 加入技术交流群 TDengine官方社群「物联网大数据群」对外开放,欢迎您加入讨论。搜索微信号 "tdengine",加小T为好友,即可入群。 + +# [谁在使用TDengine](https://github.com/taosdata/TDengine/issues/2432) +欢迎所有TDengine用户及贡献者在[这里](https://github.com/taosdata/TDengine/issues/2432)下分享您在当前工作中开发/使用TDengine的故事。 From ff5dcfddbef96f54662292c67a98cad95932b7c9 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 23 Mar 2021 15:57:26 +0800 Subject: [PATCH 22/22] [TD-2639] : fix minor typo. --- README-CN.md | 11 +++++++---- README.md | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README-CN.md b/README-CN.md index 112ba0e4ea..d4c10e71d6 100644 --- a/README-CN.md +++ b/README-CN.md @@ -258,13 +258,16 @@ TDengine 社区生态中也有一些非常友好的第三方连接器,可以 TDengine 的测试框架和所有测试例全部开源。 -点击[这里](tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md),了解如何运行测试例和添加新的测试例。 +点击 [这里](tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md),了解如何运行测试例和添加新的测试例。 # 成为社区贡献者 -点击[这里](https://www.taosdata.com/cn/contributor/),了解如何成为 TDengine 的贡献者。 + +点击 [这里](https://www.taosdata.com/cn/contributor/),了解如何成为 TDengine 的贡献者。 # 加入技术交流群 -TDengine官方社群「物联网大数据群」对外开放,欢迎您加入讨论。搜索微信号 "tdengine",加小T为好友,即可入群。 + +TDengine 官方社群「物联网大数据群」对外开放,欢迎您加入讨论。搜索微信号 "tdengine",加小T为好友,即可入群。 # [谁在使用TDengine](https://github.com/taosdata/TDengine/issues/2432) -欢迎所有TDengine用户及贡献者在[这里](https://github.com/taosdata/TDengine/issues/2432)下分享您在当前工作中开发/使用TDengine的故事。 + +欢迎所有 TDengine 用户及贡献者在 [这里](https://github.com/taosdata/TDengine/issues/2432) 分享您在当前工作中开发/使用 TDengine 的故事。 diff --git a/README.md b/README.md index b4c37f2624..45a955f458 100644 --- a/README.md +++ b/README.md @@ -252,4 +252,4 @@ Add WeChat “tdengine” to join the group,you can communicate with other use # [User List](https://github.com/taosdata/TDengine/issues/2432) -If you are using TDengine and think that it helps you or want do some contributions to it, please add your company to to the [user list](https://github.com/taosdata/TDengine/issues/2432) to let us know your needs. +If you are using TDengine and feel it helps or you'd like to do some contributions, please add your company to [user list](https://github.com/taosdata/TDengine/issues/2432) and let us know your needs.