From ba3ad7c662c748e49111133685a187f98763e684 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 25 Jun 2021 16:28:57 +0800 Subject: [PATCH 1/8] [TD-4826]: new error code 0x0406 (Dnode is exiting) for tsc --- src/dnode/src/dnodeShell.c | 9 ++++++++- src/inc/taoserror.h | 1 + src/util/src/terror.c | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 50343cfd32..7bc1fd9140 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -116,7 +116,14 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pMsg->pCont == NULL) return; - if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { + SRunStatus dnodeStatus = dnodeGetRunStatus(); + if (dnodeStatus == TSDB_RUN_STATUS_STOPPED) { + dError("RPC %p, shell msg:%s is ignored since dnode exiting", pMsg->handle, taosMsg[pMsg->msgType]); + rpcMsg.code = TSDB_CODE_DND_EXITING; + rpcSendResponse(&rpcMsg); + rpcFreeCont(pMsg->pCont); + return; + } else if (dnodeStatus != TSDB_RUN_STATUS_RUNING) { dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); rpcMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rpcMsg); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index bfeb53513b..619869efa5 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -198,6 +198,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0403) //"Invalid message length") #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0404) //"Action in progress") #define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) //"Too many vnode directories") +#define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) //"Dnode is exiting" // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 9594022d3a..3d527cc1a2 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -210,6 +210,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, "No permission for dis TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length") TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_TOO_MANY_VNODES, "Too many vnode directories") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING, "Dnode is exiting") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, "Action in progress") From ae1fcd260a935106f32055bc1c18b650518ae85e Mon Sep 17 00:00:00 2001 From: Baosheng Chang Date: Fri, 25 Jun 2021 18:52:40 +0800 Subject: [PATCH 2/8] update master to develop's version --- tests/pytest/insert/boundary2.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/pytest/insert/boundary2.py b/tests/pytest/insert/boundary2.py index 99784b7cd9..8a6fd1e6a1 100644 --- a/tests/pytest/insert/boundary2.py +++ b/tests/pytest/insert/boundary2.py @@ -50,14 +50,16 @@ class TDTestCase: sql += "'%s')" % self.get_random_string(22) tdSql.execute(sql % (self.ts + i)) - tdSql.query("select * from stb") - tdSql.checkRows(4096) + time.sleep(10) + tdSql.query("select count(*) from stb") + tdSql.checkData(0, 0, 4096) tdDnodes.stop(1) tdDnodes.start(1) - - tdSql.query("select * from stb") - tdSql.checkRows(4096) + + time.sleep(1) + tdSql.query("select count(*) from stb") + tdSql.checkData(0, 0, 4096) endTime = time.time() From 8f6783f6761cefc5e1c5218da527540af0d6c1fa Mon Sep 17 00:00:00 2001 From: Baosheng Chang Date: Fri, 25 Jun 2021 19:05:53 +0800 Subject: [PATCH 3/8] Update boundary2.py From 6b265fb3b8203e3fa1a0de60fca09107d416ff57 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 26 Jun 2021 11:32:11 +0800 Subject: [PATCH 4/8] [TD-4912]: fix coredump when drop vnode --- src/tsdb/src/tsdbMain.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 99929f3542..5f59f776b9 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -126,6 +126,8 @@ int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { tsdbSyncCommit(repo); } + tsem_wait(&(pRepo->readyToCommit)); + tsdbUnRefMemTable(pRepo, pRepo->mem); tsdbUnRefMemTable(pRepo, pRepo->imem); pRepo->mem = NULL; From a2c1754168c35840a2c31c765c674d5465f1a9e7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 26 Jun 2021 22:34:30 +0800 Subject: [PATCH 5/8] Hotfix/sangshuduo/td 4892 taosdemo sub fetch (#6634) * [TD-4892]: taosdemo subscribe fetch result. * fix stbname length. * restrict prefix length. * submit empty * fix minor code. * fix crash if no result file. --- src/kit/taosdemo/taosdemo.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index abf3c74436..b5781c6070 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1204,23 +1204,24 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { return ; } - int totalLen = 0; - char temp[16000]; + int64_t totalLen = 0; // fetch the records row by row while((row = taos_fetch_row(res))) { - if ((strlen(pThreadInfo->filePath) > 0) - && (totalLen >= 100*1024*1024 - 32000)) { - appendResultBufToFile(databuf, pThreadInfo); + if (totalLen >= 100*1024*1024 - 32000) { + if (strlen(pThreadInfo->filePath) > 0) + appendResultBufToFile(databuf, pThreadInfo); totalLen = 0; memset(databuf, 0, 100*1024*1024); } num_rows++; + char temp[16000] = {0}; int len = taos_print_row(temp, row, fields, num_fields); len += sprintf(temp + len, "\n"); //printf("query result:%s\n", temp); memcpy(databuf + totalLen, temp, len); totalLen += len; + debugPrint("totalLen: %"PRId64"\n", totalLen); } verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", From 55d3e7a2ea682a682b5d81ac55f9bb8dafd58ee7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 27 Jun 2021 21:45:18 +0800 Subject: [PATCH 6/8] Hotfix/sangshuduo/td 4902 for mastere (#6623) * [TD-4902] FIX with 1 char * Fix prompt of -m @ taosdemo Co-authored-by: SunShine Chan Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index b5781c6070..743c4ac668 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -982,9 +982,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->len_of_binary = atoi(argv[++i]); } else if (strcmp(argv[i], "-m") == 0) { if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + (isStringNumber(argv[i+1]))) { printHelp(); - errorPrint("%s", "\n\t-m need a number following!\n"); + errorPrint("%s", "\n\t-m need a letter-initial string following!\n"); exit(EXIT_FAILURE); } arguments->tb_prefix = argv[++i]; From eadaa5341801a43dc064ffe75ff6774f3cd1005a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 28 Jun 2021 11:45:53 +0800 Subject: [PATCH 7/8] Hotfix/sangshuduo/td 3801 taosdump coverity scan issue (#6642) * [TD-3801]: taosdump coverity scan issue. * merge from commit: 069169e7c1 and fix coverity scan issue. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- src/kit/taosdump/taosdump.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 743c4ac668..d573ba8ea9 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5098,7 +5098,7 @@ static int32_t generateStbDataTail( } else { retLen = getRowDataFromSample( data, - remainderBufLen, + remainderBufLen < MAX_DATA_SIZE ? remainderBufLen : MAX_DATA_SIZE, startTime + superTblInfo->timeStampStep * k, superTblInfo, pSamplePos); diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 8525496b0e..aac406a742 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -29,6 +29,9 @@ #define COMMAND_SIZE 65536 //#define DEFAULT_DUMP_FILE "taosdump.sql" +// for strncpy buffer overflow +#define min(a, b) (((a) < (b)) ? (a) : (b)) + int converStringToReadable(char *str, int size, char *buf, int bufsize); int convertNCharToReadable(char *str, int size, char *buf, int bufsize); void taosDumpCharset(FILE *fp); @@ -1119,12 +1122,11 @@ int taosGetTableDes( TAOS_FIELD *fields = taos_fetch_fields(res); tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); - while ((row = taos_fetch_row(res)) != NULL) { strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); @@ -1575,7 +1577,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); + min(TSDB_TABLE_NAME_LEN, fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes)); taosWrite(fd, &tableRecord, sizeof(STableRecord)); From c504199d90f1e89fb80eb870a78efeb2ef238f64 Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Mon, 28 Jun 2021 15:47:18 +0800 Subject: [PATCH 8/8] [TD-4932]fix jenkins errors on master branch (#6645) * [TD-4932]fix jenkins errors on master branch * fix * fix * fix * fix --- Jenkinsfile | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6b6ef420e6..d685df3d0c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -119,16 +119,32 @@ pipeline { abortPreviousBuilds() } sh''' + rm -rf ${WORKSPACE}.tes cp -r ${WORKSPACE} ${WORKSPACE}.tes cd ${WORKSPACE}.tes - git checkout develop - git pull + + ''' + script { + if (env.CHANGE_TARGET == 'master') { + sh ''' + git checkout master + git pull origin master + ''' + } + else { + sh ''' + git checkout develop + git pull origin develop + ''' + } + } + sh''' git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD ''' script{ - env.skipstage=sh(script:"cd ${WORKSPACE}.tes && git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 0 ",returnStdout:true) + env.skipstage=sh(script:"cd ${WORKSPACE}.tes && git --no-pager diff --name-only FETCH_HEAD ${env.CHANGE_TARGET}|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 0 ",returnStdout:true) } println env.skipstage sh'''