Merge branch 'main' of https://github.com/taosdata/TDengine into 3.0.3.00302

Jenkinsfile2

@@ -40,7 +40,7 @@ def check_docs() {
         sh '''
             cd ${WKC}
             git reset --hard
-            git clean -fxd
+            git clean -f
             rm -rf examples/rust/
             git remote prune origin
             git fetch
@@ -86,7 +86,7 @@ def pre_test(){
         git fetch
         cd ${WKC}
         git reset --hard
-        git clean -fxd
+        git clean -f
         rm -rf examples/rust/
         git remote prune origin
         git fetch
@@ -201,7 +201,7 @@ def pre_test_win(){
     '''
     bat '''
         cd %WIN_COMMUNITY_ROOT%
-        git clean -fxd
+        git clean -f
         git reset --hard
         git remote prune origin
         git fetch

source/dnode/vnode/src/inc/tsdb.h

@@ -766,6 +766,7 @@ typedef struct SCacheRowsReader {
   TdThreadMutex      readerMutex;
   SVnode            *pVnode;
   STSchema          *pSchema;
+  STSchema          *pCurrSchema;
   uint64_t           uid;
   uint64_t           suid;
   char             **transferBuf;  // todo remove it soon

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -1390,17 +1390,57 @@ _err:
   return code;
 }
 
-static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr) {
+static int32_t cloneTSchema(STSchema *pSrc, STSchema **ppDst) {
-  int32_t code = 0;
+  int32_t len = sizeof(STSchema) + sizeof(STColumn) * pSrc->numOfCols;
+  *ppDst = taosMemoryMalloc(len);
+  if (NULL == *ppDst) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  memcpy(*ppDst, pSrc, len);
+  return TSDB_CODE_SUCCESS;
+}
 
+static int32_t updateTSchema(int32_t sversion, SCacheRowsReader *pReader, uint64_t uid) {
+  if (NULL == pReader->pCurrSchema && sversion == pReader->pSchema->version) {
+    return cloneTSchema(pReader->pSchema, &pReader->pCurrSchema);
+  }
+
+  if (NULL != pReader->pCurrSchema && sversion == pReader->pCurrSchema->version) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  taosMemoryFreeClear(pReader->pCurrSchema);
+  return metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pCurrSchema);
+}
+
+static int32_t initLastColArray(STSchema *pTSchema, SArray **ppColArray) {
+  SArray *pColArray = taosArrayInit(pTSchema->numOfCols, sizeof(SLastCol));
+  if (NULL == pColArray) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  for (int32_t i = 0; i < pTSchema->numOfCols; ++i) {
+    SLastCol col = {.ts = 0, .colVal = COL_VAL_NULL(pTSchema->columns[i].colId, pTSchema->columns[i].type)};
+    taosArrayPush(pColArray, &col);
+  }
+  *ppColArray = pColArray;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr) {
   STSchema *pTSchema = pr->pSchema;  // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1);
-  int16_t   nCol = pTSchema->numOfCols;
+  int16_t   nLastCol = pTSchema->numOfCols;
-  int16_t   iCol = 0;
   int16_t   noneCol = 0;
   bool      setNoneCol = false;
-  SArray   *pColArray = taosArrayInit(nCol, sizeof(SLastCol));
+  bool      hasRow = false;
+  SArray   *pColArray = NULL;
   SColVal  *pColVal = &(SColVal){0};
 
+  int32_t code = initLastColArray(pTSchema, &pColArray);
+  if (TSDB_CODE_SUCCESS != code) {
+    return code;
+  }
+
   TSKEY lastRowTs = TSKEY_MAX;
 
   CacheNextRowIter iter = {0};
@@ -1415,6 +1455,15 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
       break;
     }
 
+    hasRow = true;
+
+    code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid);
+    if (TSDB_CODE_SUCCESS != code) {
+      goto _err;
+    }
+    pTSchema = pr->pCurrSchema;
+    int16_t nCol = pTSchema->numOfCols;
+
     TSKEY rowTs = TSDBROW_TS(pRow);
 
     if (lastRowTs == TSKEY_MAX) {
@@ -1422,28 +1471,27 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
       STColumn *pTColumn = &pTSchema->columns[0];
 
       *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs});
-      if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
+      taosArraySet(pColArray, 0, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal});
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _err;
-      }
 
-      for (iCol = 1; iCol < nCol; ++iCol) {
+      for (int16_t iCol = 1; iCol < nCol; ++iCol) {
+        if (iCol >= nLastCol) {
+          break;
+        }
+        SLastCol *pCol = taosArrayGet(pColArray, iCol);
+        if (pCol->colVal.cid != pTSchema->columns[iCol].colId) {
+          continue;
+        }
         tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
 
-        SLastCol lastCol = {.ts = lastRowTs, .colVal = *pColVal};
+        *pCol = (SLastCol){.ts = lastRowTs, .colVal = *pColVal};
         if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) {
-          lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData);
+          pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
-          if (lastCol.colVal.value.pData == NULL) {
+          if (pCol->colVal.value.pData == NULL) {
             terrno = TSDB_CODE_OUT_OF_MEMORY;
             code = TSDB_CODE_OUT_OF_MEMORY;
             goto _err;
           }
-          memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
+          memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
-        }
-
-        if (taosArrayPush(pColArray, &lastCol) == NULL) {
-          code = TSDB_CODE_OUT_OF_MEMORY;
-          goto _err;
         }
 
         if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
@@ -1461,9 +1509,15 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
 
     // merge into pColArray
     setNoneCol = false;
-    for (iCol = noneCol; iCol < nCol; ++iCol) {
+    for (int16_t iCol = noneCol; iCol < nCol; ++iCol) {
+      if (iCol >= nLastCol) {
+        break;
+      }
       // high version's column value
       SLastCol *lastColVal = (SLastCol *)taosArrayGet(pColArray, iCol);
+      if (lastColVal->colVal.cid != pTSchema->columns[iCol].colId) {
+        continue;
+      }
       SColVal *tColVal = &lastColVal->colVal;
 
       tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
@@ -1494,6 +1548,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
   //*ppLastArray = NULL;
   // taosArrayDestroy(pColArray);
   //} else {
+  if (!hasRow) {
+    taosArrayClear(pColArray);
+  }
   *ppLastArray = pColArray;
   //}
 

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -209,6 +209,8 @@ void* tsdbCacherowsReaderClose(void* pReader) {
     taosMemoryFree(p->pSchema);
   }
 
+  taosMemoryFree(p->pCurrSchema);
+
   destroyLastBlockLoadInfo(p->pLoadInfo);
 
   taosMemoryFree((void*)p->idstr);
@@ -303,7 +305,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
 
   for (int32_t i = 0; i < pr->pSchema->numOfCols; ++i) {
     struct STColumn* pCol = &pr->pSchema->columns[i];
-    SLastCol         p = {.ts = INT64_MIN, .colVal.type = pCol->type};
+    SLastCol         p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
 
     if (IS_VAR_DATA_TYPE(pCol->type)) {
       p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char));

source/libs/parser/src/parTranslater.c

@@ -2797,6 +2797,10 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
   bool    other;
   int32_t code = translateOrderByPosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other);
   if (TSDB_CODE_SUCCESS == code) {
+    if (0 == LIST_LENGTH(pSelect->pOrderByList)) {
+      NODES_DESTORY_LIST(pSelect->pOrderByList);
+      return TSDB_CODE_SUCCESS;
+    }
     if (!other) {
       return TSDB_CODE_SUCCESS;
     }

source/libs/wal/src/walWrite.c

@@ -387,20 +387,33 @@ END:
 
 int32_t walRollImpl(SWal *pWal) {
   int32_t code = 0;
+
   if (pWal->pIdxFile != NULL) {
+    code = taosFsyncFile(pWal->pIdxFile);
+    if (code != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      goto END;
+    }
     code = taosCloseFile(&pWal->pIdxFile);
     if (code != 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       goto END;
     }
   }
+
   if (pWal->pLogFile != NULL) {
+    code = taosFsyncFile(pWal->pLogFile);
+    if (code != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      goto END;
+    }
     code = taosCloseFile(&pWal->pLogFile);
     if (code != 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       goto END;
     }
   }
+
   TdFilePtr pIdxFile, pLogFile;
   // create new file
   int64_t newFileFirstVer = pWal->vers.lastVer + 1;

tests/parallel_test/container_build.sh

@@ -51,10 +51,24 @@ else
     REP_DIR=/home/TDinternal
     REP_REAL_PATH=$WORKDIR/TDinternal
     REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal
+    
 fi
 date 
 docker run \
     -v $REP_MOUNT_PARAM \
+    -v /root/.cargo/registry:/root/.cargo/registry \
+    -v /root/.cargo/git:/root/.cargo/git \
+    -v /root/go/pkg/mod:/root/go/pkg/mod \
+    -v /root/.cache/go-build:/root/.cache/go-build \
+    -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
+    -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
+    -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
+    -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
+    -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
+    -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
+    -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
+    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
+    -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
     --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true;make -j || exit 1"
 
 if [[ -d ${WORKDIR}/debugNoSan  ]] ;then
@@ -70,6 +84,19 @@ mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugNoSan
 date
 docker run \
     -v $REP_MOUNT_PARAM \
+    -v /root/.cargo/registry:/root/.cargo/registry \
+    -v /root/.cargo/git:/root/.cargo/git \
+    -v /root/go/pkg/mod:/root/go/pkg/mod \
+    -v /root/.cache/go-build:/root/.cache/go-build \
+    -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
+    -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
+    -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
+    -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
+    -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
+    -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
+    -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
+    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
+    -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
     --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true   -DBUILD_SANITIZER=1  -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true;make -j || exit 1 "
 
 mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugSan

tests/system-test/0-others/show.py

@@ -11,10 +11,6 @@
 
 # -*- coding: utf-8 -*-
 
-
-
-
-
 import re
 from util.log import *
 from util.cases import *
@@ -31,10 +27,35 @@ class TDTestCase:
         self.ins_param_list = ['dnodes','mnodes','qnodes','cluster','functions','users','grants','topics','subscriptions','streams']
         self.perf_param = ['apps','connections','consumers','queries','transactions']
         self.perf_param_list = ['apps','connections','consumers','queries','trans']
-        
+        self.dbname = "db"
+        self.vgroups = 10
+        self.stbname = f'`{tdCom.getLongName(5)}`'
+        self.tbname = f'`{tdCom.getLongName(3)}`'
+        self.db_param = {
+            "database":f"{self.dbname}",
+            "buffer":100,
+            "cachemodel":"'none'",
+            "cachesize":1,
+            "comp":2,
+            "maxrows":1000,
+            "minrows":200,
+            "pages":512,
+            "pagesize":16,
+            "precision":"'ms'",
+            "replica":1,
+            "wal_level":1,
+            "wal_fsync_period":6000,
+            "wal_roll_period":0,
+            "wal_segment_size":1024,
+            "vgroups":self.vgroups,
+            "stt_trigger":1,
+            "tsdb_pagesize":16
+        }
     def ins_check(self):
         tdSql.prepare()
         for param in self.ins_param_list:
+            if param.lower() == 'qnodes':
+                tdSql.execute('create qnode on dnode 1')
             tdSql.query(f'show {param}')
             show_result = tdSql.queryResult
             tdSql.query(f'select * from information_schema.ins_{param}')
@@ -62,11 +83,32 @@ class TDTestCase:
             tag_sql += f"{k} {v}, "
         create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})'
         return create_stb_sql
-    def show_sql(self):
+    
-        tdSql.prepare()
+    def set_create_database_sql(self,sql_dict):
-        tdSql.execute('use db')
+        create_sql = 'create'
-        stbname = f'`{tdCom.getLongName(5)}`'
+        for key,value in sql_dict.items():
-        tbname = f'`{tdCom.getLongName(3)}`'
+            create_sql += f' {key} {value}'
+        return create_sql
+    def show_create_sql(self):
+        create_db_sql = self.set_create_database_sql(self.db_param)
+        print(create_db_sql)
+        tdSql.execute(create_db_sql)
+        tdSql.query(f'show create database {self.dbname}')
+        tdSql.checkEqual(self.dbname,tdSql.queryResult[0][0])
+        for key,value in self.db_param.items():
+            if key == 'database':
+                continue
+            else:
+                param = f'{key} {value}'
+                if param in tdSql.queryResult[0][1].lower():
+                    tdLog.info(f'show create database check success with {key} {value}')
+                    continue
+                else:
+                    tdLog.exit(f"show create database check failed with {key} {value}")
+        tdSql.query('show vnodes 1')
+        tdSql.checkRows(self.vgroups)
+        tdSql.execute(f'use {self.dbname}')
+        
         column_dict = {
             '`ts`': 'timestamp',
             '`col1`': 'tinyint',
@@ -101,21 +143,21 @@ class TDTestCase:
             '`t14`': 'timestamp'
             
         }
-        create_table_sql = self.set_stb_sql(stbname,column_dict,tag_dict)
+        create_table_sql = self.set_stb_sql(self.stbname,column_dict,tag_dict)
         tdSql.execute(create_table_sql)
-        tdSql.query(f'show create table {stbname}')
+        tdSql.query(f'show create stable {self.stbname}')
         query_result = tdSql.queryResult
         tdSql.checkEqual(query_result[0][1].lower(),create_table_sql)
-        tdSql.execute(f'create table {tbname} using {stbname} tags(1,1,1,1,1,1,1,1,1.000000e+00,1.000000e+00,true,"abc","abc123",0)')
+        tdSql.execute(f'create table {self.tbname} using {self.stbname} tags(1,1,1,1,1,1,1,1,1.000000e+00,1.000000e+00,true,"abc","abc123",0)')
         tag_sql = '('
         for tag_keys in tag_dict.keys():
             tag_sql += f'{tag_keys}, '
         tags = f'{tag_sql[:-2]})' 
-        sql = f'create table {tbname} using {stbname} {tags} tags (1, 1, 1, 1, 1, 1, 1, 1, 1.000000e+00, 1.000000e+00, true, "abc", "abc123", 0)'
+        sql = f'create table {self.tbname} using {self.stbname} {tags} tags (1, 1, 1, 1, 1, 1, 1, 1, 1.000000e+00, 1.000000e+00, true, "abc", "abc123", 0)'
-        tdSql.query(f'show create table {tbname}')
+        tdSql.query(f'show create table {self.tbname}')
         query_result = tdSql.queryResult
         tdSql.checkEqual(query_result[0][1].lower(),sql)
-        tdSql.execute('drop database db')
+        tdSql.execute(f'drop database {self.dbname}')
     def check_gitinfo(self):
         taosd_gitinfo_sql = ''
         tdSql.query('show dnode 1 variables')
@@ -133,11 +175,24 @@ class TDTestCase:
         taosd_info = os.popen('taosd -V').read()
         taosd_gitinfo = re.findall("^gitinfo.*",taosd_info,re.M)
         tdSql.checkEqual(taosd_gitinfo_sql,taosd_gitinfo[0])
+    
+    def show_base(self):
+        for sql in ['dnodes','mnodes','cluster']:
+            tdSql.query(f'show {sql}')
+            print(tdSql.queryResult)
+            tdSql.checkRows(1)
+        tdSql.query('show grants')
+        grants_info = tdSql.queryResult
+        tdSql.query('show licences')
+        licences_info = tdSql.queryResult
+        tdSql.checkEqual(grants_info,licences_info)
+
     def run(self):
         self.check_gitinfo()
+        self.show_base()
         self.ins_check()
         self.perf_check()
-        self.show_sql()
+        self.show_create_sql()
 
     def stop(self):
         tdSql.close()
@@ -145,4 +200,3 @@ class TDTestCase:
 
 tdCases.addWindows(__file__, TDTestCase())
 tdCases.addLinux(__file__, TDTestCase())
-

tests/system-test/2-query/projectionDesc.py

@@ -38,6 +38,7 @@ class TDTestCase:
         #tdSql.checkData(0,0,1537146000000)
         tdSql.checkData(0,1,10)
 
+        tdSql.query(f"select * from {dbname}.stb_1 order by 'aaa' desc")
 
     def stop(self):
         tdSql.close()