:wqMerge branch 'develop' into hotfix/py-testcase-refine

README.md

@@ -121,7 +121,7 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo
 
 1.  sudo apt install
     build-essential cmake net-tools python-pip python-setuptools python3-pip
-    python3-setuptools valgrind
+    python3-setuptools valgrind psmisc curl
 
 2.  git clone <https://github.com/taosdata/TDengine>; cd TDengine
 

src/client/src/tscFunctionImpl.c

@@ -1691,10 +1691,7 @@ static void last_function(SQLFunctionCtx *pCtx) {
 }
 
 static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  assert(pCtx->order != TSDB_ORDER_ASC);
-    return;
-  }
-  
   void *pData = GET_INPUT_CHAR_INDEX(pCtx, index);
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
@@ -2912,7 +2909,7 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
 }
 
 static void date_col_output_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->scanFlag == SUPPLEMENTARY_SCAN) {
+  if (pCtx->scanFlag == REVERSE_SCAN) {
     return;
   }
   
@@ -2969,11 +2966,12 @@ static void tag_project_function(SQLFunctionCtx *pCtx) {
     char* output = pCtx->aOutputBuf;
   
     if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) {
-      *(int16_t*) output = pCtx->tag.nLen;
+      varDataSetLen(output, pCtx->tag.nLen);
-      output += VARSTR_HEADER_SIZE;
+      tVariantDump(&pCtx->tag, varDataVal(output), pCtx->outputType);
+    } else {
+      tVariantDump(&pCtx->tag, output, pCtx->outputType);
     }
     
-    tVariantDump(&pCtx->tag, output, pCtx->outputType);
     pCtx->aOutputBuf += pCtx->outputBytes;
   }
 }

src/client/src/tscSQLParser.c

@@ -5911,7 +5911,7 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
   }
   
   if (pSqlExpr->pLeft == NULL) {
-    if (pSqlExpr->nSQLOptr >= TK_TINYINT && pSqlExpr->nSQLOptr <= TK_DOUBLE) {
+    if (pSqlExpr->nSQLOptr >= TK_BOOL && pSqlExpr->nSQLOptr <= TK_STRING) {
       *pExpr = calloc(1, sizeof(tExprNode));
       (*pExpr)->nodeType = TSQL_NODE_VALUE;
       (*pExpr)->pVal = calloc(1, sizeof(tVariant));

src/client/src/tscServer.c

@@ -492,13 +492,15 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pMsg += sizeof(pQueryInfo->type);
 
   // todo valid the vgroupId at the client side
-  if (UTIL_TABLE_IS_SUPERTABLE(pQueryInfo->pTableMetaInfo[0])) {
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-    SVgroupsInfo* pVgroupInfo = pQueryInfo->pTableMetaInfo[0]->vgroupList;
+  
-    assert(pVgroupInfo->numOfVgroups == 1); // todo fix me
+  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    int32_t vgIndex = pTableMetaInfo->vgroupIndex;
     
-    pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[0].vgId);
+    SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList;
+    pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId);
   } else {
-    STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[0]->pTableMeta;
+    STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
     pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId);
   }
   

src/client/src/tscSubquery.c

@@ -1436,8 +1436,8 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
       return;
     } else {  // reach the maximum retry count, abort
       atomic_val_compare_exchange_32(&pState->code, TSDB_CODE_SUCCESS, numOfRows);
-      tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql,
+      tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%s", pPObj, pSql,
-               numOfRows, subqueryIndex, tstrerror(pState->code));
+               tstrerror(numOfRows), subqueryIndex, tstrerror(pState->code));
     }
   }
   
@@ -1450,7 +1450,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
   }
   
   // all subqueries are failed
-  tscError("%p retrieve from %d vnode(s) completed,code:%d.FAILED.", pPObj, pState->numOfTotal, pState->code);
+  tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pPObj, pState->numOfTotal, tstrerror(pState->code));
   pPObj->res.code = pState->code;
   
   // release allocated resource

src/client/src/tscUtil.c

@@ -1675,10 +1675,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
   pTableMetaInfo->pTableMeta = pTableMeta;
   
   if (vgroupList != NULL) {
-    assert(vgroupList->numOfVgroups == 1);  // todo fix me
-    
     size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups;
-    
     pTableMetaInfo->vgroupList = malloc(size);
     memcpy(pTableMetaInfo->vgroupList, vgroupList, size);
   }

src/common/inc/tdataformat.h

@@ -115,10 +115,8 @@ static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t o
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
       return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
-      break;
     default:
       return POINTER_SHIFT(row, offset);
-      break;
   }
 }
 

src/inc/taosdef.h

@@ -36,6 +36,11 @@ extern "C" {
 typedef int32_t VarDataOffsetT;
 typedef int16_t VarDataLenT;
 
+typedef struct tstr {
+  VarDataLenT len;
+  char        data[];
+} tstr;
+
 #define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
 
 #define varDataLen(v)       ((VarDataLenT *)(v))[0]
@@ -74,10 +79,10 @@ extern const int32_t TYPE_BYTES[11];
 #define TSDB_DATA_NULL_STR              "NULL"
 #define TSDB_DATA_NULL_STR_L            "null"
 
-#define TSDB_TRUE 1
+#define TSDB_TRUE   1
-#define TSDB_FALSE 0
+#define TSDB_FALSE  0
-#define TSDB_OK 0
+#define TSDB_OK     0
-#define TSDB_ERR -1
+#define TSDB_ERR   -1
 
 #define TS_PATH_DELIMITER "."
 

src/inc/tsdb.h

@@ -102,7 +102,7 @@ int  tsdbTableSetSName(STableCfg *config, char *sname, bool dup);
 void tsdbClearTableCfg(STableCfg *config);
 
 int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId* id, int32_t colId, int16_t *type, int16_t *bytes, char **val);
-int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name);
+char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes);
 
 int   tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
 int   tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);

src/query/inc/queryExecutor.h

@@ -110,6 +110,14 @@ typedef struct STableQueryInfo {  // todo merge with the STableQueryInfo struct
   SWindowResInfo windowResInfo;
 } STableQueryInfo;
 
+typedef struct SQueryCostSummary {
+} SQueryCostSummary;
+
+typedef struct SGroupItem {
+  STableId id;
+  STableQueryInfo* info;
+} SGroupItem;
+
 typedef struct SQuery {
   int16_t           numOfCols;
   int16_t           numOfTags;
@@ -131,17 +139,15 @@ typedef struct SQuery {
   SColumnInfo*      tagColList;
   int32_t           numOfFilterCols;
   int64_t*          defaultVal;
-  TSKEY             lastKey;
+//  TSKEY             lastKey;
   uint32_t          status;  // query status
   SResultRec        rec;
   int32_t           pos;
   SData**           sdata;
+  STableQueryInfo*  current;
   SSingleColumnFilterInfo* pFilterInfo;
 } SQuery;
 
-typedef struct SQueryCostSummary {
-} SQueryCostSummary;
-
 typedef struct SQueryRuntimeEnv {
   SResultInfo*       resultInfo;  // todo refactor to merge with SWindowResInfo
   SQuery*            pQuery;

src/query/inc/tsqlfunction.h

@@ -104,7 +104,7 @@ extern "C" {
 
 enum {
   MASTER_SCAN           = 0x0u,
-  SUPPLEMENTARY_SCAN    = 0x1u,
+  REVERSE_SCAN          = 0x1u,
   REPEAT_SCAN           = 0x2u,  //repeat scan belongs to the master scan
   FIRST_STAGE_MERGE     = 0x10u,
   SECONDARY_STAGE_MERGE = 0x20u,

src/query/src/qast.c

@@ -552,18 +552,18 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
   if (cond.start != NULL) {
     int32_t optr = cond.start->optr;
     
-    if (optr == TSDB_RELATION_EQUAL) {
+    if (optr == TSDB_RELATION_EQUAL) {   // equals
       while(tSkipListIterNext(iter)) {
         SSkipListNode* pNode = tSkipListIterGet(iter);
 
         int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v);
-        if (ret == 0) {
+        if (ret != 0) {
-          taosArrayPush(result, SL_GET_NODE_DATA(pNode));
-        } else {
           break;
         }
+        
+        taosArrayPush(result, SL_GET_NODE_DATA(pNode));
       }
-    } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) {
+    } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal
       bool comp = true;
       int32_t ret = 0;
       
@@ -582,8 +582,33 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
           comp = false;
         }
       }
-    } else if (optr == TSDB_RELATION_NOT_EQUAL) {
+    } else if (optr == TSDB_RELATION_NOT_EQUAL) {   // not equal
-      assert(0);
+      bool comp = true;
+      
+      while(tSkipListIterNext(iter)) {
+        SSkipListNode* pNode = tSkipListIterGet(iter);
+        comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
+        if (comp) {
+          continue;
+        }
+        
+        taosArrayPush(result, SL_GET_NODE_DATA(pNode));
+      }
+      
+      tSkipListDestroyIter(iter);
+  
+      comp = true;
+      iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+      while(tSkipListIterNext(iter)) {
+        SSkipListNode* pNode = tSkipListIterGet(iter);
+        comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
+        if (comp) {
+          continue;
+        }
+  
+        taosArrayPush(result, SL_GET_NODE_DATA(pNode));
+      }
+  
     } else {
       assert(0);
     }
@@ -773,8 +798,6 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki
   tSkipListDestroyIter(iter);
 }
 
-
-
 static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) {
   SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
 
@@ -785,7 +808,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
     char* pTable = SL_GET_NODE_DATA(pNode);
     
     //todo refactor:
-    char* name = (*(STable**) pTable)->name;
+    tstr* name = ((STableIndexElem*) pTable)->pTable->name;
 //    char* name = NULL;
 //    tsdbGetTableName(tsdb, pTable, &name);
 

src/tsdb/inc/tsdbMain.h

@@ -81,7 +81,6 @@ typedef struct {
 // ---------- TSDB TABLE DEFINITION
 typedef struct STable {
   int8_t         type;
-  char *         name;
   STableId       tableId;
   int64_t        superUid;  // Super table UID
   int32_t        sversion;
@@ -96,9 +95,10 @@ typedef struct STable {
   TSKEY          lastKey;        // lastkey inserted in this table, initialized as 0, TODO: make a structure
   struct STable *next;           // TODO: remove the next
   struct STable *prev;
+  tstr *         name;           // NOTE: there a flexible string here
 } STable;
 
-#define TSDB_GET_TABLE_LAST_KEY(pTable) ((pTable)->lastKey)
+#define TSDB_GET_TABLE_LAST_KEY(tb) ((tb)->lastKey)
 
 void *  tsdbEncodeTable(STable *pTable, int *contLen);
 STable *tsdbDecodeTable(void *cont, int contLen);
@@ -121,6 +121,12 @@ typedef struct {
   int        maxCols;
 } STsdbMeta;
 
+// element put in skiplist for each table
+typedef struct STableIndexElem {
+  STsdbMeta* pMeta;
+  STable*    pTable;
+} STableIndexElem;
+
 STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables);
 int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
 STSchema * tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable);
@@ -150,7 +156,7 @@ int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId);
 STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId);
 // int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable);
 STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid);
-char *  getTupleKey(const void *data);
+char *getTSTupleKey(const void * data);
 
 typedef struct {
   int  blockId;

src/tsdb/src/tsdbMain.c

@@ -808,7 +808,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   if (pTable->mem == NULL) {
     pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable));
     if (pTable->mem == NULL) return -1;
-    pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey);
+    pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey);
     pTable->mem->keyFirst = INT64_MAX;
     pTable->mem->keyLast = 0;
   }
@@ -831,7 +831,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   if (pTable->mem == NULL) {
     pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable));
     if (pTable->mem == NULL) return -1;
-    pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey);
+    pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey);
     pTable->mem->keyFirst = INT64_MAX;
     pTable->mem->keyLast = 0;
   }

src/tsdb/src/tsdbMeta.c

@@ -1,6 +1,4 @@
 #include <stdlib.h>
-
-// #include "taosdef.h"
 #include "tskiplist.h"
 #include "tsdb.h"
 #include "taosdef.h"
@@ -10,6 +8,8 @@
 #define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here
 #define TSDB_META_FILE_NAME "META"
 
+const int32_t DEFAULT_TAG_INDEX_COLUMN = 0;
+
 static int     tsdbFreeTable(STable *pTable);
 static int32_t tsdbCheckTableCfg(STableCfg *pCfg);
 static int     tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx);
@@ -39,11 +39,12 @@ void *tsdbEncodeTable(STable *pTable, int *contLen) {
 
   void *ptr = ret;
   T_APPEND_MEMBER(ptr, pTable, STable, type);
-  // Encode name
+  // Encode name, todo refactor
-  *(int *)ptr = strlen(pTable->name);
+  *(int *)ptr = varDataLen(pTable->name);
   ptr = (char *)ptr + sizeof(int);
-  memcpy(ptr, pTable->name, strlen(pTable->name));
+  memcpy(ptr, varDataVal(pTable->name), varDataLen(pTable->name));
-  ptr = (char *)ptr + strlen(pTable->name);
+  ptr = (char *)ptr + varDataLen(pTable->name);
+  
   T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, uid);
   T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, tid);
   T_APPEND_MEMBER(ptr, pTable, STable, superUid);
@@ -79,9 +80,12 @@ STable *tsdbDecodeTable(void *cont, int contLen) {
   T_READ_MEMBER(ptr, int8_t, pTable->type);
   int len = *(int *)ptr;
   ptr = (char *)ptr + sizeof(int);
-  pTable->name = calloc(1, len + 1);
+  pTable->name = calloc(1, len + VARSTR_HEADER_SIZE);
   if (pTable->name == NULL) return NULL;
-  memcpy(pTable->name, ptr, len);
+  
+  varDataSetLen(pTable->name, len);
+  memcpy(pTable->name->data, ptr, len);
+  
   ptr = (char *)ptr + len;
   T_READ_MEMBER(ptr, int64_t, pTable->tableId.uid);
   T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid);
@@ -105,8 +109,13 @@ void tsdbFreeEncode(void *cont) {
 }
 
 static char* getTagIndexKey(const void* pData) {
-  STable* table = *(STable**) pData;
+  STableIndexElem* elem = (STableIndexElem*) pData;
-  return getTupleKey(table->tagVal);
+  
+  SDataRow row = elem->pTable->tagVal;
+  STSchema* pSchema = tsdbGetTableTagSchema(elem->pMeta, elem->pTable);
+  STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN];
+  
+  return tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
 }
 
 int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
@@ -225,36 +234,28 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) {
   }
 }
 
+// todo refactor table name definition
 int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t* type, int16_t* bytes, char** val) {
   STsdbMeta* pMeta = tsdbGetMeta(repo);
   STable* pTable = tsdbGetTableByUid(pMeta, id->uid);
   
   STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable);
-  
   STColumn* pCol = NULL;
-  int32_t offset = 0;
+  
   for(int32_t col = 0; col < schemaNCols(pSchema); ++col) {
     STColumn* p = schemaColAt(pSchema, col);
     if (p->colId == colId) {
       pCol = p;
       break;
     }
-  
-    if (p->type == TSDB_DATA_TYPE_BINARY || p->type == TSDB_DATA_TYPE_NCHAR) {
-      offset += sizeof(int32_t);
-    } else {
-      offset += p->bytes;
-    }
   }
   
   if (pCol == NULL) {
     return -1;  // No matched tags. Maybe the modification of tags has not been done yet.
   }
   
-  assert(pCol != NULL);
-  
   SDataRow row = (SDataRow)pTable->tagVal;
-  char* d = tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + offset);
+  char* d = tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
   
   *val = d;
   *type  = pCol->type;
@@ -263,15 +264,22 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name) {
+char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes) {
   STsdbMeta* pMeta = tsdbGetMeta(repo);
   STable* pTable = tsdbGetTableByUid(pMeta, id->uid);
   
-  *name = strndup(pTable->name, TSDB_TABLE_NAME_LEN);
+  if (pTable == NULL) {
-  if (*name == NULL) {
+    if (bytes != NULL) {
-    return -1;
+      *bytes = 0;
+    }
+    
+    return NULL;
   } else {
-    return 0;
+    if (bytes != NULL) {
+      *bytes = varDataLen(pTable->name);
+    }
+    
+    return (char*) pTable->name;
   }
 }
 
@@ -296,7 +304,11 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
       super->schema = tdDupSchema(pCfg->schema);
       super->tagSchema = tdDupSchema(pCfg->tagSchema);
       super->tagVal = NULL;
-      super->name = strdup(pCfg->sname);
+      
+      // todo refactor extract method
+      size_t size = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN);
+      super->name = malloc(size + VARSTR_HEADER_SIZE);
+      STR_WITH_SIZE_TO_VARSTR(super->name, pCfg->sname, size);
 
       // index the first tag column
       STColumn* pColSchema = schemaColAt(super->tagSchema, 0);
@@ -322,7 +334,11 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
   }
 
   table->tableId = pCfg->tableId;
-  table->name = strdup(pCfg->name);
+  
+  size_t size = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN);
+  table->name = malloc(size + VARSTR_HEADER_SIZE);
+  STR_WITH_SIZE_TO_VARSTR(table->name, pCfg->name, size);
+  
   table->lastKey = 0;
   if (IS_CREATE_STABLE(pCfg)) { // TSDB_CHILD_TABLE
     table->type = TSDB_CHILD_TABLE;
@@ -513,11 +529,14 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) {
   
   // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
   // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
-  SSkipListNode* pNode = calloc(1, headSize + POINTER_BYTES);
+  SSkipListNode* pNode = calloc(1, headSize + sizeof(STableIndexElem));
   pNode->level = level;
   
   SSkipList* list = pSTable->pIndex;
-  memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES);
+  STableIndexElem* elem = (STableIndexElem*) (SL_GET_NODE_DATA(pNode));
+  
+  elem->pTable = pTable;
+  elem->pMeta = pMeta;
   
   tSkipListPut(list, pNode);
   return 0;
@@ -539,7 +558,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
 static int tsdbEstimateTableEncodeSize(STable *pTable) {
   int size = 0;
   size += T_MEMBER_SIZE(STable, type);
-  size += sizeof(int) + strlen(pTable->name);
+  size += sizeof(int) + varDataLen(pTable->name);
   size += T_MEMBER_SIZE(STable, tableId);
   size += T_MEMBER_SIZE(STable, superUid);
   size += T_MEMBER_SIZE(STable, sversion);
@@ -556,8 +575,7 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) {
   return size;
 }
 
-char *getTupleKey(const void * data) {
+char *getTSTupleKey(const void * data) {
   SDataRow row = (SDataRow)data;
-
   return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE);
 }

src/tsdb/src/tsdbRead.c

@@ -620,9 +620,13 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf
         if (pCol->info.type != TSDB_DATA_TYPE_BINARY && pCol->info.type != TSDB_DATA_TYPE_NCHAR) {
           memmove(pCol->pData, src->pData + bytes * start, bytes * pQueryHandle->realNumOfRows);
         } else { // handle the var-string
+          char* dst = pCol->pData;
+  
+          // todo refactor, only copy one-by-one
           for(int32_t k = start; k < pQueryHandle->realNumOfRows + start; ++k) {
             char* p = tdGetColDataOfRow(src, k);
-            memcpy(pCol->pData + k * bytes, p, varDataTLen(p));  // todo refactor
+            memcpy(dst, p, varDataTLen(p));
+            dst += varDataTLen(p);
           }
         }
         
@@ -1076,17 +1080,13 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY
       }
       
       assert(offset != -1); // todo handle error
+      void *value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset);
       
       if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
-        void *value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset);
         memcpy(pData, value, varDataTLen(value));
-  
-        offset += sizeof(int32_t);
       } else {
-        memcpy(pData, dataRowTuple(row) + offset, pColInfo->info.bytes);
+        memcpy(pData, value, pColInfo->info.bytes);
-        offset += pColInfo->info.bytes;
       }
-      
     }
 
     numOfRows++;
@@ -1225,8 +1225,8 @@ static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) {
   while (tSkipListIterNext(iter)) {
     SSkipListNode* pNode = tSkipListIterGet(iter);
     
-    STable* t = *(STable**)SL_GET_NODE_DATA(pNode);
+    STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*) pNode));
-    taosArrayPush(list, &t->tableId);
+    taosArrayPush(list, &elem->pTable->tableId);
   }
   
   tSkipListDestroyIter(iter);
@@ -1235,6 +1235,7 @@ static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) {
 
 /**
  * convert the result pointer to table id instead of table object pointer
+ * todo remove it by using callback function to change the final result in-time.
  * @param pRes
  */
 static void convertQueryResult(SArray* pRes, SArray* pTableList) {
@@ -1244,8 +1245,8 @@ static void convertQueryResult(SArray* pRes, SArray* pTableList) {
 
   size_t size = taosArrayGetSize(pTableList);
   for (int32_t i = 0; i < size; ++i) {  // todo speedup  by using reserve space.
-    STable* pTable = taosArrayGetP(pTableList, i);
+    STableIndexElem* elem = taosArrayGet(pTableList, i);
-    taosArrayPush(pRes, &pTable->tableId);
+    taosArrayPush(pRes, &elem->pTable->tableId);
   }
 }
 
@@ -1309,7 +1310,12 @@ void filterPrepare(void* expr, void* param) {
     pInfo->q = (char*) pCond->arr;
   } else {
     pInfo->q = calloc(1, pSchema->bytes);
-    tVariantDump(pCond, pInfo->q, pSchema->type);
+    if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
+      varDataSetLen(pInfo->q, pCond->nLen);
+      tVariantDump(pCond, varDataVal(pInfo->q), pSchema->type);
+    } else {
+      tVariantDump(pCond, pInfo->q, pSchema->type);
+    }
   }
 }
 
@@ -1341,16 +1347,16 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
     int32_t bytes = 0;
     
     if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
-      f1 = pTable1->name;
+      f1 = (char*) pTable1->name;
-      f2 = pTable2->name;
+      f2 = (char*) pTable2->name;
       type = TSDB_DATA_TYPE_BINARY;
-      bytes = TSDB_TABLE_NAME_LEN;
+      bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
     } else {
-      f1 = dataRowTuple(pTable1->tagVal);
+      STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex);
-      f2 = dataRowTuple(pTable2->tagVal);
+      bytes = pCol->bytes;
-
+  
-      type = schemaColAt(pTableGroupSupp->pTagSchema, colIndex)->type;
+      f1 = tdGetRowDataOfCol(pTable1->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
-      bytes = schemaColAt(pTableGroupSupp->pTagSchema, colIndex)->bytes;
+      f2 = tdGetRowDataOfCol(pTable2->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
     }
     
     int32_t ret = doCompare(f1, f2, type, bytes);
@@ -1428,24 +1434,20 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
 
 bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
   tQueryInfo* pInfo = (tQueryInfo*) param;
-
+  
-  STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
+  STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
 
   char*  val = NULL;
   int8_t type = pInfo->sch.type;
 
   if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
-    val = pTable->name;
+    val = (char*) elem->pTable->name;
     type = TSDB_DATA_TYPE_BINARY;
   } else {
     STSchema* pTSchema = (STSchema*) pInfo->param; // todo table schema is identical to stable schema??
     
     int32_t offset = pTSchema->columns[pInfo->colIndex].offset;
-    if (pInfo->sch.type == TSDB_DATA_TYPE_BINARY || pInfo->sch.type == TSDB_DATA_TYPE_NCHAR) {
+    val = tdGetRowDataOfCol(elem->pTable->tagVal, pInfo->sch.type, TD_DATA_ROW_HEAD_SIZE + offset);
-      val = tdGetRowDataOfCol(pTable->tagVal, pInfo->sch.type, TD_DATA_ROW_HEAD_SIZE + offset);
-    } else {
-      val = dataRowTuple(pTable->tagVal) + offset;
-    }
   }
 
   int32_t ret = 0;
@@ -1456,8 +1458,6 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
       ret = pInfo->compare(val, pInfo->q);
     }
   } else {
-//    tVariant t = {0};
-//    tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type);
     ret = pInfo->compare(val, pInfo->q);
   }
 
@@ -1502,7 +1502,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
       .pExtInfo = pSTable->tagSchema,
       };
 
-  SArray* pTableList = taosArrayInit(8, POINTER_BYTES);
+  SArray* pTableList = taosArrayInit(8, sizeof(STableIndexElem));
 
   tExprTreeTraverse(pExpr, pSTable->pIndex, pTableList, &supp);
   tExprTreeDestroy(&pExpr, destroyHelper);

src/util/src/talgo.c

@@ -144,11 +144,11 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const
   }
   
   if (leftPartEnd > start) {
-    tqsortImpl(src, size, start, leftPartEnd, param, comparFn, buf);
+    tqsortImpl(src, start, leftPartEnd, size, param, comparFn, buf);
   }
   
   if (rightPartStart < end) {
-    tqsortImpl(src, size, rightPartStart, end, param, comparFn, buf);
+    tqsortImpl(src, rightPartStart, end, size, param, comparFn, buf);
   }
 }
 

src/util/src/tcompare.c

@@ -60,8 +60,15 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
   }
 }
 
-int32_t compareStrVal(const void *pLeft, const void *pRight) {
+int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
-  return (int32_t)strcmp(pLeft, pRight);
+  int32_t len1 = varDataLen(pLeft);
+  int32_t len2 = varDataLen(pRight);
+  
+  if (len1 != len2) {
+    return len1 > len2? 1:-1;
+  } else {
+    return (int32_t) strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
+  }
 }
 
 int32_t compareWStrVal(const void *pLeft, const void *pRight) {
@@ -267,7 +274,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
       } else if (optr == TSDB_RELATION_IN) {
         comparFn = compareFindStrInArray;
       } else { /* normal relational comparFn */
-        comparFn = compareStrVal;
+        comparFn = compareLenPrefixedStr;
       }
     
       break;
@@ -296,6 +303,7 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) {
   
   switch (keyType) {
     case TSDB_DATA_TYPE_TINYINT:
+    case TSDB_DATA_TYPE_BOOL:
       comparFn = compareInt8Val;
       break;
     case TSDB_DATA_TYPE_SMALLINT:
@@ -308,17 +316,13 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) {
     case TSDB_DATA_TYPE_TIMESTAMP:
       comparFn = compareInt64Val;
       break;
-    case TSDB_DATA_TYPE_BOOL:
-      comparFn = compareInt32Val;
-      break;
-    
     case TSDB_DATA_TYPE_FLOAT:
     case TSDB_DATA_TYPE_DOUBLE:
       comparFn = compareDoubleVal;
       break;
     
     case TSDB_DATA_TYPE_BINARY:
-      comparFn = compareStrVal;
+      comparFn = compareLenPrefixedStr;
       break;
     
     case TSDB_DATA_TYPE_NCHAR:
@@ -349,13 +353,20 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
       }
       return (ret < 0) ? -1 : 1;
     }
-    default: {
+    default: {  // todo refactor
-      int32_t ret = strncmp(f1, f2, (size_t)size);
+      tstr* t1 = (tstr*) f1;
-      if (ret == 0) {
+      tstr* t2 = (tstr*) f2;
-        return ret;
-      }
       
-      return (ret < 0) ? -1 : 1;
+      if (t1->len != t2->len) {
+        return t1->len > t2->len? 1:-1;
+      } else {
+        int32_t ret = strncmp(t1->data, t2->data, t1->len);
+        if (ret == 0) {
+          return 0;
+        } else {
+          return ret < 0? -1:1;
+        }
+      }
     }
   }
 }

tests/script/general/cache/restart_metrics.sim

@@ -27,11 +27,15 @@ sql create table $tb using $mt tags( "1" )
 sql insert into $tb values (now, 1)
 
 sql select * from $tb
-print ===>rows $rows, data $data01
+#print ===>rows $rows, data $data01
+
 if $rows != 1 then
+  print expect 1, actual: $rows
   return -1
-endi  
+endi
+
 if $data01 != 1 then
+  print expect 1 actual: $data01
   return -1
 endi  
 
@@ -66,8 +70,10 @@ sql select * from $tb
 print ===>rows $rows, data $data01
 if $rows != 1 then
   return -1
-endi  
+endi
+
 if $data01 != 1 then
+  print expect 1, actual $data01
   return -1
 endi  
 
@@ -80,6 +86,7 @@ if $data01 != 1 then
   return -1
 endi  
 if $data02 != 3 then
+  print expect 3 actual: $data02
   return -1
 endi 
 

tests/script/general/field/bigint.sim

@@ -132,7 +132,9 @@ endi
 print =============== step6
 sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 
-if $data00 != 100 then 
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16
+if $data00 != 100 then
+  print expect 100, actual $data00
   return -1
 endi
 

tests/script/general/parser/lastrow_query.sim

@@ -43,9 +43,11 @@ if $data07 != 1 then
   return -1
 endi
 if $data08 != BINARY then
+  print expect BINARY actual: $data08
   return -1
 endi
 if $data09 != NCHAR then
+  print expect NCHAR actual: $data09
   return -1
 endi
 

tests/script/general/parser/testSuite.sim

@@ -1,113 +1,97 @@
-# run general/parser/alter.sim
+#run general/parser/alter.sim
-# sleep 2000
+#sleep 2000
-# run general/parser/alter1.sim
+#run general/parser/alter1.sim
-# sleep 2000
+#sleep 2000
-# run general/parser/alter_stable.sim
+#run general/parser/alter_stable.sim
-# sleep 2000
+#sleep 2000
-# run general/parser/auto_create_tb.sim
+#run general/parser/auto_create_tb.sim
-# sleep 2000
+#sleep 2000
-# run general/parser/auto_create_tb_drop_tb.sim
+#run general/parser/auto_create_tb_drop_tb.sim
-# sleep 2000
 
-run general/parser/binary_escapeCharacter.sim
-sleep 2000
-run general/parser/bug.sim
 sleep 2000
 run general/parser/col_arithmetic_operation.sim
 sleep 2000
-run general/parser/columnValue_bigint.sim
+run general/parser/columnValue.sim
 sleep 2000
-run general/parser/columnValue_bool.sim
+run general/parser/commit.sim
 sleep 2000
-run general/parser/columnValue_double.sim
+run general/parser/create_db.sim
 sleep 2000
-run general/parser/columnValue_float.sim
+run general/parser/create_mt.sim
 sleep 2000
-run general/parser/columnValue_int.sim
+run general/parser/create_tb.sim
+sleep 2000
+run general/parser/dbtbnameValidate.sim
+sleep 2000
+run general/parser/import_commit1.sim
+sleep 2000
+run general/parser/import_commit2.sim
+sleep 2000
+run general/parser/import_commit3.sim
+sleep 2000
+run general/parser/insert_tb.sim
+sleep 2000
+run general/parser/first_last.sim
+sleep 2000
+run general/parser/import_file.sim
+sleep 2000
+run general/parser/fill.sim
+sleep 2000
+run general/parser/fill_stb.sim
+sleep 2000
+run general/parser/tags_dynamically_specifiy.sim
+sleep 2000
+run general/parser/interp.sim
+run general/parser/lastrow.sim
+sleep 2000
+run general/parser/limit.sim
+sleep 2000
+run general/parser/limit1.sim
+sleep 2000
+run general/parser/limit1_tblocks100.sim
+sleep 2000
+run general/parser/limit2.sim
+sleep 2000
+run general/parser/mixed_blocks.sim
+sleep 2000
+run general/parser/nchar.sim
+sleep 2000
+run general/parser/null_char.sim
+sleep 2000
+run general/parser/selectResNum.sim
+sleep 2000
+run general/parser/select_across_vnodes.sim
 sleep 2000
-
-# sleep 2000
-# run general/parser/col_arithmetic_operation.sim
-# sleep 2000
-# run general/parser/columnValue.sim
-# sleep 2000
-# run general/parser/commit.sim
-# run general/parser/create_db.sim
-# sleep 2000
-# run general/parser/create_mt.sim
-# sleep 2000
-# run general/parser/create_tb.sim
-# sleep 2000
-# run general/parser/dbtbnameValidate.sim
-# sleep 2000
-# run general/parser/fill.sim
-# sleep 2000
-# run general/parser/fill_stb.sim
-# sleep 2000
-# run general/parser/first_last.sim
-# sleep 2000
-# run general/parser/import_commit1.sim
-# sleep 2000
-# run general/parser/import_commit2.sim
-# sleep 2000
-# run general/parser/import_commit3.sim
-# sleep 2000
-# run general/parser/import_file.sim
-# sleep 2000
-# run general/parser/insert_tb.sim
-# sleep 2000
-# run general/parser/tags_dynamically_specifiy.sim
-# sleep 2000
-# run general/parser/interp.sim
-# run general/parser/lastrow.sim
-# sleep 2000
-# run general/parser/limit.sim
-# sleep 2000
-# run general/parser/limit1.sim
-# sleep 2000
-# run general/parser/limit1_tblocks100.sim
-# sleep 2000
-# run general/parser/limit2.sim
-# sleep 2000
-# run general/parser/mixed_blocks.sim
-# sleep 2000
-# run general/parser/nchar.sim
-# sleep 2000
-# run general/parser/null_char.sim
-# sleep 2000
-# run general/parser/selectResNum.sim
-# sleep 2000
-# run general/parser/select_across_vnodes.sim
-# sleep 2000
 run general/parser/select_from_cache_disk.sim
 sleep 2000
-# run general/parser/set_tag_vals.sim
+run general/parser/set_tag_vals.sim
-# sleep 2000
+sleep 2000
-# run general/parser/single_row_in_tb.sim
+run general/parser/single_row_in_tb.sim
-# sleep 2000
+sleep 2000
-# run general/parser/slimit.sim
+run general/parser/slimit.sim
-# sleep 2000
+sleep 2000
 run general/parser/slimit1.sim
 sleep 2000
-run general/parser/slimit1_query.sim
+run general/parser/slimit_alter_tags.sim
 sleep 2000
-# run general/parser/slimit_alter_tags.sim
+run general/parser/stream_on_sys.sim
-# sleep 2000
+sleep 2000
-# run general/parser/stream_on_sys.sim
+run general/parser/stream.sim
-# sleep 2000
+sleep 2000
-# run general/parser/stream.sim
+run general/parser/tbnameIn.sim
-# sleep 2000
+sleep 2000
-# run general/parser/tbnameIn.sim
+run general/parser/where.sim
-# sleep 2000
+sleep 2000
-# run general/parser/where.sim
+#run general/parser/repeatAlter.sim
-# sleep 2000
+sleep 2000
-# #run general/parser/repeatAlter.sim
+#run general/parser/repeatStream.sim
-# sleep 2000
+sleep 2000
-# #run general/parser/repeatStream.sim
+run general/parser/join.sim
-# sleep 2000
+sleep 2000
-# run general/parser/join.sim
+run general/parser/join_multivnode.sim
-# run general/parser/join_multivnode.sim
+sleep 2000
-# run general/parser/projection_limit_offset.sim
+run general/parser/projection_limit_offset.sim
-# sleep 2000
+sleep 2000
-# run general/parser/select_with_tags.sim
+run general/parser/select_with_tags.sim
-# run general/parser/groupby.sim
+sleep 2000
+run general/parser/groupby.sim

tests/script/general/tag/bool.sim

@@ -122,7 +122,8 @@ if $rows != 100 then
   return -1
 endi
 sql select * from $mt where tgcol = 1
-if $rows != 100 then 
+if $rows != 100 then
+  print expect 100, actual:$rows
   return -1
 endi
 sql select * from $mt where tgcol <> 1

tests/script/general/tag/filter.sim

@@ -122,7 +122,8 @@ endi
 print =============== step14
 sql select count(tbcol) as c from $mt where ts > 1000 group by tgcol
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 
-if $data00 != 100 then 
+if $data00 != 100 then
+  print expect 100, actual $data00
   return -1
 endi
 

tests/script/test.sh

@@ -99,7 +99,8 @@ echo "cDebugFlag       135"                  >> $TAOS_CFG
 echo "httpDebugFlag    135"                  >> $TAOS_CFG
 echo "monitorDebugFlag 135"                  >> $TAOS_CFG
 echo "udebugFlag       135"                  >> $TAOS_CFG
-echo "clog             0"                    >> $TAOS_CFG
+echo "tablemetakeeptimer 5"                  >> $TAOS_CFG
+echo "wal              0"                    >> $TAOS_CFG
 echo "asyncLog         0"                    >> $TAOS_CFG
 echo "locale           en_US.UTF-8"          >> $TAOS_CFG
 echo " "                                     >> $TAOS_CFG