[td-171] refactor codes

src/client/inc/tscUtil.h

@@ -71,6 +71,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
                            STableMeta* pTableMeta, STableDataBlocks** dataBlocks);
 void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks);
 void tscDestroyDataBlock(STableDataBlocks* pDataBlock);
+void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf);
 
 SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes,
                                    uint32_t offset);
@@ -133,9 +134,8 @@ SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FI
 SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index);
 TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index);
 
-void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo);
+void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);
-void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size);
+void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src);
-void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src);
 void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo);
 
 int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index);
@@ -235,11 +235,6 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex);
 
 int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid);
 
-TAOS* taos_connect_a(char* ip, char* user, char* pass, char* db, uint16_t port, void (*fp)(void*, TAOS_RES*, int),
-                     void* param, void** taos);
-
-void sortRemoveDuplicates(STableDataBlocks* dataBuf);
-
 void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);
 
 bool hasMoreVnodesToTry(SSqlObj *pSql);

src/client/inc/tsclient.h

@@ -22,20 +22,28 @@ extern "C" {
 
 #include "os.h"
 
-#include "qsqlparser.h"
-#include "qsqltype.h"
-#include "qtsbuf.h"
 #include "taos.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tarray.h"
 #include "tglobal.h"
-#include "tutil.h"
 #include "tsqlfunction.h"
+#include "tutil.h"
+
+#include "qsqlparser.h"
+#include "qsqltype.h"
+#include "qtsbuf.h"
 #include "queryExecutor.h"
 
 // forward declaration
 struct SSqlInfo;
+struct SLocalReducer;
+
+// data source from sql string or from file
+enum {
+  DATA_FROM_SQL_STRING = 1,
+  DATA_FROM_DATA_FILE = 2,
+};
 
 typedef SCMSTableVgroupRspMsg SVgroupsInfo;
 
@@ -48,7 +56,10 @@ typedef struct STableComInfo {
 
 typedef struct STableMeta {
   // super table if it is created according to super table, otherwise, tableInfo is used
-  union { struct STableMeta* pSTable; STableComInfo tableInfo; };
+  union {
+    struct STableMeta *pSTable;
+    STableComInfo      tableInfo;
+  };
   uint8_t       tableType;
   int16_t       sversion;
   SCMVgroupInfo vgroupInfo;
@@ -108,8 +119,6 @@ typedef struct SColumn {
   SColumnFilterInfo *filterInfo;
 } SColumn;
 
-struct SLocalReducer;
-
 typedef struct SCond {
   uint64_t uid;
   int32_t  len;  // length of tag query condition data
@@ -215,12 +224,6 @@ typedef struct SQueryInfo {
   int64_t prjOffset;
 } SQueryInfo;
 
-// data source from sql string or from file
-enum {
-  DATA_FROM_SQL_STRING = 1,
-  DATA_FROM_DATA_FILE = 2,
-};
-
 typedef struct {
   int     command;
   uint8_t msgType;
@@ -259,8 +262,6 @@ typedef struct SResRec {
   int numOfTotal;
 } SResRec;
 
-struct STSBuf;
-
 typedef struct {
   int64_t               numOfRows;                  // num of results in current retrieved
   int64_t               numOfTotal;                 // num of total results
@@ -360,12 +361,9 @@ typedef struct SSqlStream {
 } SSqlStream;
 
 int32_t tscInitRpc(const char *user, const char *secret);
-
-// tscSql API
-int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion);
-
 void    tscInitMsgsFp();
-extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
+
+int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion);
 
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg);
 int  tscProcessSql(SSqlObj *pSql);
@@ -379,12 +377,8 @@ int tscProcessLocalCmd(SSqlObj *pSql);
 int tscCfgDynamicOptions(char *msg);
 int taos_retrieve(TAOS_RES *res);
 
-/*
+int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
- * transfer function for metric query in stream computing, the function need to be change
+void    tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
- * before send query message to vnode
- */
-int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo *pQueryInfo);
-void    tscRestoreSQLFunctionForMetricQuery(SQueryInfo *pQueryInfo);
 
 int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo);
 void    tscDestroyResPointerInfo(SSqlRes *pRes);
@@ -414,10 +408,13 @@ void tscFreeSqlObj(SSqlObj *pObj);
 
 void tscCloseTscObj(STscObj *pObj);
 
+TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
+                     void *param, void **taos);
+
 void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen);
 
 void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql);
-void tscKillMetricQuery(SSqlObj *pSql);
+void tscKillSTableQuery(SSqlObj *pSql);
 void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
 bool tscIsUpdateQuery(STscObj *pObj);
 bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
@@ -441,6 +438,8 @@ extern int        tsInsertHeadSize;
 extern int       tscNumOfThreads;
 extern SRpcIpSet tscMgmtIpSet;
 
+extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
+
 typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);
 
 #ifdef __cplusplus

src/client/src/tscAsync.c

@@ -488,7 +488,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
      */
     SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
     
-    tscTansformSQLFunctionForSTableQuery(pQueryInfo);
+    tscTansformSQLFuncForSTableQuery(pQueryInfo);
     tscIncStreamExecutionCount(pSql->pStream);
   } else {
     tscTrace("%p get tableMeta successfully", pSql);

src/client/src/tscLocal.c

@@ -296,7 +296,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) {
 
   int32_t rowLen =
       tscBuildMeterSchemaResultFields(pSql, NUM_OF_DESCRIBE_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, note_field_length);
-  tscFieldInfoCalOffset(pQueryInfo);
+  tscFieldInfoUpdateOffset(pQueryInfo);
   return tscSetValueToResObj(pSql, rowLen);
 }
 

src/client/src/tscParseInsert.c

@@ -613,7 +613,7 @@ static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, in
 }
 
 // data block is disordered, sort it in ascending order
-void sortRemoveDuplicates(STableDataBlocks *dataBuf) {
+void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) {
   SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData;
 
   // size is less than the total size, since duplicated rows may be removed yet.

src/client/src/tscSQLParser.c

@@ -1258,7 +1258,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
      * transfer sql functions that need secondary merge into another format
      * in dealing with metric queries such as: count/first/last
      */
-    tscTansformSQLFunctionForSTableQuery(pQueryInfo);
+    tscTansformSQLFuncForSTableQuery(pQueryInfo);
 
     if (hasUnsupportFunctionsForSTableQuery(pQueryInfo)) {
       return TSDB_CODE_INVALID_SQL;
@@ -2242,7 +2242,7 @@ bool validateIpAddress(const char* ip, size_t size) {
   return ipAddr != INADDR_NONE;
 }
 
-int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) {
+int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
   if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
@@ -2282,7 +2282,7 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) {
 }
 
 /* transfer the field-info back to original input format */
-void tscRestoreSQLFunctionForMetricQuery(SQueryInfo* pQueryInfo) {
+void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
     return;
@@ -2547,7 +2547,7 @@ void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo) {
   if (QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
     tscFieldInfoUpdateOffsetForInterResult(pQueryInfo);
   } else {
-    tscFieldInfoCalOffset(pQueryInfo);
+    tscFieldInfoUpdateOffset(pQueryInfo);
   }
 }
 

src/client/src/tscSchemaUtil.c

@@ -168,7 +168,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
   
   pTableMeta->sid = pTableMetaMsg->sid;
   pTableMeta->uid = pTableMetaMsg->uid;
-  pTableMeta->vgroupInfo = pTableMetaMsg->vgroup;
+//  pTableMeta->vgroupInfo = pTableMetaMsg->vgroup;
   
   memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize);
   

src/client/src/tscSecondaryMerge.c

@@ -261,8 +261,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   pReducer->pCtx = (SQLFunctionCtx *)calloc(size, sizeof(SQLFunctionCtx));
   pReducer->rowSize = pMemBuffer[0]->nElemSize;
 
-  tscRestoreSQLFunctionForMetricQuery(pQueryInfo);
+  tscRestoreSQLFuncForSTableQuery(pQueryInfo);
-  tscFieldInfoCalOffset(pQueryInfo);
+  tscFieldInfoUpdateOffset(pQueryInfo);
 
   if (pReducer->rowSize > pMemBuffer[0]->pageSize) {
     assert(false);  // todo fixed row size is larger than the minimum page size;

src/client/src/tscServer.c

@@ -458,7 +458,7 @@ int tscProcessSql(SSqlObj *pSql) {
   return doProcessSql(pSql);
 }
 
-void tscKillMetricQuery(SSqlObj *pSql) {
+void tscKillSTableQuery(SSqlObj *pSql) {
   SSqlCmd* pCmd = &pSql->cmd;
   
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
@@ -2202,7 +2202,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
                      pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes);
   }
 
-  tscFieldInfoCalOffset(pQueryInfo);
+  tscFieldInfoUpdateOffset(pQueryInfo);
   
   tfree(pTableMeta);
   return 0;

src/client/src/tscSql.c

@@ -964,7 +964,7 @@ void taos_stop_query(TAOS_RES *res) {
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
-    tscKillMetricQuery(pSql);
+    tscKillSTableQuery(pSql);
     return;
   }
 

src/client/src/tscStream.c

@@ -86,7 +86,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
     if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
   }
 
-  tscTansformSQLFunctionForSTableQuery(pQueryInfo);
+  tscTansformSQLFuncForSTableQuery(pQueryInfo);
 
   // failed to get meter/metric meta, retry in 10sec.
   if (code != TSDB_CODE_SUCCESS) {

src/client/src/tscSubquery.c

@@ -305,7 +305,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
     tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond);
   
     pQueryInfo->exprsInfo = tscSqlExprCopy(pSupporter->exprsInfo, pSupporter->uid, false);
-    tscFieldInfoCopyAll(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo);
+    tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo);
     
     pSupporter->fieldsInfo.numOfOutput = 0;
     
@@ -321,7 +321,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
     SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
     assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
   
-    tscFieldInfoCalOffset(pNewQueryInfo);
+    tscFieldInfoUpdateOffset(pNewQueryInfo);
   
     STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
   
@@ -859,7 +859,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu
     tscColumnListAssign(pSupporter->colList, pNewQueryInfo->colList, 0);
   
     pSupporter->exprsInfo = tscSqlExprCopy(pNewQueryInfo->exprsInfo, pSupporter->uid, false);
-    tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo);
+    tscFieldInfoCopy(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo);
     
     tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond);
     

src/client/src/tscUtil.c

@@ -475,6 +475,7 @@ SDataBlockList* tscCreateBlockArrayList() {
   if (pDataBlockArrayList == NULL) {
     return NULL;
   }
+  
   pDataBlockArrayList->nAlloc = DEFAULT_INITIAL_NUM_OF_BLOCK;
   pDataBlockArrayList->pData = calloc(1, POINTER_BYTES * pDataBlockArrayList->nAlloc);
   if (pDataBlockArrayList->pData == NULL) {
@@ -716,7 +717,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
     }
 
     SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
-    sortRemoveDuplicates(pOneTableBlock);
+    tscSortRemoveDataBlockDupRows(pOneTableBlock);
 
     char* e = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
     
@@ -838,7 +839,7 @@ SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FI
   return taosArrayInsert(pFieldInfo->pSupportInfo, index, &info);
 }
 
-void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo) {
+void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) {
   size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
   
   SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0);
@@ -869,26 +870,7 @@ void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) {
   }
 }
 
-void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size) {
+void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) {
-  if (src == NULL) {
-    return;
-  }
-
-  if (size <= 0) {
-    tscFieldInfoCopyAll(dst, src);
-  } else {  // only copy the required column
-    for (int32_t i = 0; i < size; ++i) {
-      assert(indexList[i] >= 0 && indexList[i] <= src->numOfOutput);
-      TAOS_FIELD* p = taosArrayGet(src->pFields, indexList[i]);
-      SFieldSupInfo* pInfo = taosArrayGet(src->pSupportInfo, indexList[i]);
-      
-      SFieldSupInfo* pInfo1 = tscFieldInfoAppend(dst, p);
-      *pInfo1 = *pInfo;
-    }
-  }
-}
-
-void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src) {
   dst->numOfOutput = src->numOfOutput;
 
   taosArrayCopy(dst->pFields, src->pFields);
@@ -1818,19 +1800,21 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
 
   int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo);
 
-  if (numOfOutput > 0) {
+  if (numOfOutput > 0) {  // todo refactor to extract method
-    int32_t* indexList = calloc(1, numOfOutput * sizeof(int32_t));
     size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
+    SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo;
     
-    for (int32_t i = 0, j = 0; i < numOfExprs; ++i) {
+    for (int32_t i = 0; i < numOfExprs; ++i) {
       SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
-      if (pExpr->uid == uid) {
-        indexList[j++] = i;
-      }
-    }
       
-    tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pNewQueryInfo->fieldsInfo, indexList, numOfOutput);
+      if (pExpr->uid == uid) {
-    free(indexList);
+        TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i);
+        SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i);
+  
+        SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p);
+        *pInfo1 = *pInfo;
+      }
+    }
 
     //     make sure the the sqlExpr for each fields is correct
 // todo handle the agg arithmetic expression
@@ -1930,9 +1914,7 @@ bool tscIsUpdateQuery(STscObj* pObj) {
 
   SSqlCmd* pCmd = &pObj->pSql->cmd;
   return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) ||
-          TSDB_SQL_USE_DB == pCmd->command)
+          TSDB_SQL_USE_DB == pCmd->command);
-             ? 1
-             : 0;
 }
 
 int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) {

src/mnode/src/mgmtTable.c

@@ -364,7 +364,7 @@ static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCta
   
   bool find = false;
   int32_t pos = 0;
-  for (int pos = 0; pos < pStable->vgLen; ++pos) {
+  for (pos = 0; pos < pStable->vgLen; ++pos) {
     if (pStable->vgList[pos] == 0) break;
     if (pStable->vgList[pos] == pCtable->vgId) {
       find = true;
@@ -1134,7 +1134,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
 static int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) {
   int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    strcpy(pSchema->name, pTable->schema[i].name);
+    strncpy(pSchema->name, pTable->schema[i].name, TSDB_TABLE_ID_LEN);
     pSchema->type  = pTable->schema[i].type;
     pSchema->bytes = htons(pTable->schema[i].bytes);
     pSchema->colId = htons(pTable->schema[i].colId);
@@ -1154,7 +1154,7 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) {
   pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns);
   pMeta->tableType    = pTable->info.type;
   pMeta->contLen      = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable);
-  strcpy(pMeta->tableId, pTable->info.tableId);
+  strncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN);
 
   SRpcMsg rpcRsp = {
     .handle = pMsg->thandle,