fix:conflict from 3.0

include/client/taos.h

@@ -338,8 +338,11 @@ typedef struct tmq_raw_data {
 DLL_EXPORT int32_t     tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw);
 DLL_EXPORT int32_t     tmq_write_raw(TAOS *taos, tmq_raw_data raw);
 DLL_EXPORT int         taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname);
+DLL_EXPORT int         taos_write_raw_block_with_reqid(TAOS *taos, int numOfRows, char *pData, const char *tbname, int64_t reqid);
 DLL_EXPORT int         taos_write_raw_block_with_fields(TAOS *taos, int rows, char *pData, const char *tbname,
                                                         TAOS_FIELD *fields, int numFields);
+DLL_EXPORT int         taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
+                                                        TAOS_FIELD *fields, int numFields, int64_t reqid);
 DLL_EXPORT void        tmq_free_raw(tmq_raw_data raw);
 
 // Returning null means error. Returned result need to be freed by tmq_free_json_meta

source/client/src/clientRawBlockWrite.c

@@ -1382,6 +1382,11 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
 
 int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields,
                                      int numFields) {
+  return taos_write_raw_block_with_fields_with_reqid(taos, rows, pData, tbname, fields, numFields, 0);
+}
+
+int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
+    TAOS_FIELD *fields, int numFields, int64_t reqid){
   if (!taos || !pData || !tbname) {
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
@@ -1391,7 +1396,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
   SQuery*     pQuery = NULL;
   SHashObj*   pVgHash = NULL;
 
-  SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
+  SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid);
   if (!pRequest) {
     return terrno;
   }
@@ -1465,6 +1470,10 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
 }
 
 int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) {
+  return taos_write_raw_block_with_reqid(taos, rows, pData, tbname, 0);
+}
+
+int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname, int64_t reqid) {
   if (!taos || !pData || !tbname) {
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
@@ -1474,7 +1483,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
   SQuery*     pQuery = NULL;
   SHashObj*   pVgHash = NULL;
 
-  SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
+  SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid);
   if (!pRequest) {
     return terrno;
   }

tests/parallel_test/container_build.sh

@@ -68,9 +68,9 @@ docker run \
     -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
     -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
     -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
+    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
     --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1"
  # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
-# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
 
 if [[ -d ${WORKDIR}/debugNoSan  ]] ;then
     echo "delete  ${WORKDIR}/debugNoSan"
@@ -97,10 +97,10 @@ docker run \
     -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
     -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
     -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
+    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
     -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
     --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true   -DBUILD_SANITIZER=1  -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 "
 
-    # -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
 mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugSan
 
 ret=$?

tools/shell/src/shellEngine.c

@@ -22,6 +22,23 @@
 #include "shellAuto.h"
 #include "shellInt.h"
 
+typedef struct {
+  const char *sql;
+  bool        vertical;
+  tsem_t      sem;
+  int64_t     numOfRows;  // the num of this batch
+  int64_t     numOfAllRows;
+
+  int32_t     numFields;
+  TAOS_FIELD *fields;
+  int32_t     precision;
+
+  int32_t maxColNameLen;            // for vertical print
+  int32_t width[TSDB_MAX_COLUMNS];  // for horizontal print
+
+  uint64_t resShowMaxNum;
+} tsDumpInfo;
+
 static bool    shellIsEmptyCommand(const char *cmd);
 static int32_t shellRunSingleCommand(char *command);
 static void    shellRecordCommandToHistory(char *command);
@@ -31,8 +48,8 @@ static char   *shellFormatTimestamp(char *buf, int64_t val, int32_t precision);
 static int64_t shellDumpResultToFile(const char *fname, TAOS_RES *tres);
 static void    shellPrintNChar(const char *str, int32_t length, int32_t width);
 static void    shellPrintGeometry(const unsigned char *str, int32_t length, int32_t width);
-static int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql);
+static void    shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo* dump_info);
-static int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql);
+static void    shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo* dump_info);
 static int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql);
 static void    shellReadHistory();
 static void    shellWriteHistory();
@@ -702,49 +719,67 @@ bool shellIsShowQuery(const char *sql) {
   return false;
 }
 
-int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) {
+void init_dump_info(tsDumpInfo *dump_info, TAOS_RES *tres, const char *sql, bool vertical) {
-  TAOS_ROW row = taos_fetch_row(tres);
+  dump_info->sql = sql;
-  if (row == NULL) {
+  dump_info->vertical = vertical;
-    return 0;
+  tsem_init(&dump_info->sem, 0, 0);
+  dump_info->numOfAllRows = 0;
+
+  dump_info->numFields = taos_num_fields(tres);
+  dump_info->fields = taos_fetch_fields(tres);
+  dump_info->precision = taos_result_precision(tres);
+
+  dump_info->resShowMaxNum = UINT64_MAX;
+
+  if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(dump_info->sql)) {
+    dump_info->resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM;
   }
 
-  int32_t     num_fields = taos_num_fields(tres);
+  if (vertical) {
-  TAOS_FIELD *fields = taos_fetch_fields(tres);
+    dump_info->maxColNameLen = 0;
-  int32_t     precision = taos_result_precision(tres);
+    for (int32_t col = 0; col < dump_info->numFields; col++) {
-
+      int32_t len = (int32_t)strlen(dump_info->fields[col].name);
-  int32_t maxColNameLen = 0;
+      if (len > dump_info->maxColNameLen) {
-  for (int32_t col = 0; col < num_fields; col++) {
+        dump_info->maxColNameLen = len;
-    int32_t len = (int32_t)strlen(fields[col].name);
+      }
-    if (len > maxColNameLen) {
+    }
-      maxColNameLen = len;
+  } else {
+    for (int32_t col = 0; col < dump_info->numFields; col++) {
+      dump_info->width[col] = shellCalcColWidth(dump_info->fields + col, dump_info->precision);
     }
   }
+}
 
-  uint64_t resShowMaxNum = UINT64_MAX;
+void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) {
-
+  TAOS_ROW row = taos_fetch_row(tres);
-  if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) {
+  if (row == NULL) {
-    resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM;
+    printf("\033[31mtaos_fetch_row failed.\033[0m\n");
+    return;
   }
 
-  int64_t numOfRows = 0;
+  int64_t numOfPintRows = dump_info->numOfAllRows;
-  int32_t showMore = 1;
+  int     numOfPrintRowsThisOne = 0;
-  do {
-    if (numOfRows < resShowMaxNum) {
-      printf("*************************** %"PRId64".row ***************************\r\n", numOfRows + 1);
 
-      int32_t *length = taos_fetch_lengths(tres);
+  while (row != NULL) {
+    printf("*************************** %" PRId64 ".row ***************************\r\n", numOfPintRows + 1);
 
-      for (int32_t i = 0; i < num_fields; i++) {
+    int32_t *length = taos_fetch_lengths(tres);
-        TAOS_FIELD *field = fields + i;
 
-        int32_t padding = (int32_t)(maxColNameLen - strlen(field->name));
+    for (int32_t i = 0; i < dump_info->numFields; i++) {
-        printf("%*.s%s: ", padding, " ", field->name);
+      TAOS_FIELD *field = dump_info->fields + i;
 
-        shellPrintField((const char *)row[i], field, 0, length[i], precision);
+      int32_t padding = (int32_t)(dump_info->maxColNameLen - strlen(field->name));
-        putchar('\r');
+      printf("%*.s%s: ", padding, " ", field->name);
-        putchar('\n');
+
-      }
+      shellPrintField((const char *)row[i], field, 0, length[i], dump_info->precision);
-    } else if (showMore) {
+      putchar('\r');
+      putchar('\n');
+    }
+
+    numOfPintRows++;
+    numOfPrintRowsThisOne++;
+
+    if (numOfPintRows == dump_info->resShowMaxNum) {
       printf("\r\n");
       printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM);
       printf("         You can use the `LIMIT` clause to get fewer result to show.\r\n");
@@ -752,14 +787,16 @@ int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) {
       printf("\r\n");
       printf("         You can use Ctrl+C to stop the underway fetching.\r\n");
       printf("\r\n");
-      showMore = 0;
+      return;
     }
 
-    numOfRows++;
+    if (numOfPrintRowsThisOne == dump_info->numOfRows) {
-    row = taos_fetch_row(tres);
+      return;
-  } while (row != NULL);
+    }
 
-  return numOfRows;
+    row = taos_fetch_row(tres);
+  }
+  return;
 }
 
 int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
@@ -856,47 +893,38 @@ void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields) {
   putchar('\n');
 }
 
-int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) {
+void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) {
   TAOS_ROW row = taos_fetch_row(tres);
   if (row == NULL) {
-    return 0;
+    printf("\033[31mtaos_fetch_row failed.\033[0m\n");
+    return;
   }
 
-  int32_t     num_fields = taos_num_fields(tres);
+  int64_t numOfPintRows = dump_info->numOfAllRows;
-  TAOS_FIELD *fields = taos_fetch_fields(tres);
+  int     numOfPrintRowsThisOne = 0;
-  int32_t     precision = taos_result_precision(tres);
+  if (numOfPintRows == 0) {
-
+    shellPrintHeader(dump_info->fields, dump_info->width, dump_info->numFields);
-  int32_t width[TSDB_MAX_COLUMNS];
-  for (int32_t col = 0; col < num_fields; col++) {
-    width[col] = shellCalcColWidth(fields + col, precision);
   }
 
-  shellPrintHeader(fields, width, num_fields);
+  while (row != NULL) {
-
-  uint64_t resShowMaxNum = UINT64_MAX;
-
-  if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) {
-    resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM;
-  }
-
-  int64_t numOfRows = 0;
-  int32_t showMore = 1;
-
-  do {
     int32_t *length = taos_fetch_lengths(tres);
-    if (numOfRows < resShowMaxNum) {
+    for (int32_t i = 0; i < dump_info->numFields; i++) {
-      for (int32_t i = 0; i < num_fields; i++) {
+      putchar(' ');
-        putchar(' ');
+      shellPrintField((const char *)row[i], dump_info->fields + i, dump_info->width[i], length[i],
-        shellPrintField((const char *)row[i], fields + i, width[i], length[i], precision);
+                      dump_info->precision);
-        putchar(' ');
+      putchar(' ');
-        putchar('|');
+      putchar('|');
-      }
+    }
-      putchar('\r');
+    putchar('\r');
-      putchar('\n');
+    putchar('\n');
-    } else if (showMore) {
+
+    numOfPintRows++;
+    numOfPrintRowsThisOne++;
+
+    if (numOfPintRows == dump_info->resShowMaxNum) {
       printf("\r\n");
       printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM);
-      if (shellIsShowQuery(sql)) {
+      if (shellIsShowQuery(dump_info->sql)) {
         printf("         You can use '>>' to redirect the whole set of the result to a specified file.\r\n");
       } else {
         printf("         You can use the `LIMIT` clause to get fewer result to show.\r\n");
@@ -905,28 +933,53 @@ int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) {
       printf("\r\n");
       printf("         You can use Ctrl+C to stop the underway fetching.\r\n");
       printf("\r\n");
-      showMore = 0;
+      return;
     }
 
-    numOfRows++;
+    if (numOfPrintRowsThisOne == dump_info->numOfRows) {
-    row = taos_fetch_row(tres);
+      return;
-  } while (row != NULL);
+    }
 
-  return numOfRows;
+    row = taos_fetch_row(tres);
+  }
+  return;
+}
+
+void shellDumpResultCallback(void *param, TAOS_RES *tres, int num_of_rows) {
+  tsDumpInfo *dump_info = (tsDumpInfo *)param;
+  if (num_of_rows > 0) {
+    dump_info->numOfRows = num_of_rows;
+    if (dump_info->numOfAllRows < dump_info->resShowMaxNum) {
+      if (dump_info->vertical) {
+        shellVerticalPrintResult(tres, dump_info);
+      } else {
+        shellHorizontalPrintResult(tres, dump_info);
+      }
+    }
+    dump_info->numOfAllRows += num_of_rows;
+    taos_fetch_rows_a(tres, shellDumpResultCallback, param);
+  } else {
+    if (num_of_rows < 0) {
+      printf("\033[31masync retrieve failed, code: %d\033[0m\n", num_of_rows);
+    }
+    tsem_post(&dump_info->sem);
+  }
 }
 
 int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql) {
-  int64_t numOfRows = 0;
+  int64_t num_of_rows = 0;
   if (fname != NULL) {
-    numOfRows = shellDumpResultToFile(fname, tres);
+    num_of_rows = shellDumpResultToFile(fname, tres);
-  } else if (vertical) {
-    numOfRows = shellVerticalPrintResult(tres, sql);
   } else {
-    numOfRows = shellHorizontalPrintResult(tres, sql);
+    tsDumpInfo dump_info;
+    init_dump_info(&dump_info, tres, sql, vertical);
+    taos_fetch_rows_a(tres, shellDumpResultCallback, &dump_info);
+    tsem_wait(&dump_info.sem);
+    num_of_rows = dump_info.numOfAllRows;
   }
 
   *error_no = taos_errno(tres);
-  return numOfRows;
+  return num_of_rows;
 }
 
 void shellReadHistory() {