From cccd5dbd8bda0ad56d736add278419754a30aefa Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 05:28:08 +0000
Subject: [PATCH 01/25] refact data

---
 include/common/tdataformat.h    |   3 +
 include/common/tmsg.h           |  13 ++-
 source/common/src/tdataformat.c | 165 ++++++++++++++++++++++++--------
 3 files changed, 138 insertions(+), 43 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 2e8a214a9d..b01b7ac240 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -38,6 +38,8 @@ typedef struct SKVIdx        SKVIdx;
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
+                  int8_t *flags);
 
 // STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
@@ -86,6 +88,7 @@ struct STSRowBuilder {
   uint8_t  *pKVBuf;
   int32_t   szTPBuf;
   uint8_t  *pTPBuf;
+  uint8_t  *pBitBuf;
   int32_t   nCols;
   int32_t   kvVLen;
   int32_t   tpVLen;
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 7f17c1673b..6294e14adb 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -271,12 +271,13 @@ typedef struct {
 
 int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
 int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
-void tFreeSSubmitRsp(SSubmitRsp *pRsp);
-
-#define COL_SMA_ON  ((int8_t)0x1)
-#define COL_IDX_ON  ((int8_t)0x2)
-#define COL_VAL_SET ((int8_t)0x4)
+void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
+#define COL_SMA_ON   ((int8_t)0x1)
+#define COL_IDX_ON   ((int8_t)0x2)
+#define COL_SET_VAL  ((int8_t)0x10)
+#define COL_SET_NONE ((int8_t)0x20)
+#define COL_SET_NULL ((int8_t)0x40)
 typedef struct SSchema {
   int8_t   type;
   int8_t   flags;
@@ -285,6 +286,8 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
+#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0)
+
 #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON)
 
 #define SSCHMEA_TYPE(s)  ((s)->type)
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index ad020fe7d9..d492d98b63 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -19,13 +19,17 @@
 #include "tdatablock.h"
 #include "tlog.h"
 
-#define TD_KV_ROW 0x1U
+#define TD_HAS_NONE 0x1U
+#define TD_HAS_NULL 0x2U
+#define TD_HAS_VAL  0x4U
+#define TD_KV_ROW   0x10U
 
 struct SKVIdx {
   int32_t cid;
   int32_t offset;
 };
 
+// STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
   if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
   if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1;
@@ -50,6 +54,59 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
   return 0;
 }
 
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
+                  int8_t *flags) {
+  if (cid == 0) {
+    *ppData = (uint8_t *)&pRow->ts;
+    *nData = sizeof(TSKEY);
+    *flags = 0;
+  } else {
+    uint32_t tflags = pRow->flags & 0xf;
+    *ppData = NULL;
+    *nData = 0;
+
+    switch (tflags) {
+      case TD_HAS_NONE:
+        *flags = -1;
+        break;
+      case TD_HAS_NULL:
+        *flags = 1;
+        break;
+      case TD_HAS_VAL:
+        *flags = 0;
+        // find the row
+        break;
+      case TD_HAS_NULL | TD_HAS_NONE:
+        // read bit map (todo)
+        if (0) {
+          *flags = 1;
+        } else {
+          *flags = -1;
+        }
+        break;
+      case TD_HAS_VAL | TD_HAS_NONE:
+      case TD_HAS_VAL | TD_HAS_NULL:
+        // read bitmap (todo)
+        if (0) {
+          if (tflags & TD_HAS_NONE) {
+            *flags = -1;
+          } else {
+            *flags = 1;
+          }
+        } else {
+          // get value (todo)
+        }
+        break;
+      case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
+        break;
+      default:
+        return -1;
+    }
+  }
+  return 0;
+}
+
+// STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) {
   *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols);
   if (*ppTSchema == NULL) {
@@ -87,6 +144,7 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema *
 
 void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); }
 
+// STSRowBuilder
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
   int32_t  kvBufLen;
   int32_t  tpBufLen;
@@ -131,7 +189,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
   for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) {
     pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
 
-    pBuilder->pTColumn->flags &= (~COL_VAL_SET);
+    pBuilder->pTColumn->flags &= 0xf;
   }
 
   pBuilder->nCols = 0;
@@ -144,7 +202,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   int32_t  iCol;
   uint8_t *p;
 
-  // search column
+  // search column (todo: make here faster)
   if (pBuilder->pTColumn->colId < cid) {
     iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
     for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
@@ -160,7 +218,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   }
 
   // check
-  if (pBuilder->pTColumn->colId != cid || pBuilder->pTColumn->flags & COL_VAL_SET) {
+  if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) {
     return -1;
   }
 
@@ -168,9 +226,12 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   if (cid == 0) {
     ASSERT(pData && nData == sizeof(TSKEY));
     pBuilder->row.ts = *(TSKEY *)pData;
+
+    pBuilder->pTColumn->flags |= COL_SET_VAL;
   } else {
     if (pData) {
-      // ASSERT(!IS_NULL(pData));
+      pBuilder->row.flags |= TD_HAS_VAL;
+      pBuilder->pTColumn->flags |= COL_SET_VAL;
 
       // set tuple data
       p = pBuilder->pTPBuf + pBuilder->pTColumn->offset;
@@ -197,51 +258,79 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
         pBuilder->kvVLen += nData;
       }
     } else {
-      // set NULL val
+      pBuilder->row.flags |= TD_HAS_NULL;
+      pBuilder->pTColumn->flags |= COL_SET_NULL;
+
+      p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
+      ((SKVIdx *)p)->cid = cid;
+      ((SKVIdx *)p)->offset = -1;
     }
+
+    pBuilder->nCols++;
   }
 
-  pBuilder->pTColumn->flags |= COL_VAL_SET;
-  pBuilder->nCols++;
   return 0;
 }
 
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
-  if ((pBuilder->pTSchema->columns[0].flags & COL_VAL_SET) == 0) {
+  int32_t  tpDataLen, kvDataLen;
+  uint32_t flags;
+
+  // error not set ts
+  if (!COL_IS_SET(pBuilder->pTSchema->columns)) {
     return -1;
   }
 
-  if (pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen < pBuilder->pTSchema->flen + pBuilder->tpVLen) {
-    // encode as TD_KV_ROW
-    pBuilder->row.flags |= TD_KV_ROW;
-    pBuilder->row.ncols = pBuilder->nCols;
-    pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen;
-    pBuilder->row.pData = pBuilder->pKVBuf;
+  if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
+    pBuilder->row.flags |= TD_HAS_NONE;
+  }
 
-    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) {
-      memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
-              pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen);
-    }
-  } else {
-    // encode as TD_TUPLE_ROW
-    pBuilder->row.flags &= (~TD_KV_ROW);
-    pBuilder->row.sver = pBuilder->pTSchema->version;
-    pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
-    pBuilder->row.pData = pBuilder->pTPBuf;
-
-    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) {
-      // set non-set cols as None
-      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
-        pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
-        if (pBuilder->pTColumn->flags & COL_VAL_SET) continue;
-
-        {
-          // set None (todo)
-        }
-
-        pBuilder->pTColumn->flags |= COL_VAL_SET;
+  flags = pBuilder->row.flags & 0xf;
+  switch (flags) {
+    case TD_HAS_NONE:
+    case TD_HAS_NULL:
+      pBuilder->row.sver = pBuilder->pTSchema->version;
+      pBuilder->row.nData = 0;
+      pBuilder->row.pData = NULL;
+      break;
+    case TD_HAS_VAL:
+      pBuilder->row.sver = pBuilder->pTSchema->version;
+      pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
+      pBuilder->row.pData = pBuilder->pTPBuf;
+      break;
+    case TD_HAS_NULL | TD_HAS_NONE:
+      pBuilder->row.sver = pBuilder->pTSchema->version;
+      // set bitmap (todo)
+      pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1;
+      pBuilder->row.pData = pBuilder->pBitBuf;
+      break;
+    case TD_HAS_VAL | TD_HAS_NONE:
+    case TD_HAS_VAL | TD_HAS_NULL:
+    case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
+      if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) {
+        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
+      } else {
+        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
       }
-    }
+      kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
+
+      if (kvDataLen < tpDataLen) {
+        pBuilder->row.flags |= TD_KV_ROW;
+        pBuilder->row.ncols = pBuilder->nCols;
+        pBuilder->row.nData = kvDataLen;
+        pBuilder->row.pData = pBuilder->pKVBuf;
+        // memmove(); todo
+        // qsort
+      } else {
+        pBuilder->row.sver = pBuilder->pTSchema->numOfCols;
+        // set bitmap etc (todo)
+        pBuilder->row.nData = tpDataLen;
+        pBuilder->row.pData = pBuilder->pTPBuf;
+      }
+      break;
+    default:
+      ASSERT(0);
+      return -1;
   }
 
   *ppRow = &pBuilder->row;

From 793b49dbf61c29fe6802e7d7658223e7442b94d9 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 10:38:35 +0000
Subject: [PATCH 02/25] feat: data format

---
 include/common/tdataformat.h    |  12 +-
 include/common/tmsg.h           |   8 +-
 source/common/src/tdataformat.c | 194 +++++++++++++++++++-------------
 3 files changed, 126 insertions(+), 88 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index b01b7ac240..de7c2a49f6 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -33,9 +33,9 @@ typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
 typedef struct SKVIdx        SKVIdx;
 
-// STSchema
-
 // STSRow2
+#define TSROW_SVER(r) (((r)->flags & TSROW_KV_ROW) ? -1 : (r)->sver)
+
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
@@ -70,9 +70,13 @@ struct STSchema {
   STColumn columns[];
 };
 
+#define TSROW_HAS_NONE ((uint8_t)0x1)
+#define TSROW_HAS_NULL ((uint8_t)0x2U)
+#define TSROW_HAS_VAL  ((uint8_t)0x4U)
+#define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY    ts;
-  uint32_t flags;
+  TSKEY   ts;
+  uint8_t flags;
   union {
     int32_t sver;
     int32_t ncols;
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 8eb9ae62d8..d860ce5b99 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -275,9 +275,8 @@ void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
 #define COL_SMA_ON   ((int8_t)0x1)
 #define COL_IDX_ON   ((int8_t)0x2)
-#define COL_SET_VAL  ((int8_t)0x10)
-#define COL_SET_NONE ((int8_t)0x20)
-#define COL_SET_NULL ((int8_t)0x40)
+#define COL_SET_NULL ((int8_t)0x10)
+#define COL_SET_VAL  ((int8_t)0x20)
 typedef struct SSchema {
   int8_t   type;
   int8_t   flags;
@@ -286,7 +285,8 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
-#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0)
+#define COL_IS_SET(FLG)  ((FLG) & (COL_SET_VAL | COL_SET_NULL) != 0)
+#define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL)))
 
 #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON)
 
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index d492d98b63..0fed7c86fe 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -19,38 +19,66 @@
 #include "tdatablock.h"
 #include "tlog.h"
 
-#define TD_HAS_NONE 0x1U
-#define TD_HAS_NULL 0x2U
-#define TD_HAS_VAL  0x4U
-#define TD_KV_ROW   0x10U
-
 struct SKVIdx {
   int32_t cid;
   int32_t offset;
 };
 
+#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
+
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
   if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
-  if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1;
-  if (pRow->flags & TD_KV_ROW) {
-    if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1;
-  } else {
-    if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
+  if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1;
+
+  ASSERT(pRow->flags & 0xf != 0);
+
+  switch (pRow->flags & 0xf) {
+    case TSROW_HAS_NONE:
+    case TSROW_HAS_NULL:
+      ASSERT(TSROW_IS_KV_ROW(pRow) && pRow->nData == 0 && pRow->pData == NULL);
+      return 0;
+    case TSROW_HAS_VAL:
+      ASSERT(!TSROW_IS_KV_ROW(pRow));
+    default:
+      ASSERT(pRow->nData && pRow->pData);
+      if (TSROW_IS_KV_ROW(pRow)) {
+        if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1;
+      } else {
+        if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
+      }
+      if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1;
+      break;
   }
-  if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData) < 0) return -1;
+
   return 0;
 }
 
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
   if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1;
-  if (tDecodeU32v(pDecoder, &pRow->flags) < 0) return -1;
-  if (pRow->flags & TD_KV_ROW) {
-    if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1;
-  } else {
-    if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
+  if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1;
+
+  ASSERT(pRow->flags & 0xf != 0);
+
+  switch (pRow->flags & 0xf) {
+    case TSROW_HAS_NONE:
+    case TSROW_HAS_NULL:
+      ASSERT(TSROW_IS_KV_ROW(pRow));
+      pRow->nData = 0;
+      pRow->pData = NULL;
+      return 0;
+    case TSROW_HAS_VAL:
+      ASSERT(!TSROW_IS_KV_ROW(pRow));
+    default:
+      if (TSROW_IS_KV_ROW(pRow)) {
+        if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1;
+      } else {
+        if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
+      }
+      if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1;
+      break;
   }
-  if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData) < 0) return -1;
+
   return 0;
 }
 
@@ -66,17 +94,17 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui
     *nData = 0;
 
     switch (tflags) {
-      case TD_HAS_NONE:
+      case TSROW_HAS_NONE:
         *flags = -1;
         break;
-      case TD_HAS_NULL:
+      case TSROW_HAS_NULL:
         *flags = 1;
         break;
-      case TD_HAS_VAL:
+      case TSROW_HAS_VAL:
         *flags = 0;
         // find the row
         break;
-      case TD_HAS_NULL | TD_HAS_NONE:
+      case TSROW_HAS_NULL | TSROW_HAS_NONE:
         // read bit map (todo)
         if (0) {
           *flags = 1;
@@ -84,11 +112,11 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui
           *flags = -1;
         }
         break;
-      case TD_HAS_VAL | TD_HAS_NONE:
-      case TD_HAS_VAL | TD_HAS_NULL:
+      case TSROW_HAS_VAL | TSROW_HAS_NONE:
+      case TSROW_HAS_VAL | TSROW_HAS_NULL:
         // read bitmap (todo)
         if (0) {
-          if (tflags & TD_HAS_NONE) {
+          if (tflags & TSROW_HAS_NONE) {
             *flags = -1;
           } else {
             *flags = 1;
@@ -97,7 +125,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui
           // get value (todo)
         }
         break;
-      case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
+      case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
         break;
       default:
         return -1;
@@ -199,18 +227,15 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
 }
 
 int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) {
-  int32_t  iCol;
-  uint8_t *p;
-
-  // search column (todo: make here faster)
+  // search column (TODO: bsearch with interp)
   if (pBuilder->pTColumn->colId < cid) {
-    iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
+    int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
     for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
       pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
       if (pBuilder->pTColumn->colId == cid) break;
     }
   } else if (pBuilder->pTColumn->colId > cid) {
-    iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1;
+    int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1;
     for (; iCol >= 0; iCol--) {
       pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
       if (pBuilder->pTColumn->colId == cid) break;
@@ -218,19 +243,20 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   }
 
   // check
-  if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) {
+  if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn->flags)) {
     return -1;
   }
 
   // set value
+  uint8_t *p;
   if (cid == 0) {
     ASSERT(pData && nData == sizeof(TSKEY));
     pBuilder->row.ts = *(TSKEY *)pData;
 
     pBuilder->pTColumn->flags |= COL_SET_VAL;
   } else {
-    if (pData) {
-      pBuilder->row.flags |= TD_HAS_VAL;
+    if (pData) {  // set val
+      pBuilder->row.flags |= TSROW_HAS_VAL;
       pBuilder->pTColumn->flags |= COL_SET_VAL;
 
       // set tuple data
@@ -250,20 +276,20 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
       ((SKVIdx *)p)->cid = cid;
       ((SKVIdx *)p)->offset = pBuilder->kvVLen;
 
-      p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols + pBuilder->kvVLen;
+      p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->kvVLen;
       if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) {
         pBuilder->kvVLen += tPutBinary(p, pData, nData);
       } else {
         memcpy(p, pData, nData);
         pBuilder->kvVLen += nData;
       }
-    } else {
-      pBuilder->row.flags |= TD_HAS_NULL;
+    } else {  // set NULL
+      pBuilder->row.flags |= TSROW_HAS_NULL;
       pBuilder->pTColumn->flags |= COL_SET_NULL;
 
       p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
       ((SKVIdx *)p)->cid = cid;
-      ((SKVIdx *)p)->offset = -1;
+      ((SKVIdx *)p)->offset = -1;  // for TSROW_KV_ROW, offset -1 means NULL
     }
 
     pBuilder->nCols++;
@@ -272,68 +298,76 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   return 0;
 }
 
+static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) {
+  SKVIdx *pKVIdx1 = (SKVIdx *)p1;
+  SKVIdx *pKVIdx2 = (SKVIdx *)p2;
+  if (pKVIdx1->cid > pKVIdx2->cid) {
+    return 1;
+  } else if (pKVIdx1->cid < pKVIdx2->cid) {
+    return -1;
+  }
+  return 0;
+}
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   int32_t  tpDataLen, kvDataLen;
   uint32_t flags;
 
   // error not set ts
-  if (!COL_IS_SET(pBuilder->pTSchema->columns)) {
+  if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) {
     return -1;
   }
 
+  ASSERT(pBuilder->nCols < pBuilder->pTSchema->numOfCols);
   if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
-    pBuilder->row.flags |= TD_HAS_NONE;
+    pBuilder->row.flags |= TSROW_HAS_NONE;
   }
 
-  flags = pBuilder->row.flags & 0xf;
-  switch (flags) {
-    case TD_HAS_NONE:
-    case TD_HAS_NULL:
-      pBuilder->row.sver = pBuilder->pTSchema->version;
+  ASSERT(pBuilder->row.flags & 0xf != 0);
+  *(ppRow) = &pBuilder->row;
+  switch (pBuilder->row.flags & 0xf) {
+    case TSROW_HAS_NONE:
+    case TSROW_HAS_NULL:
+      pBuilder->row.flags |= TSROW_KV_ROW;
       pBuilder->row.nData = 0;
       pBuilder->row.pData = NULL;
+      return 0;
+    case TSROW_HAS_NULL | TSROW_HAS_NONE:
+      tpDataLen = (pBuilder->pTSchema->numOfCols - 1) / 8;
       break;
-    case TD_HAS_VAL:
-      pBuilder->row.sver = pBuilder->pTSchema->version;
-      pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
-      pBuilder->row.pData = pBuilder->pTPBuf;
+    case TSROW_HAS_VAL:
+      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen;
       break;
-    case TD_HAS_NULL | TD_HAS_NONE:
-      pBuilder->row.sver = pBuilder->pTSchema->version;
-      // set bitmap (todo)
-      pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1;
-      pBuilder->row.pData = pBuilder->pBitBuf;
+    case TSROW_HAS_VAL | TSROW_HAS_NONE:
+    case TSROW_HAS_VAL | TSROW_HAS_NULL:
+      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 8;
       break;
-    case TD_HAS_VAL | TD_HAS_NONE:
-    case TD_HAS_VAL | TD_HAS_NULL:
-    case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
-      if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) {
-        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
-      } else {
-        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
-      }
-      kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
-
-      if (kvDataLen < tpDataLen) {
-        pBuilder->row.flags |= TD_KV_ROW;
-        pBuilder->row.ncols = pBuilder->nCols;
-        pBuilder->row.nData = kvDataLen;
-        pBuilder->row.pData = pBuilder->pKVBuf;
-        // memmove(); todo
-        // qsort
-      } else {
-        pBuilder->row.sver = pBuilder->pTSchema->numOfCols;
-        // set bitmap etc (todo)
-        pBuilder->row.nData = tpDataLen;
-        pBuilder->row.pData = pBuilder->pTPBuf;
-      }
+    case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
+      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 4;
       break;
     default:
-      ASSERT(0);
-      return -1;
+      // decide chose tuple or kv
+      kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
+      break;
+  }
+
+  if (kvDataLen < tpDataLen) {
+    pBuilder->row.flags |= TSROW_KV_ROW;
+    pBuilder->row.ncols = pBuilder->nCols;
+
+    pBuilder->row.nData = kvDataLen;
+    pBuilder->row.pData = pBuilder->pKVBuf;
+    qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
+    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
+      memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
+              pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->kvVLen);
+    }
+  } else {
+    pBuilder->row.sver = pBuilder->pTSchema->version;
+
+    pBuilder->row.nData = tpDataLen;
+    pBuilder->row.pData
   }
 
-  *ppRow = &pBuilder->row;
   return 0;
 }
 

From 6972b1d6d14ac119589434b64a3cdd43beececa0 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 10:41:29 +0000
Subject: [PATCH 03/25] make compile

---
 source/common/src/tdataformat.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 0fed7c86fe..efc8819435 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -365,7 +365,6 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
     pBuilder->row.sver = pBuilder->pTSchema->version;
 
     pBuilder->row.nData = tpDataLen;
-    pBuilder->row.pData
   }
 
   return 0;

From 038e9558bbf40a030deb95bc25e18f975d1e5c70 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 13 May 2022 01:19:34 +0000
Subject: [PATCH 04/25] more data

---
 include/common/tdataformat.h    |   9 +-
 source/common/src/tdataformat.c | 222 +++++++++++++++++++-------------
 2 files changed, 140 insertions(+), 91 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index aed7f6ddc6..ad47b82e7f 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -86,16 +86,17 @@ struct STSRow2 {
 };
 
 struct STSRowBuilder {
-  STColumn *pTColumn;
   STSchema *pTSchema;
+  int32_t   szBitMap1;
+  int32_t   szBitMap2;
   int32_t   szKVBuf;
   uint8_t  *pKVBuf;
   int32_t   szTPBuf;
   uint8_t  *pTPBuf;
-  uint8_t  *pBitBuf;
+  int32_t   iCol;
   int32_t   nCols;
-  int32_t   kvVLen;
-  int32_t   tpVLen;
+  int32_t   vlenKV;
+  int32_t   vlenTP;
   STSRow2   row;
 };
 
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index efc8819435..a951efbe20 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -25,6 +25,8 @@ struct SKVIdx {
 };
 
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
+#define SET_BIT1(p, i, v)
+#define SET_BIT2(p, i, v)
 
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
@@ -170,126 +172,135 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema *
   return 0;
 }
 
-void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); }
+void tTSchemaDestroy(STSchema *pTSchema) {
+  if (pTSchema) taosMemoryFree(pTSchema);
+}
 
 // STSRowBuilder
-int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
-  int32_t  kvBufLen;
-  int32_t  tpBufLen;
-  uint8_t *p;
+static int32_t tTSRowBitMapLen1(int32_t nCols) { return nCols / 8 + ((nCols % 8) == 0 ? 0 : 1); }
+static int32_t tTSRowBitMapLen2(int32_t nCols) { return nCols / 4 + ((nCols % 4) == 0 ? 0 : 1); }
 
+int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
   if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1;
 
-  kvBufLen = sizeof(SKVIdx) * nCols + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
-  tpBufLen = pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
-
-  if (pBuilder->szKVBuf < kvBufLen) {
-    p = taosMemoryRealloc(pBuilder->pKVBuf, kvBufLen);
-    if (p == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-    pBuilder->pKVBuf = p;
-    pBuilder->szKVBuf = kvBufLen;
+  pBuilder->szBitMap1 = tTSRowBitMapLen1(nCols - 1);
+  pBuilder->szBitMap2 = tTSRowBitMapLen2(nCols - 1);
+  pBuilder->szKVBuf = sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
+  pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
+  pBuilder->pKVBuf = taosMemoryMalloc(pBuilder->szKVBuf);
+  if (pBuilder->pKVBuf == NULL) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return -1;
   }
-
-  if (pBuilder->szTPBuf < tpBufLen) {
-    p = taosMemoryRealloc(pBuilder->pTPBuf, tpBufLen);
-    if (p == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-    pBuilder->pTPBuf = p;
-    pBuilder->szTPBuf = tpBufLen;
+  pBuilder->pTPBuf = taosMemoryMalloc(pBuilder->szTPBuf);
+  if (pBuilder->pTPBuf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
   }
 
-  tTSRowBuilderReset(pBuilder);
-
   return 0;
 }
 
 void tTSRowBuilderClear(STSRowBuilder *pBuilder) {
-  taosMemoryFree(pBuilder->pKVBuf);
-  taosMemoryFree(pBuilder->pTPBuf);
+  tTSchemaDestroy(pBuilder->pTSchema);
+  pBuilder->pTSchema = NULL;
+  if (pBuilder->pKVBuf) {
+    taosMemoryFree(pBuilder->pKVBuf);
+    pBuilder->pKVBuf = NULL;
+  }
+  if (pBuilder->pTPBuf) {
+    taosMemoryFree(pBuilder->pTPBuf);
+    pBuilder->pTPBuf = NULL;
+  }
 }
 
 void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
   for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) {
-    pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
-
-    pBuilder->pTColumn->flags &= 0xf;
+    STColumn *pTColumn = &pBuilder->pTSchema->columns[iCol];
+    COL_CLR_SET(pTColumn->flags);
   }
 
+  pBuilder->iCol = 0;
   pBuilder->nCols = 0;
-  pBuilder->kvVLen = 0;
-  pBuilder->tpVLen = 0;
+  pBuilder->vlenKV = 0;
+  pBuilder->vlenTP = 0;
   pBuilder->row.flags = 0;
 }
 
 int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) {
-  // search column (TODO: bsearch with interp)
-  if (pBuilder->pTColumn->colId < cid) {
-    int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
-    for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
-      pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
-      if (pBuilder->pTColumn->colId == cid) break;
+  STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol];
+  uint8_t  *p;
+  int32_t   iCol;
+
+  // use interp search (todo)
+  if (pTColumn->colId > cid) {
+    for (iCol = pBuilder->iCol + 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
+      pTColumn = &pBuilder->pTSchema->columns[iCol];
+      if (pTColumn->colId == cid) break;
     }
-  } else if (pBuilder->pTColumn->colId > cid) {
-    int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1;
-    for (; iCol >= 0; iCol--) {
-      pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
-      if (pBuilder->pTColumn->colId == cid) break;
+  } else if (pTColumn->colId < cid) {
+    for (iCol = pBuilder->iCol - 1; iCol >= 0; iCol--) {
+      pTColumn = &pBuilder->pTSchema->columns[iCol];
+      if (pTColumn->colId == cid) break;
     }
   }
 
-  // check
-  if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn->flags)) {
+  if (pTColumn->colId != cid || COL_IS_SET(pTColumn->flags)) {
     return -1;
   }
 
+  pBuilder->iCol = iCol;
+
   // set value
-  uint8_t *p;
   if (cid == 0) {
-    ASSERT(pData && nData == sizeof(TSKEY));
+    ASSERT(pData && nData == sizeof(TSKEY) && iCol == 0);
     pBuilder->row.ts = *(TSKEY *)pData;
-
-    pBuilder->pTColumn->flags |= COL_SET_VAL;
+    pTColumn->flags |= COL_SET_VAL;
   } else {
-    if (pData) {  // set val
+    if (pData) {
+      // set VAL
+
       pBuilder->row.flags |= TSROW_HAS_VAL;
-      pBuilder->pTColumn->flags |= COL_SET_VAL;
+      pTColumn->flags |= COL_SET_VAL;
 
-      // set tuple data
-      p = pBuilder->pTPBuf + pBuilder->pTColumn->offset;
-      if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) {
-        *(int32_t *)p = pBuilder->tpVLen;
+      /* KV */
+      if (1) {  // avoid KV at some threshold (todo)
+        p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
+        ((SKVIdx *)p)->cid = cid;
+        ((SKVIdx *)p)->offset = pBuilder->vlenKV;
 
-        // encode the variant-length data
-        p = pBuilder->pTPBuf + pBuilder->pTSchema->flen + pBuilder->tpVLen;
-        pBuilder->tpVLen += tPutBinary(p, pData, nData);
-      } else {
-        memcpy(p, pData, nData);
+        p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->vlenKV;
+        if (IS_VAR_DATA_TYPE(pTColumn->type)) {
+          ASSERT(nData <= pTColumn->bytes);
+          pBuilder->vlenKV += tPutBinary(p, pData, nData);
+        } else {
+          ASSERT(nData == pTColumn->bytes);
+          memcpy(p, pData, nData);
+          pBuilder->vlenKV += nData;
+        }
       }
 
-      // set kv data
-      p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
-      ((SKVIdx *)p)->cid = cid;
-      ((SKVIdx *)p)->offset = pBuilder->kvVLen;
+      /* TUPLE */
+      p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pTColumn->offset;
+      if (IS_VAR_DATA_TYPE(pTColumn->type)) {
+        ASSERT(nData <= pTColumn->bytes);
+        *(int32_t *)p = pBuilder->vlenTP;
 
-      p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->kvVLen;
-      if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) {
-        pBuilder->kvVLen += tPutBinary(p, pData, nData);
+        p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen;
+        pBuilder->vlenTP += tPutBinary(p, pData, nData);
       } else {
+        ASSERT(nData == pTColumn->bytes);
         memcpy(p, pData, nData);
-        pBuilder->kvVLen += nData;
       }
-    } else {  // set NULL
+    } else {
+      // set NULL
+
       pBuilder->row.flags |= TSROW_HAS_NULL;
-      pBuilder->pTColumn->flags |= COL_SET_NULL;
+      pTColumn->flags |= COL_SET_NULL;
 
       p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
       ((SKVIdx *)p)->cid = cid;
-      ((SKVIdx *)p)->offset = -1;  // for TSROW_KV_ROW, offset -1 means NULL
+      ((SKVIdx *)p)->offset = -1;
     }
 
     pBuilder->nCols++;
@@ -309,7 +320,7 @@ static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) {
   return 0;
 }
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
-  int32_t  tpDataLen, kvDataLen;
+  int32_t  nDataTP, nDataKV;
   uint32_t flags;
 
   // error not set ts
@@ -332,39 +343,76 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
       pBuilder->row.pData = NULL;
       return 0;
     case TSROW_HAS_NULL | TSROW_HAS_NONE:
-      tpDataLen = (pBuilder->pTSchema->numOfCols - 1) / 8;
+      nDataTP = pBuilder->szBitMap1;
       break;
     case TSROW_HAS_VAL:
-      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen;
+      nDataTP = pBuilder->pTSchema->flen + pBuilder->vlenTP;
       break;
     case TSROW_HAS_VAL | TSROW_HAS_NONE:
     case TSROW_HAS_VAL | TSROW_HAS_NULL:
-      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 8;
+      nDataTP = pBuilder->szBitMap1 + pBuilder->pTSchema->flen + pBuilder->vlenTP;
       break;
     case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
-      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 4;
+      nDataTP = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP;
       break;
     default:
-      // decide chose tuple or kv
-      kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
-      break;
+      ASSERT(0);
   }
 
-  if (kvDataLen < tpDataLen) {
+  nDataKV = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->vlenKV;
+  ASSERT(pBuilder->row.flags & TSROW_KV_ROW == 0);
+  if (nDataKV < nDataTP) {
+    // generate KV row
+
     pBuilder->row.flags |= TSROW_KV_ROW;
     pBuilder->row.ncols = pBuilder->nCols;
-
-    pBuilder->row.nData = kvDataLen;
+    pBuilder->row.nData = nDataKV;
     pBuilder->row.pData = pBuilder->pKVBuf;
+
     qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
     if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
       memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
-              pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->kvVLen);
+              pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->vlenKV);
     }
   } else {
-    pBuilder->row.sver = pBuilder->pTSchema->version;
+    // generate TUPLE row
 
-    pBuilder->row.nData = tpDataLen;
+    uint8_t  *p;
+    STColumn *pTColumn;
+    pBuilder->row.sver = pBuilder->pTSchema->version;
+    pBuilder->row.nData = nDataTP;
+    if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL) {
+      // no bitmap
+      pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2;
+    } else if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) {
+      // bitmap2
+      p = pBuilder->pTPBuf;
+
+      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
+        pTColumn = &pBuilder->pTSchema->columns[iCol];
+
+        if (pTColumn->flags & COL_SET_VAL) {
+          SET_BIT2(p, iCol - 1, 0x2);
+        } else if (pTColumn->flags & COL_SET_VAL) {
+          SET_BIT2(p, iCol - 1, 0x1);
+        }
+      }
+
+      pBuilder->row.pData = p;
+    } else {
+      // bitmap1
+      p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
+
+      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
+        pTColumn = &pBuilder->pTSchema->columns[iCol];
+
+        if (1) {
+          SET_BIT1(p, iCol - 1, 0x1);
+        }
+      }
+
+      pBuilder->row.pData = p;
+    }
   }
 
   return 0;

From e9ffb086a5443082e2f768dce2dac95352838c81 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 13 May 2022 15:36:26 +0000
Subject: [PATCH 05/25] more data format

---
 include/common/tdataformat.h    |  12 +--
 source/common/src/tdataformat.c | 171 +++++++++++++++++---------------
 2 files changed, 96 insertions(+), 87 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index ad47b82e7f..b8a4f6ac9d 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -34,8 +34,6 @@ typedef struct STSRowBuilder STSRowBuilder;
 typedef struct SKVIdx        SKVIdx;
 
 // STSRow2
-#define TSROW_SVER(r) (((r)->flags & TSROW_KV_ROW) ? -1 : (r)->sver)
-
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
@@ -75,12 +73,9 @@ struct STSchema {
 #define TSROW_HAS_VAL  ((uint8_t)0x4U)
 #define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY   ts;
-  uint8_t flags;
-  union {
-    int32_t sver;
-    int32_t ncols;
-  };
+  TSKEY          ts;
+  uint8_t        flags;
+  int32_t        sver;
   uint32_t       nData;
   const uint8_t *pData;
 };
@@ -94,7 +89,6 @@ struct STSRowBuilder {
   int32_t   szTPBuf;
   uint8_t  *pTPBuf;
   int32_t   iCol;
-  int32_t   nCols;
   int32_t   vlenKV;
   int32_t   vlenTP;
   STSRow2   row;
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index a951efbe20..055eb9df80 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -24,31 +24,33 @@ struct SKVIdx {
   int32_t offset;
 };
 
+#pragma pack(push, 1)
+typedef struct {
+  int16_t nCols;
+  SKVIdx  idx[];
+} STSKVRow;
+#pragma pack(pop)
+
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
-#define SET_BIT1(p, i, v)
-#define SET_BIT2(p, i, v)
+#define SET_BIT1(p, i, v)  ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8)))
+#define SET_BIT2(p, i, v)  ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4)))
 
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
   if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
   if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1;
+  if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
 
   ASSERT(pRow->flags & 0xf != 0);
 
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      ASSERT(TSROW_IS_KV_ROW(pRow) && pRow->nData == 0 && pRow->pData == NULL);
       return 0;
     case TSROW_HAS_VAL:
       ASSERT(!TSROW_IS_KV_ROW(pRow));
     default:
       ASSERT(pRow->nData && pRow->pData);
-      if (TSROW_IS_KV_ROW(pRow)) {
-        if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1;
-      } else {
-        if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
-      }
       if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1;
       break;
   }
@@ -59,24 +61,19 @@ int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
   if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1;
   if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
 
   ASSERT(pRow->flags & 0xf != 0);
 
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      ASSERT(TSROW_IS_KV_ROW(pRow));
       pRow->nData = 0;
       pRow->pData = NULL;
       return 0;
     case TSROW_HAS_VAL:
       ASSERT(!TSROW_IS_KV_ROW(pRow));
     default:
-      if (TSROW_IS_KV_ROW(pRow)) {
-        if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1;
-      } else {
-        if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
-      }
       if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1;
       break;
   }
@@ -86,6 +83,7 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
 
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
                   int8_t *flags) {
+#if 0
   if (cid == 0) {
     *ppData = (uint8_t *)&pRow->ts;
     *nData = sizeof(TSKEY);
@@ -133,6 +131,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui
         return -1;
     }
   }
+#endif
   return 0;
 }
 
@@ -177,24 +176,25 @@ void tTSchemaDestroy(STSchema *pTSchema) {
 }
 
 // STSRowBuilder
-static int32_t tTSRowBitMapLen1(int32_t nCols) { return nCols / 8 + ((nCols % 8) == 0 ? 0 : 1); }
-static int32_t tTSRowBitMapLen2(int32_t nCols) { return nCols / 4 + ((nCols % 4) == 0 ? 0 : 1); }
-
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
   if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1;
 
-  pBuilder->szBitMap1 = tTSRowBitMapLen1(nCols - 1);
-  pBuilder->szBitMap2 = tTSRowBitMapLen2(nCols - 1);
-  pBuilder->szKVBuf = sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
+  pBuilder->szBitMap1 = (nCols - 2) / 8 + 1;
+  pBuilder->szBitMap2 = (nCols - 2) / 4 + 1;
+  pBuilder->szKVBuf =
+      sizeof(STSKVRow) + sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
   pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
   pBuilder->pKVBuf = taosMemoryMalloc(pBuilder->szKVBuf);
   if (pBuilder->pKVBuf == NULL) {
-    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    tTSchemaDestroy(pBuilder->pTSchema);
     return -1;
   }
   pBuilder->pTPBuf = taosMemoryMalloc(pBuilder->szTPBuf);
   if (pBuilder->pTPBuf == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(pBuilder->pKVBuf);
+    tTSchemaDestroy(pBuilder->pTSchema);
     return -1;
   }
 
@@ -202,16 +202,16 @@ int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchem
 }
 
 void tTSRowBuilderClear(STSRowBuilder *pBuilder) {
-  tTSchemaDestroy(pBuilder->pTSchema);
-  pBuilder->pTSchema = NULL;
-  if (pBuilder->pKVBuf) {
-    taosMemoryFree(pBuilder->pKVBuf);
-    pBuilder->pKVBuf = NULL;
-  }
   if (pBuilder->pTPBuf) {
     taosMemoryFree(pBuilder->pTPBuf);
     pBuilder->pTPBuf = NULL;
   }
+  if (pBuilder->pKVBuf) {
+    taosMemoryFree(pBuilder->pKVBuf);
+    pBuilder->pKVBuf = NULL;
+  }
+  tTSchemaDestroy(pBuilder->pTSchema);
+  pBuilder->pTSchema = NULL;
 }
 
 void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
@@ -221,7 +221,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
   }
 
   pBuilder->iCol = 0;
-  pBuilder->nCols = 0;
+  ((STSKVRow *)pBuilder->pKVBuf)->nCols = 0;
   pBuilder->vlenKV = 0;
   pBuilder->vlenTP = 0;
   pBuilder->row.flags = 0;
@@ -231,6 +231,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol];
   uint8_t  *p;
   int32_t   iCol;
+  STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf;
 
   // use interp search (todo)
   if (pTColumn->colId > cid) {
@@ -265,11 +266,11 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
 
       /* KV */
       if (1) {  // avoid KV at some threshold (todo)
-        p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
-        ((SKVIdx *)p)->cid = cid;
-        ((SKVIdx *)p)->offset = pBuilder->vlenKV;
+        pTSKVRow->idx[pTSKVRow->nCols].cid = cid;
+        pTSKVRow->idx[pTSKVRow->nCols].offset = pBuilder->vlenKV;
 
-        p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->vlenKV;
+        p = pBuilder->pKVBuf + sizeof(STSKVRow) + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) +
+            pBuilder->vlenKV;
         if (IS_VAR_DATA_TYPE(pTColumn->type)) {
           ASSERT(nData <= pTColumn->bytes);
           pBuilder->vlenKV += tPutBinary(p, pData, nData);
@@ -286,7 +287,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
         ASSERT(nData <= pTColumn->bytes);
         *(int32_t *)p = pBuilder->vlenTP;
 
-        p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen;
+        p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP;
         pBuilder->vlenTP += tPutBinary(p, pData, nData);
       } else {
         ASSERT(nData == pTColumn->bytes);
@@ -298,12 +299,11 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
       pBuilder->row.flags |= TSROW_HAS_NULL;
       pTColumn->flags |= COL_SET_NULL;
 
-      p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
-      ((SKVIdx *)p)->cid = cid;
-      ((SKVIdx *)p)->offset = -1;
+      pTSKVRow->idx[pTSKVRow->nCols].cid = cid;
+      pTSKVRow->idx[pTSKVRow->nCols].offset = -1;
     }
 
-    pBuilder->nCols++;
+    pTSKVRow->nCols++;
   }
 
   return 0;
@@ -319,17 +319,54 @@ static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) {
   }
   return 0;
 }
+static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) {
+  int32_t   bidx;
+  STColumn *pTColumn;
+
+  for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
+    pTColumn = &pTSchema->columns[iCol];
+    bidx = iCol - 1;
+
+    switch (flags) {
+      case TSROW_HAS_NULL | TSROW_HAS_NONE:
+        if (pTColumn->flags & COL_SET_NULL) {
+          SET_BIT1(p, bidx, (uint8_t)1);
+        } else {
+          SET_BIT1(p, bidx, (uint8_t)0);
+        }
+        break;
+      case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
+        if (pTColumn->flags & COL_SET_NULL) {
+          SET_BIT2(p, bidx, (uint8_t)1);
+        } else if (pTColumn->flags & COL_SET_NULL) {
+          SET_BIT2(p, bidx, (uint8_t)2);
+        } else {
+          SET_BIT2(p, bidx, (uint8_t)0);
+        }
+        break;
+      default:
+        if (pTColumn->flags & COL_SET_VAL) {
+          SET_BIT1(p, bidx, (uint8_t)1);
+        } else {
+          SET_BIT1(p, bidx, (uint8_t)0);
+        }
+
+        break;
+    }
+  }
+}
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
-  int32_t  nDataTP, nDataKV;
-  uint32_t flags;
+  int32_t   nDataTP, nDataKV;
+  uint32_t  flags;
+  STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf;
 
   // error not set ts
   if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) {
     return -1;
   }
 
-  ASSERT(pBuilder->nCols < pBuilder->pTSchema->numOfCols);
-  if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
+  ASSERT(pTSKVRow->nCols < pBuilder->pTSchema->numOfCols);
+  if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) {
     pBuilder->row.flags |= TSROW_HAS_NONE;
   }
 
@@ -338,7 +375,6 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   switch (pBuilder->row.flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      pBuilder->row.flags |= TSROW_KV_ROW;
       pBuilder->row.nData = 0;
       pBuilder->row.pData = NULL;
       return 0;
@@ -359,58 +395,37 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
       ASSERT(0);
   }
 
-  nDataKV = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->vlenKV;
-  ASSERT(pBuilder->row.flags & TSROW_KV_ROW == 0);
+  nDataKV = sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pBuilder->vlenKV;
+  pBuilder->row.sver = pBuilder->pTSchema->version;
   if (nDataKV < nDataTP) {
     // generate KV row
 
     pBuilder->row.flags |= TSROW_KV_ROW;
-    pBuilder->row.ncols = pBuilder->nCols;
     pBuilder->row.nData = nDataKV;
     pBuilder->row.pData = pBuilder->pKVBuf;
 
-    qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
-    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
-      memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
-              pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->vlenKV);
+    qsort(pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
+    if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) {
+      memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[pBuilder->pTSchema->numOfCols - 1], pBuilder->vlenKV);
     }
   } else {
     // generate TUPLE row
 
-    uint8_t  *p;
-    STColumn *pTColumn;
-    pBuilder->row.sver = pBuilder->pTSchema->version;
     pBuilder->row.nData = nDataTP;
-    if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL) {
-      // no bitmap
+
+    uint8_t *p;
+    uint8_t  flags = pBuilder->row.flags & 0xf;
+
+    if (flags == TSROW_HAS_VAL) {
       pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2;
-    } else if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) {
-      // bitmap2
-      p = pBuilder->pTPBuf;
-
-      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
-        pTColumn = &pBuilder->pTSchema->columns[iCol];
-
-        if (pTColumn->flags & COL_SET_VAL) {
-          SET_BIT2(p, iCol - 1, 0x2);
-        } else if (pTColumn->flags & COL_SET_VAL) {
-          SET_BIT2(p, iCol - 1, 0x1);
-        }
-      }
-
-      pBuilder->row.pData = p;
     } else {
-      // bitmap1
-      p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
-
-      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
-        pTColumn = &pBuilder->pTSchema->columns[iCol];
-
-        if (1) {
-          SET_BIT1(p, iCol - 1, 0x1);
-        }
+      if (flags == TSROW_HAS_VAL) {
+        p = pBuilder->pTPBuf;
+      } else {
+        p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
       }
 
+      setBitMap(p, pBuilder->pTSchema, flags);
       pBuilder->row.pData = p;
     }
   }

From 34a219b5fc5589ed45bbc6434d0138edb5e35233 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 14 May 2022 06:18:07 +0000
Subject: [PATCH 06/25] feat: impl tsrow read

---
 include/common/tdataformat.h    |  52 +++++++---
 source/common/src/tdataformat.c | 163 +++++++++++++++++++++-----------
 2 files changed, 150 insertions(+), 65 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index b8a4f6ac9d..ab0415d5d8 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -29,20 +29,41 @@ extern "C" {
 typedef struct SSchema       SSchema;
 typedef struct STColumn      STColumn;
 typedef struct STSchema      STSchema;
+typedef struct SColVal       SColVal;
 typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
-typedef struct SKVIdx        SKVIdx;
-
-// STSRow2
-int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
-int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
-int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
-                  int8_t *flags);
 
 // STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
 void    tTSchemaDestroy(STSchema *pTSchema);
 
+// SColVal
+#define COL_VAL_SET_NONE(CV) \
+  do {                       \
+    (CV)->type = COL_NONE;   \
+    (CV)->nData = 0;         \
+    (CV)->pData = NULL;      \
+  } while (0)
+
+#define COL_VAL_SET_NULL(CV) \
+  do {                       \
+    (CV)->type = COL_NULL;   \
+    (CV)->nData = 0;         \
+    (CV)->pData = NULL;      \
+  } while (0)
+
+#define COL_VAL_SET_VAL(CV, PDATA, NDATA) \
+  do {                                    \
+    (CV)->type = COL_VAL;                 \
+    (CV)->nData = (NDATA);                \
+    (CV)->pData = (PDATA);                \
+  } while (0)
+
+// STSRow2
+int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
+int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
+
 // STSRowBuilder
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols);
 void    tTSRowBuilderClear(STSRowBuilder *pBuilder);
@@ -73,11 +94,11 @@ struct STSchema {
 #define TSROW_HAS_VAL  ((uint8_t)0x4U)
 #define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY          ts;
-  uint8_t        flags;
-  int32_t        sver;
-  uint32_t       nData;
-  const uint8_t *pData;
+  TSKEY    ts;
+  uint8_t  flags;
+  int32_t  sver;
+  uint32_t nData;
+  uint8_t *pData;
 };
 
 struct STSRowBuilder {
@@ -94,6 +115,13 @@ struct STSRowBuilder {
   STSRow2   row;
 };
 
+typedef enum { COL_VAL = 0, COL_NONE = 1, COL_NULL = 2 } EColValT;
+struct SColVal {
+  EColValT type;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
 #if 1  //====================================
 // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
 #define TD_SUPPORT_BITMAP
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 055eb9df80..4b1df237da 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -19,10 +19,10 @@
 #include "tdatablock.h"
 #include "tlog.h"
 
-struct SKVIdx {
+typedef struct SKVIdx {
   int32_t cid;
   int32_t offset;
-};
+} SKVIdx;
 
 #pragma pack(push, 1)
 typedef struct {
@@ -34,6 +34,8 @@ typedef struct {
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
 #define SET_BIT1(p, i, v)  ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8)))
 #define SET_BIT2(p, i, v)  ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4)))
+#define GET_BIT1(p, i)     (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
+#define GET_BIT2(p, i)     (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3))
 
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
@@ -81,57 +83,112 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
   return 0;
 }
 
-int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
-                  int8_t *flags) {
-#if 0
-  if (cid == 0) {
-    *ppData = (uint8_t *)&pRow->ts;
-    *nData = sizeof(TSKEY);
-    *flags = 0;
-  } else {
-    uint32_t tflags = pRow->flags & 0xf;
-    *ppData = NULL;
-    *nData = 0;
+static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
+  col_id_t cid = *(col_id_t *)p1;
+  SKVIdx  *pKVIdx = (SKVIdx *)p2;
 
-    switch (tflags) {
-      case TSROW_HAS_NONE:
-        *flags = -1;
-        break;
-      case TSROW_HAS_NULL:
-        *flags = 1;
-        break;
-      case TSROW_HAS_VAL:
-        *flags = 0;
-        // find the row
-        break;
-      case TSROW_HAS_NULL | TSROW_HAS_NONE:
-        // read bit map (todo)
-        if (0) {
-          *flags = 1;
-        } else {
-          *flags = -1;
-        }
-        break;
-      case TSROW_HAS_VAL | TSROW_HAS_NONE:
-      case TSROW_HAS_VAL | TSROW_HAS_NULL:
-        // read bitmap (todo)
-        if (0) {
-          if (tflags & TSROW_HAS_NONE) {
-            *flags = -1;
-          } else {
-            *flags = 1;
-          }
-        } else {
-          // get value (todo)
-        }
-        break;
-      case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
-        break;
-      default:
-        return -1;
-    }
+  if (cid < pKVIdx->cid) {
+    return -1;
+  } else if (cid > pKVIdx->cid) {
+    return 1;
   }
-#endif
+  return 0;
+}
+
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
+  uint32_t  n;
+  uint8_t  *p;
+  uint8_t   v;
+  int32_t   bidx = iCol - 1;
+  STColumn *pTColumn = &pTSchema->columns[iCol];
+  STSKVRow *pTSKVRow;
+  SKVIdx   *pKVIdx;
+
+  ASSERT(pTColumn->colId != 0);
+
+  ASSERT(pRow->flags & 0xf != 0);
+  switch (pRow->flags & 0xf) {
+    case TSROW_HAS_NONE:
+      COL_VAL_SET_NONE(pColVal);
+      return 0;
+    case TSROW_HAS_NULL:
+      COL_VAL_SET_NULL(pColVal);
+      return 0;
+  }
+
+  if (TSROW_IS_KV_ROW(pRow)) {
+    ASSERT((pRow->flags & 0xf) != TSROW_HAS_VAL);
+
+    pTSKVRow = (STSKVRow *)pRow->pData;
+    pKVIdx = bsearch(&pTColumn->colId, pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), kvRowCmprFn);
+    if (pKVIdx == NULL) {
+      COL_VAL_SET_NONE(pColVal);
+    } else if (pKVIdx->offset < 0) {
+      COL_VAL_SET_NULL(pColVal);
+    } else {
+      p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset;
+      tGetBinary(p, &p, &n);
+      COL_VAL_SET_VAL(pColVal, p, n);
+    }
+  } else {
+    // get bitmap
+    switch (pRow->flags & 0xf) {
+      p = pRow->pData;
+      case TSROW_HAS_NULL | TSROW_HAS_NONE:
+        v = GET_BIT1(p, bidx);
+        if (v == 0) {
+          COL_VAL_SET_NONE(pColVal);
+        } else {
+          COL_VAL_SET_NULL(pColVal);
+        }
+        return 0;
+      case TSROW_HAS_VAL | TSROW_HAS_NONE:
+        v = GET_BIT1(p, bidx);
+        if (v == 1) {
+          p = p + (pTSchema->numOfCols - 2) / 8 + 1;
+          break;
+        } else {
+          COL_VAL_SET_NONE(pColVal);
+          return 0;
+        }
+      case TSROW_HAS_VAL | TSROW_HAS_NULL:
+        v = GET_BIT1(p, bidx);
+        if (v == 1) {
+          p = p + (pTSchema->numOfCols - 2) / 8 + 1;
+          break;
+        } else {
+          COL_VAL_SET_NULL(pColVal);
+          return 0;
+        }
+      case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
+        v = GET_BIT2(p, bidx);
+        if (v == 0) {
+          COL_VAL_SET_NONE(pColVal);
+          return 0;
+        } else if (v == 1) {
+          COL_VAL_SET_NULL(pColVal);
+          return 0;
+        } else if (v == 2) {
+          p = p + (pTSchema->numOfCols - 2) / 4 + 1;
+          break;
+        } else {
+          ASSERT(0);
+        }
+      default:
+        break;
+    }
+
+    // get real value
+    p = p + pTColumn->offset;
+    if (IS_VAR_DATA_TYPE(pTColumn->type)) {
+      p = pTSchema->flen + *(int32_t *)p;
+      tGetBinary(p, &p, &n);
+    } else {
+      n = pTColumn->bytes;
+    }
+    COL_VAL_SET_VAL(pColVal, p, n);
+  }
+
   return 0;
 }
 
@@ -445,8 +502,8 @@ int         tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   spaceNeeded += (int)nBitmapBytes;
   // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more
   // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered.
-  // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove
-  // the additional space
+  // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus
+  // remove the additional space
 #endif
 
   if (pCol->spaceSize < spaceNeeded) {

From acf1b14b21e1612097b4f3fd294f9cc7842ed218 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 16 May 2022 01:47:35 +0000
Subject: [PATCH 07/25] fix compile problem

---
 include/common/tdataformat.h    | 16 ++++++++--------
 source/common/src/tdataformat.c | 19 +++++++++----------
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index ab0415d5d8..1a31b384d8 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -94,11 +94,11 @@ struct STSchema {
 #define TSROW_HAS_VAL  ((uint8_t)0x4U)
 #define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY    ts;
-  uint8_t  flags;
-  int32_t  sver;
-  uint32_t nData;
-  uint8_t *pData;
+  TSKEY          ts;
+  uint8_t        flags;
+  int32_t        sver;
+  uint32_t       nData;
+  const uint8_t *pData;
 };
 
 struct STSRowBuilder {
@@ -117,9 +117,9 @@ struct STSRowBuilder {
 
 typedef enum { COL_VAL = 0, COL_NONE = 1, COL_NULL = 2 } EColValT;
 struct SColVal {
-  EColValT type;
-  uint32_t nData;
-  uint8_t *pData;
+  EColValT       type;
+  uint32_t       nData;
+  const uint8_t *pData;
 };
 
 #if 1  //====================================
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 4b1df237da..d421c452ca 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -96,13 +96,13 @@ static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
 }
 
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
-  uint32_t  n;
-  uint8_t  *p;
-  uint8_t   v;
-  int32_t   bidx = iCol - 1;
-  STColumn *pTColumn = &pTSchema->columns[iCol];
-  STSKVRow *pTSKVRow;
-  SKVIdx   *pKVIdx;
+  uint32_t       n;
+  const uint8_t *p;
+  uint8_t        v;
+  int32_t        bidx = iCol - 1;
+  STColumn      *pTColumn = &pTSchema->columns[iCol];
+  STSKVRow      *pTSKVRow;
+  SKVIdx        *pKVIdx;
 
   ASSERT(pTColumn->colId != 0);
 
@@ -132,8 +132,8 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     }
   } else {
     // get bitmap
+    p = pRow->pData;
     switch (pRow->flags & 0xf) {
-      p = pRow->pData;
       case TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT1(p, bidx);
         if (v == 0) {
@@ -181,8 +181,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     // get real value
     p = p + pTColumn->offset;
     if (IS_VAR_DATA_TYPE(pTColumn->type)) {
-      p = pTSchema->flen + *(int32_t *)p;
-      tGetBinary(p, &p, &n);
+      tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n);
     } else {
       n = pTColumn->bytes;
     }

From de5828e4856c8d02251ea1b6c8598cd2113d7c11 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 16 May 2022 11:35:37 +0000
Subject: [PATCH 08/25] refact more

---
 source/common/test/dataformatTest.cpp |  0
 source/common/type/type.c             | 14 --------------
 source/common/type/typeBigint.c       | 14 --------------
 source/common/type/typeBinary.c       | 14 --------------
 source/common/type/typeBlob.c         | 14 --------------
 source/common/type/typeBool.c         | 14 --------------
 source/common/type/typeDecimal.c      | 14 --------------
 source/common/type/typeDouble.c       | 14 --------------
 source/common/type/typeFloat.c        | 14 --------------
 source/common/type/typeInt.c          | 14 --------------
 source/common/type/typeJson.c         | 14 --------------
 source/common/type/typeLongblob.c     | 14 --------------
 source/common/type/typeNchar.c        | 14 --------------
 source/common/type/typeNull.c         | 14 --------------
 source/common/type/typeSmallint.c     | 14 --------------
 source/common/type/typeTimestamp.c    | 14 --------------
 source/common/type/typeTinyint.c      | 14 --------------
 source/common/type/typeUBigint.c      | 14 --------------
 source/common/type/typeUSmallint.c    | 14 --------------
 source/common/type/typeUTinyint.c     | 14 --------------
 source/common/type/typeUint.c         | 14 --------------
 source/common/type/typeVarchar.c      | 14 --------------
 22 files changed, 294 deletions(-)
 create mode 100644 source/common/test/dataformatTest.cpp
 delete mode 100644 source/common/type/type.c
 delete mode 100644 source/common/type/typeBigint.c
 delete mode 100644 source/common/type/typeBinary.c
 delete mode 100644 source/common/type/typeBlob.c
 delete mode 100644 source/common/type/typeBool.c
 delete mode 100644 source/common/type/typeDecimal.c
 delete mode 100644 source/common/type/typeDouble.c
 delete mode 100644 source/common/type/typeFloat.c
 delete mode 100644 source/common/type/typeInt.c
 delete mode 100644 source/common/type/typeJson.c
 delete mode 100644 source/common/type/typeLongblob.c
 delete mode 100644 source/common/type/typeNchar.c
 delete mode 100644 source/common/type/typeNull.c
 delete mode 100644 source/common/type/typeSmallint.c
 delete mode 100644 source/common/type/typeTimestamp.c
 delete mode 100644 source/common/type/typeTinyint.c
 delete mode 100644 source/common/type/typeUBigint.c
 delete mode 100644 source/common/type/typeUSmallint.c
 delete mode 100644 source/common/type/typeUTinyint.c
 delete mode 100644 source/common/type/typeUint.c
 delete mode 100644 source/common/type/typeVarchar.c

diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/source/common/type/type.c b/source/common/type/type.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/type.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeBigint.c b/source/common/type/typeBigint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeBigint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeBinary.c b/source/common/type/typeBinary.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeBinary.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeBlob.c b/source/common/type/typeBlob.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeBlob.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeBool.c b/source/common/type/typeBool.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeBool.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeDecimal.c b/source/common/type/typeDecimal.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeDecimal.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeDouble.c b/source/common/type/typeDouble.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeDouble.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeFloat.c b/source/common/type/typeFloat.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeFloat.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeInt.c b/source/common/type/typeInt.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeInt.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeJson.c b/source/common/type/typeJson.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeJson.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeLongblob.c b/source/common/type/typeLongblob.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeLongblob.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeNchar.c b/source/common/type/typeNchar.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeNchar.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeNull.c b/source/common/type/typeNull.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeNull.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeSmallint.c b/source/common/type/typeSmallint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeSmallint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeTimestamp.c b/source/common/type/typeTimestamp.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeTimestamp.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeTinyint.c b/source/common/type/typeTinyint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeTinyint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeUBigint.c b/source/common/type/typeUBigint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeUBigint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeUSmallint.c b/source/common/type/typeUSmallint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeUSmallint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeUTinyint.c b/source/common/type/typeUTinyint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeUTinyint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeUint.c b/source/common/type/typeUint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeUint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeVarchar.c b/source/common/type/typeVarchar.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeVarchar.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file

From 96eecc4d3ee21be96f8641b6fd9454acdcd178ed Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 16 May 2022 11:43:48 +0000
Subject: [PATCH 09/25] more refact

---
 source/common/test/CMakeLists.txt     |  9 +++++++++
 source/common/test/dataformatTest.cpp |  1 +
 source/common/test/trowTest.cpp       | 23 -----------------------
 3 files changed, 10 insertions(+), 23 deletions(-)
 delete mode 100644 source/common/test/trowTest.cpp

diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt
index a0406e099c..0535b08be7 100644
--- a/source/common/test/CMakeLists.txt
+++ b/source/common/test/CMakeLists.txt
@@ -17,6 +17,15 @@ TARGET_INCLUDE_DIRECTORIES(
         PRIVATE "${TD_SOURCE_DIR}/source/libs/common/inc"
 )
 
+# dataformatTest.cpp 
+add_executable(dataformatTest "")
+target_sources(
+    dataformatTest
+    PRIVATE
+    "dataformatTest.cpp"
+)
+target_link_libraries(dataformatTest gtest gtest_main util)
+
 # tmsg test
 # add_executable(tmsgTest "")
 # target_sources(tmsgTest 
diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp
index e69de29bb2..3497014c22 100644
--- a/source/common/test/dataformatTest.cpp
+++ b/source/common/test/dataformatTest.cpp
@@ -0,0 +1 @@
+#include "gtest/gtest.h"
\ No newline at end of file
diff --git a/source/common/test/trowTest.cpp b/source/common/test/trowTest.cpp
deleted file mode 100644
index d7f0783d4a..0000000000
--- a/source/common/test/trowTest.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-#include <gtest/gtest.h>
-
-#include "trow.h"
-
-TEST(td_row_test, build_row_to_target) {
-#if 0
-  char        dst[1024];
-  SRow*       pRow = (SRow*)dst;
-  int         ncols = 10;
-  col_id_t    cid;
-  void*       pData;
-  SRowBuilder rb = trbInit(TD_OR_ROW_BUILDER, NULL, 0, pRow, 1024);
-
-  trbSetRowInfo(&rb, false, 0);
-  trbSetRowTS(&rb, 1637550210000);
-  for (int c = 0; c < ncols; c++) {
-    cid = c;
-    if (trbWriteCol(&rb, pData, cid) < 0) {
-      // TODO
-    }
-  }
-#endif
-}
\ No newline at end of file

From 93c095b1a56e515be082a7b5ff1b6fed4aa9f58d Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 05:46:22 +0000
Subject: [PATCH 10/25] make compile

---
 include/common/tdataformat.h    |   4 +-
 include/util/tencode.h          | 205 ++++++++++++++++++++++++++------
 source/common/src/tdataformat.c |   6 +-
 3 files changed, 177 insertions(+), 38 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 1a31b384d8..1097f62798 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -62,13 +62,15 @@ void    tTSchemaDestroy(STSchema *pTSchema);
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
+int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
+int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 
 // STSRowBuilder
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols);
 void    tTSRowBuilderClear(STSRowBuilder *pBuilder);
 void    tTSRowBuilderReset(STSRowBuilder *pBuilder);
-int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData);
+int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData);
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow);
 
 // STRUCT =================
diff --git a/include/util/tencode.h b/include/util/tencode.h
index e49429f865..484c9443f7 100644
--- a/include/util/tencode.h
+++ b/include/util/tencode.h
@@ -456,52 +456,189 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) {
   return p;
 }
 
-static FORCE_INLINE int32_t tPutBinary(uint8_t* p, const uint8_t* pData, uint32_t nData) {
-  int      n = 0;
-  uint32_t v = nData;
+// ===========================================
+#define tPutV(p, v)                    \
+  do {                                 \
+    int32_t n = 0;                     \
+    for (;;) {                         \
+      if (v <= 0x7f) {                 \
+        if (p) p[n] = v;               \
+        n++;                           \
+        break;                         \
+      }                                \
+      if (p) p[n] = (v & 0x7f) | 0x80; \
+      n++;                             \
+      v >>= 7;                         \
+    }                                  \
+    return n;                          \
+  } while (0)
 
-  for (;;) {
-    if (v <= 0x7f) {
-      if (p) p[n] = v;
-      n++;
-      break;
-    }
+#define tGetV(p, v)                              \
+  do {                                           \
+    int32_t n = 0;                               \
+    if (v) *v = 0;                               \
+    for (;;) {                                   \
+      if (p[n] <= 0x7f) {                        \
+        if (v) (*v) |= (p[n] << (7 * n));        \
+        n++;                                     \
+        break;                                   \
+      }                                          \
+      if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \
+      n++;                                       \
+    }                                            \
+    return n;                                    \
+  } while (0)
 
-    if (p) p[n] = (v & 0x7f) | 0x80;
-    n++;
-    v >>= 7;
-  }
+// PUT
+static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) {
+  if (p) ((uint8_t*)p)[0] = v;
+  return sizeof(uint8_t);
+}
 
-  if (p) {
-    memcpy(p + n, pData, nData);
-  }
+static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) {
+  if (p) ((int8_t*)p)[0] = v;
+  return sizeof(int8_t);
+}
+
+static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) {
+  if (p) ((uint16_t*)p)[0] = v;
+  return sizeof(uint16_t);
+}
+
+static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) {
+  if (p) ((int16_t*)p)[0] = v;
+  return sizeof(int16_t);
+}
+
+static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) {
+  if (p) ((uint32_t*)p)[0] = v;
+  return sizeof(uint32_t);
+}
+
+static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) {
+  if (p) ((int32_t*)p)[0] = v;
+  return sizeof(int32_t);
+}
+
+static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) {
+  if (p) ((uint64_t*)p)[0] = v;
+  return sizeof(uint64_t);
+}
+
+static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) {
+  if (p) ((int64_t*)p)[0] = v;
+  return sizeof(int64_t);
+}
+
+static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); }
+
+static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); }
+
+static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); }
+
+// GET
+static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) {
+  if (v) *v = ((uint8_t*)p)[0];
+  return sizeof(uint8_t);
+}
+
+static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) {
+  if (v) *v = ((int8_t*)p)[0];
+  return sizeof(int8_t);
+}
+
+static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) {
+  if (v) *v = ((uint16_t*)p)[0];
+  return sizeof(uint16_t);
+}
+
+static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) {
+  if (v) *v = ((int16_t*)p)[0];
+  return sizeof(int16_t);
+}
+
+static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) {
+  if (v) *v = ((uint32_t*)p)[0];
+  return sizeof(uint32_t);
+}
+
+static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) {
+  if (v) *v = ((int32_t*)p)[0];
+  return sizeof(int32_t);
+}
+
+static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) {
+  if (v) *v = ((uint64_t*)p)[0];
+  return sizeof(uint64_t);
+}
+
+static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) {
+  if (v) *v = ((int64_t*)p)[0];
+  return sizeof(int64_t);
+}
+
+static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) {
+  int32_t  n;
+  uint16_t tv;
+
+  n = tGetU16v(p, &tv);
+  if (v) *v = ZIGZAGD(int16_t, tv);
+
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) {
+  int32_t  n;
+  uint32_t tv;
+
+  n = tGetU32v(p, &tv);
+  if (v) *v = ZIGZAGD(int32_t, tv);
+
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) {
+  int32_t  n;
+  uint64_t tv;
+
+  n = tGetU64v(p, &tv);
+  if (v) *v = ZIGZAGD(int64_t, tv);
+
+  return n;
+}
+
+// =====================
+static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) {
+  int n = 0;
+
+  n += tPutU32v(p ? p + n : p, nData);
+  if (p) memcpy(p + n, pData, nData);
   n += nData;
 
   return n;
 }
 
-static FORCE_INLINE int32_t tGetBinary(const uint8_t* p, const uint8_t** ppData, uint32_t* nData) {
+static FORCE_INLINE int32_t tGetBinary(uint8_t* p, const uint8_t** ppData, uint32_t* nData) {
   int32_t  n = 0;
-  uint32_t tv = 0;
-  uint32_t t;
+  uint32_t nt;
 
-  for (;;) {
-    if (p[n] <= 0x7f) {
-      t = p[n];
-      tv |= (t << (7 * n));
-      n++;
-      break;
-    }
-
-    t = p[n] & 0x7f;
-    tv |= (t << (7 * n));
-    n++;
-  }
-
-  if (nData) *nData = n;
+  n += tGetU32v(p, &nt);
+  if (nData) *nData = nt;
   if (ppData) *ppData = p + n;
+  n += nt;
 
-  n += tv;
   return n;
 }
 
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index d421c452ca..a37d8931ad 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -127,7 +127,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
       COL_VAL_SET_NULL(pColVal);
     } else {
       p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset;
-      tGetBinary(p, &p, &n);
+      // tGetBinary(p, &p, &n); (todo)
       COL_VAL_SET_VAL(pColVal, p, n);
     }
   } else {
@@ -181,7 +181,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     // get real value
     p = p + pTColumn->offset;
     if (IS_VAR_DATA_TYPE(pTColumn->type)) {
-      tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n);
+      // tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n); (todo)
     } else {
       n = pTColumn->bytes;
     }
@@ -283,7 +283,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
   pBuilder->row.flags = 0;
 }
 
-int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) {
+int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData) {
   STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol];
   uint8_t  *p;
   int32_t   iCol;

From 4568c672d643e5ff72fed338d91c9d9d5bac3456 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 06:40:37 +0000
Subject: [PATCH 11/25] feat: tdata

---
 include/common/tdataformat.h    | 43 ++++++--------------
 include/util/tencode.h          |  2 +-
 source/common/src/tdataformat.c | 69 ++++++++++++++++-----------------
 3 files changed, 46 insertions(+), 68 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 1097f62798..bf2ddc0920 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -38,30 +38,11 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema *
 void    tTSchemaDestroy(STSchema *pTSchema);
 
 // SColVal
-#define COL_VAL_SET_NONE(CV) \
-  do {                       \
-    (CV)->type = COL_NONE;   \
-    (CV)->nData = 0;         \
-    (CV)->pData = NULL;      \
-  } while (0)
-
-#define COL_VAL_SET_NULL(CV) \
-  do {                       \
-    (CV)->type = COL_NULL;   \
-    (CV)->nData = 0;         \
-    (CV)->pData = NULL;      \
-  } while (0)
-
-#define COL_VAL_SET_VAL(CV, PDATA, NDATA) \
-  do {                                    \
-    (CV)->type = COL_VAL;                 \
-    (CV)->nData = (NDATA);                \
-    (CV)->pData = (PDATA);                \
-  } while (0)
+#define ColValNONE               ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL})
+#define ColValNULL               ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL})
+#define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)})
 
 // STSRow2
-int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
-int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
@@ -96,11 +77,11 @@ struct STSchema {
 #define TSROW_HAS_VAL  ((uint8_t)0x4U)
 #define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY          ts;
-  uint8_t        flags;
-  int32_t        sver;
-  uint32_t       nData;
-  const uint8_t *pData;
+  TSKEY    ts;
+  uint8_t  flags;
+  int32_t  sver;
+  uint32_t nData;
+  uint8_t *pData;
 };
 
 struct STSRowBuilder {
@@ -117,11 +98,11 @@ struct STSRowBuilder {
   STSRow2   row;
 };
 
-typedef enum { COL_VAL = 0, COL_NONE = 1, COL_NULL = 2 } EColValT;
+typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_Data = 2 } EColValT;
 struct SColVal {
-  EColValT       type;
-  uint32_t       nData;
-  const uint8_t *pData;
+  EColValT type;
+  uint32_t nData;
+  uint8_t *pData;
 };
 
 #if 1  //====================================
diff --git a/include/util/tencode.h b/include/util/tencode.h
index 484c9443f7..938e3018a8 100644
--- a/include/util/tencode.h
+++ b/include/util/tencode.h
@@ -630,7 +630,7 @@ static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nDat
   return n;
 }
 
-static FORCE_INLINE int32_t tGetBinary(uint8_t* p, const uint8_t** ppData, uint32_t* nData) {
+static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData) {
   int32_t  n = 0;
   uint32_t nt;
 
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index a37d8931ad..fbc2c2306d 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -38,49 +38,46 @@ typedef struct {
 #define GET_BIT2(p, i)     (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3))
 
 // STSRow2
-int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
-  if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
-  if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1;
-  if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
+int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) {
+  int32_t n = 0;
 
-  ASSERT(pRow->flags & 0xf != 0);
+  n += tPutI64(p ? p + n : p, pRow->ts);
+  n += tPutI8(p ? p + n : p, pRow->flags);
+  n += tPutI32v(p ? p + n : p, pRow->sver);
+
+  ASSERT(pRow->flags & 0xf);
 
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      return 0;
-    case TSROW_HAS_VAL:
-      ASSERT(!TSROW_IS_KV_ROW(pRow));
+      break;
     default:
-      ASSERT(pRow->nData && pRow->pData);
-      if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1;
+      n += tPutBinary(p ? p + n : p, pRow->pData, pRow->nData);
       break;
   }
 
-  return 0;
+  return n;
 }
 
-int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
-  if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1;
-  if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
+int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) {
+  int32_t n = 0;
+  uint8_t flags;
 
-  ASSERT(pRow->flags & 0xf != 0);
+  n += tGetI64(p + n, pRow ? &pRow->ts : NULL);
+  n += tGetI8(p + n, pRow ? &pRow->flags : &flags);
+  n += tGetI32v(p + n, pRow ? &pRow->sver : NULL);
 
-  switch (pRow->flags & 0xf) {
+  if (pRow) flags = pRow->flags;
+  switch (flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      pRow->nData = 0;
-      pRow->pData = NULL;
-      return 0;
-    case TSROW_HAS_VAL:
-      ASSERT(!TSROW_IS_KV_ROW(pRow));
+      break;
     default:
-      if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1;
+      n += tGetBinary(p + n, pRow ? &pRow->pData : NULL, pRow ? &pRow->nData : NULL);
       break;
   }
 
-  return 0;
+  return n;
 }
 
 static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
@@ -109,10 +106,10 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
   ASSERT(pRow->flags & 0xf != 0);
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
-      COL_VAL_SET_NONE(pColVal);
+      // COL_VAL_SET_NONE(pColVal);
       return 0;
     case TSROW_HAS_NULL:
-      COL_VAL_SET_NULL(pColVal);
+      // COL_VAL_SET_NULL(pColVal);
       return 0;
   }
 
@@ -122,13 +119,13 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     pTSKVRow = (STSKVRow *)pRow->pData;
     pKVIdx = bsearch(&pTColumn->colId, pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), kvRowCmprFn);
     if (pKVIdx == NULL) {
-      COL_VAL_SET_NONE(pColVal);
+      // COL_VAL_SET_NONE(pColVal);
     } else if (pKVIdx->offset < 0) {
-      COL_VAL_SET_NULL(pColVal);
+      // COL_VAL_SET_NULL(pColVal);
     } else {
       p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset;
       // tGetBinary(p, &p, &n); (todo)
-      COL_VAL_SET_VAL(pColVal, p, n);
+      // COL_VAL_SET_VAL(pColVal, p, n);
     }
   } else {
     // get bitmap
@@ -137,9 +134,9 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
       case TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT1(p, bidx);
         if (v == 0) {
-          COL_VAL_SET_NONE(pColVal);
+          // COL_VAL_SET_NONE(pColVal);
         } else {
-          COL_VAL_SET_NULL(pColVal);
+          // COL_VAL_SET_NULL(pColVal);
         }
         return 0;
       case TSROW_HAS_VAL | TSROW_HAS_NONE:
@@ -148,7 +145,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
           p = p + (pTSchema->numOfCols - 2) / 8 + 1;
           break;
         } else {
-          COL_VAL_SET_NONE(pColVal);
+          // COL_VAL_SET_NONE(pColVal);
           return 0;
         }
       case TSROW_HAS_VAL | TSROW_HAS_NULL:
@@ -157,16 +154,16 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
           p = p + (pTSchema->numOfCols - 2) / 8 + 1;
           break;
         } else {
-          COL_VAL_SET_NULL(pColVal);
+          // COL_VAL_SET_NULL(pColVal);
           return 0;
         }
       case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT2(p, bidx);
         if (v == 0) {
-          COL_VAL_SET_NONE(pColVal);
+          // COL_VAL_SET_NONE(pColVal);
           return 0;
         } else if (v == 1) {
-          COL_VAL_SET_NULL(pColVal);
+          // COL_VAL_SET_NULL(pColVal);
           return 0;
         } else if (v == 2) {
           p = p + (pTSchema->numOfCols - 2) / 4 + 1;
@@ -185,7 +182,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     } else {
       n = pTColumn->bytes;
     }
-    COL_VAL_SET_VAL(pColVal, p, n);
+    // COL_VAL_SET_VAL(pColVal, p, n);
   }
 
   return 0;

From af8ac6e1b08704e77c0d558b02bf4a85529951b5 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 07:54:38 +0000
Subject: [PATCH 12/25] more tdata

---
 include/common/tdataformat.h    |  4 +-
 source/common/src/tdataformat.c | 65 ++++++++++++++++++++++++---------
 2 files changed, 50 insertions(+), 19 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index bf2ddc0920..af035d0d1f 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -45,10 +45,12 @@ void    tTSchemaDestroy(STSchema *pTSchema);
 // STSRow2
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
+int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow);
+void    tTSRowFree(STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 
 // STSRowBuilder
-int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols);
+int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema);
 void    tTSRowBuilderClear(STSRowBuilder *pBuilder);
 void    tTSRowBuilderReset(STSRowBuilder *pBuilder);
 int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData);
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index fbc2c2306d..3f179ac58a 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -32,6 +32,8 @@ typedef struct {
 #pragma pack(pop)
 
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
+#define BIT1_SIZE(n)       (((n)-1) / 8 + 1)
+#define BIT2_SIZE(n)       (((n)-1) / 4 + 1)
 #define SET_BIT1(p, i, v)  ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8)))
 #define SET_BIT2(p, i, v)  ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4)))
 #define GET_BIT1(p, i)     (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
@@ -80,6 +82,31 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) {
   return n;
 }
 
+int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow) {
+  (*ppRow) = taosMemoryMalloc(sizeof(*pRow) + pRow->nData);
+  if (*ppRow == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  (*ppRow)->ts = pRow->ts;
+  (*ppRow)->flags = pRow->flags;
+  (*ppRow)->sver = pRow->sver;
+  (*ppRow)->nData = pRow->nData;
+  if (pRow->nData) {
+    (*ppRow)->pData = (uint8_t *)(&(*ppRow)[1]);
+    memcpy((*ppRow)->pData, pRow->pData, pRow->nData);
+  } else {
+    (*ppRow)->pData = NULL;
+  }
+
+  return 0;
+}
+
+void tTSRowFree(STSRow2 *pRow) {
+  if (pRow) taosMemoryFree(pRow);
+}
+
 static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
   col_id_t cid = *(col_id_t *)p1;
   SKVIdx  *pKVIdx = (SKVIdx *)p2;
@@ -229,11 +256,11 @@ void tTSchemaDestroy(STSchema *pTSchema) {
 }
 
 // STSRowBuilder
-int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
+int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema) {
   if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1;
 
-  pBuilder->szBitMap1 = (nCols - 2) / 8 + 1;
-  pBuilder->szBitMap2 = (nCols - 2) / 4 + 1;
+  pBuilder->szBitMap1 = BIT1_SIZE(nCols - 1);
+  pBuilder->szBitMap2 = BIT2_SIZE(nCols - 1);
   pBuilder->szKVBuf =
       sizeof(STSKVRow) + sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
   pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
@@ -286,16 +313,16 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, u
   int32_t   iCol;
   STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf;
 
-  // use interp search (todo)
-  if (pTColumn->colId > cid) {
+  // use interp search
+  if (pTColumn->colId < cid) {  // right search
     for (iCol = pBuilder->iCol + 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
       pTColumn = &pBuilder->pTSchema->columns[iCol];
-      if (pTColumn->colId == cid) break;
+      if (pTColumn->colId >= cid) break;
     }
-  } else if (pTColumn->colId < cid) {
+  } else if (pTColumn->colId > cid) {  // left search
     for (iCol = pBuilder->iCol - 1; iCol >= 0; iCol--) {
       pTColumn = &pBuilder->pTSchema->columns[iCol];
-      if (pTColumn->colId == cid) break;
+      if (pTColumn->colId <= cid) break;
     }
   }
 
@@ -391,7 +418,7 @@ static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) {
       case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
         if (pTColumn->flags & COL_SET_NULL) {
           SET_BIT2(p, bidx, (uint8_t)1);
-        } else if (pTColumn->flags & COL_SET_NULL) {
+        } else if (pTColumn->flags & COL_SET_VAL) {
           SET_BIT2(p, bidx, (uint8_t)2);
         } else {
           SET_BIT2(p, bidx, (uint8_t)0);
@@ -412,14 +439,15 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   int32_t   nDataTP, nDataKV;
   uint32_t  flags;
   STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf;
+  int32_t   nCols = pBuilder->pTSchema->numOfCols;
 
   // error not set ts
   if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) {
     return -1;
   }
 
-  ASSERT(pTSKVRow->nCols < pBuilder->pTSchema->numOfCols);
-  if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) {
+  ASSERT(pTSKVRow->nCols < nCols);
+  if (pTSKVRow->nCols < nCols - 1) {
     pBuilder->row.flags |= TSROW_HAS_NONE;
   }
 
@@ -453,13 +481,15 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   if (nDataKV < nDataTP) {
     // generate KV row
 
+    ASSERT(pBuilder->row.flags & 0xf != TSROW_HAS_VAL);
+
     pBuilder->row.flags |= TSROW_KV_ROW;
     pBuilder->row.nData = nDataKV;
     pBuilder->row.pData = pBuilder->pKVBuf;
 
     qsort(pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
-    if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) {
-      memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[pBuilder->pTSchema->numOfCols - 1], pBuilder->vlenKV);
+    if (pTSKVRow->nCols < nCols - 1) {
+      memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[nCols - 1], pBuilder->vlenKV);
     }
   } else {
     // generate TUPLE row
@@ -472,14 +502,13 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
     if (flags == TSROW_HAS_VAL) {
       pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2;
     } else {
-      if (flags == TSROW_HAS_VAL) {
-        p = pBuilder->pTPBuf;
+      if (flags == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) {
+        pBuilder->row.pData = pBuilder->pTPBuf;
       } else {
-        p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
+        pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
       }
 
-      setBitMap(p, pBuilder->pTSchema, flags);
-      pBuilder->row.pData = p;
+      setBitMap(pBuilder->row.pData, pBuilder->pTSchema, flags);
     }
   }
 

From aaf78935b9a1468b6e093f126c0aed4c07a5b781 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 08:13:58 +0000
Subject: [PATCH 13/25] more tdata

---
 include/common/tdataformat.h    |  2 +-
 source/common/src/tdataformat.c | 55 ++++++++++++++-------------------
 2 files changed, 25 insertions(+), 32 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index af035d0d1f..7e411cb301 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -100,7 +100,7 @@ struct STSRowBuilder {
   STSRow2   row;
 };
 
-typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_Data = 2 } EColValT;
+typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT;
 struct SColVal {
   EColValT type;
   uint32_t nData;
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 3f179ac58a..a5447c016f 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -39,6 +39,8 @@ typedef struct {
 #define GET_BIT1(p, i)     (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
 #define GET_BIT2(p, i)     (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3))
 
+static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2);
+
 // STSRow2
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) {
   int32_t n = 0;
@@ -107,18 +109,6 @@ void tTSRowFree(STSRow2 *pRow) {
   if (pRow) taosMemoryFree(pRow);
 }
 
-static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
-  col_id_t cid = *(col_id_t *)p1;
-  SKVIdx  *pKVIdx = (SKVIdx *)p2;
-
-  if (cid < pKVIdx->cid) {
-    return -1;
-  } else if (cid > pKVIdx->cid) {
-    return 1;
-  }
-  return 0;
-}
-
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
   uint32_t       n;
   const uint8_t *p;
@@ -128,15 +118,16 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
   STSKVRow      *pTSKVRow;
   SKVIdx        *pKVIdx;
 
+  ASSERT(iCol != 0);
   ASSERT(pTColumn->colId != 0);
 
   ASSERT(pRow->flags & 0xf != 0);
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
-      // COL_VAL_SET_NONE(pColVal);
+      *pColVal = ColValNONE;
       return 0;
     case TSROW_HAS_NULL:
-      // COL_VAL_SET_NULL(pColVal);
+      *pColVal = ColValNULL;
       return 0;
   }
 
@@ -144,15 +135,16 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     ASSERT((pRow->flags & 0xf) != TSROW_HAS_VAL);
 
     pTSKVRow = (STSKVRow *)pRow->pData;
-    pKVIdx = bsearch(&pTColumn->colId, pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), kvRowCmprFn);
+    pKVIdx =
+        bsearch(&((SKVIdx){.cid = pTColumn->colId}), pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
     if (pKVIdx == NULL) {
-      // COL_VAL_SET_NONE(pColVal);
+      *pColVal = ColValNONE;
     } else if (pKVIdx->offset < 0) {
-      // COL_VAL_SET_NULL(pColVal);
+      *pColVal = ColValNULL;
     } else {
       p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset;
-      // tGetBinary(p, &p, &n); (todo)
-      // COL_VAL_SET_VAL(pColVal, p, n);
+      pColVal->type = COL_VAL_DATA;
+      tGetBinary(p, &pColVal->pData, &pColVal->nData);
     }
   } else {
     // get bitmap
@@ -161,39 +153,39 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
       case TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT1(p, bidx);
         if (v == 0) {
-          // COL_VAL_SET_NONE(pColVal);
+          *pColVal = ColValNONE;
         } else {
-          // COL_VAL_SET_NULL(pColVal);
+          *pColVal = ColValNULL;
         }
         return 0;
       case TSROW_HAS_VAL | TSROW_HAS_NONE:
         v = GET_BIT1(p, bidx);
         if (v == 1) {
-          p = p + (pTSchema->numOfCols - 2) / 8 + 1;
+          p = p + BIT1_SIZE(pTSchema->numOfCols - 1);
           break;
         } else {
-          // COL_VAL_SET_NONE(pColVal);
+          *pColVal = ColValNONE;
           return 0;
         }
       case TSROW_HAS_VAL | TSROW_HAS_NULL:
         v = GET_BIT1(p, bidx);
         if (v == 1) {
-          p = p + (pTSchema->numOfCols - 2) / 8 + 1;
+          p = p + BIT1_SIZE(pTSchema->numOfCols - 1);
           break;
         } else {
-          // COL_VAL_SET_NULL(pColVal);
+          *pColVal = ColValNULL;
           return 0;
         }
       case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT2(p, bidx);
         if (v == 0) {
-          // COL_VAL_SET_NONE(pColVal);
+          *pColVal = ColValNONE;
           return 0;
         } else if (v == 1) {
-          // COL_VAL_SET_NULL(pColVal);
+          *pColVal = ColValNULL;
           return 0;
         } else if (v == 2) {
-          p = p + (pTSchema->numOfCols - 2) / 4 + 1;
+          p = p + BIT2_SIZE(pTSchema->numOfCols - 1);
           break;
         } else {
           ASSERT(0);
@@ -204,12 +196,13 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
 
     // get real value
     p = p + pTColumn->offset;
+    pColVal->type = COL_VAL_DATA;
     if (IS_VAR_DATA_TYPE(pTColumn->type)) {
-      // tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n); (todo)
+      tGetBinary(p + pTSchema->flen + *(int32_t *)p, &pColVal->pData, &pColVal->nData);
     } else {
-      n = pTColumn->bytes;
+      pColVal->pData = p;
+      pColVal->nData = pTColumn->bytes;
     }
-    // COL_VAL_SET_VAL(pColVal, p, n);
   }
 
   return 0;

From d6521a0a52a47c3a4257dd7957e483316877bce8 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 08:16:23 +0000
Subject: [PATCH 14/25] more tdata

---
 source/common/src/tdataformat.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index a5447c016f..4b9531aa6b 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -110,13 +110,13 @@ void tTSRowFree(STSRow2 *pRow) {
 }
 
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
-  uint32_t       n;
-  const uint8_t *p;
-  uint8_t        v;
-  int32_t        bidx = iCol - 1;
-  STColumn      *pTColumn = &pTSchema->columns[iCol];
-  STSKVRow      *pTSKVRow;
-  SKVIdx        *pKVIdx;
+  uint32_t  n;
+  uint8_t  *p;
+  uint8_t   v;
+  int32_t   bidx = iCol - 1;
+  STColumn *pTColumn = &pTSchema->columns[iCol];
+  STSKVRow *pTSKVRow;
+  SKVIdx   *pKVIdx;
 
   ASSERT(iCol != 0);
   ASSERT(pTColumn->colId != 0);

From 3a851ca9f29b09cc1d2e2b37e045f8b7cad61558 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 09:50:21 +0000
Subject: [PATCH 15/25] more tdata

---
 include/common/tdataformat.h    |  18 +++++-
 source/common/src/tdataformat.c | 104 +++++++++++++++++++++++++++++++-
 2 files changed, 120 insertions(+), 2 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 7e411cb301..e13705d403 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -32,6 +32,8 @@ typedef struct STSchema      STSchema;
 typedef struct SColVal       SColVal;
 typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
+typedef struct STagVal       STagVal;
+typedef struct STag          STag;
 
 // STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
@@ -56,6 +58,13 @@ void    tTSRowBuilderReset(STSRowBuilder *pBuilder);
 int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData);
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow);
 
+// STag
+int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag);
+void    tTagFree(STag *pTag);
+void    tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData);
+int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag);
+int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag);
+
 // STRUCT =================
 struct STColumn {
   col_id_t colId;
@@ -107,7 +116,14 @@ struct SColVal {
   uint8_t *pData;
 };
 
-#if 1  //====================================
+struct STagVal {
+  int16_t  cid;
+  int8_t   type;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
+#if 1  //================================================================================================================================================
 // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
 #define TD_SUPPORT_BITMAP
 #define TD_SUPPORT_READ2
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 4b9531aa6b..8aa8ed2f14 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -31,6 +31,19 @@ typedef struct {
 } STSKVRow;
 #pragma pack(pop)
 
+typedef struct STagIdx {
+  int16_t  cid;
+  uint16_t offset;
+} STagIdx;
+
+#pragma pack(push, 1)
+struct STag {
+  uint16_t len;
+  uint16_t nTag;
+  STagIdx  idx[];
+};
+#pragma pack(pop)
+
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
 #define BIT1_SIZE(n)       (((n)-1) / 8 + 1)
 #define BIT2_SIZE(n)       (((n)-1) / 4 + 1)
@@ -508,7 +521,96 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   return 0;
 }
 
-#if 1  // ====================
+static FORCE_INLINE int tTagIdxCmprFn(const void *p1, const void *p2) {
+  STagIdx *pTagIdx1 = (STagIdx *)p1;
+  STagIdx *pTagIdx2 = (STagIdx *)p2;
+  if (pTagIdx1->cid < pTagIdx1->cid) {
+    return -1;
+  } else if (pTagIdx1->cid > pTagIdx1->cid) {
+    return 1;
+  }
+  return 0;
+}
+int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag) {
+  STagVal *pTagVal;
+  uint8_t *p;
+  int32_t  n;
+  uint16_t tsize = sizeof(STag) + sizeof(STagIdx) * nTag;
+
+  for (int16_t iTag = 0; iTag < nTag; iTag++) {
+    pTagVal = &pTagVals[iTag];
+
+    if (IS_VAR_DATA_TYPE(pTagVal->type)) {
+      tsize += tPutBinary(NULL, pTagVal->pData, pTagVal->nData);
+    } else {
+      ASSERT(pTagVal->nData == TYPE_BYTES[pTagVal->type]);
+      tsize += pTagVal->nData;
+    }
+  }
+
+  (*ppTag) = (STag *)taosMemoryMalloc(tsize);
+  if (*ppTag == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  p = (uint8_t *)&((*ppTag)->idx[nTag]);
+  n = 0;
+
+  (*ppTag)->len = tsize;
+  (*ppTag)->nTag = nTag;
+  for (int16_t iTag = 0; iTag < nTag; iTag++) {
+    pTagVal = &pTagVals[iTag];
+
+    (*ppTag)->idx[iTag].cid = pTagVal->cid;
+    (*ppTag)->idx[iTag].offset = n;
+
+    if (IS_VAR_DATA_TYPE(pTagVal->type)) {
+      n += tPutBinary(p + n, pTagVal->pData, pTagVal->nData);
+    } else {
+      memcpy(p + n, pTagVal->pData, pTagVal->nData);
+      n += pTagVal->nData;
+    }
+  }
+
+  qsort((*ppTag)->idx, (*ppTag)->nTag, sizeof(STagIdx), tTagIdxCmprFn);
+  return 0;
+}
+
+void tTagFree(STag *pTag) {
+  if (pTag) taosMemoryFree(pTag);
+}
+
+void tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData) {
+  STagIdx *pTagIdx = bsearch(&((STagIdx){.cid = cid}), pTag->idx, pTag->nTag, sizeof(STagIdx), tTagIdxCmprFn);
+  if (pTagIdx == NULL) {
+    *ppData = NULL;
+    *nData = 0;
+  } else {
+    uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag] + pTagIdx->offset;
+    if (IS_VAR_DATA_TYPE(type)) {
+      tGetBinary(p, ppData, nData);
+    } else {
+      *ppData = p;
+      *nData = TYPE_BYTES[type];
+    }
+  }
+}
+
+int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag) {
+  // return tEncodeBinary(pEncoder, (uint8_t *)pTag, pTag->len);
+  ASSERT(0);
+  return 0;
+}
+
+int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag) {
+  // uint32_t n;
+  // return tDecodeBinary(pDecoder, (const uint8_t **)ppTag, &n);
+  ASSERT(0);
+  return 0;
+}
+
+#if 1  // ===================================================================================================================
 static void dataColSetNEleNull(SDataCol *pCol, int nEle);
 int         tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   int spaceNeeded = pCol->bytes * maxPoints;

From 37e10aca292dcd483c9dbf78a6271a6cae72f6a7 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 17 May 2022 21:46:46 +0800
Subject: [PATCH 16/25] fix:precision problems in time conversion

---
 source/client/src/clientSml.c  |  10 ++-
 source/client/test/smlTest.cpp |  33 +++++++++
 source/common/src/ttime.c      | 122 +++++++++++++++++++++++++++++----
 3 files changed, 149 insertions(+), 16 deletions(-)

diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 884756ced9..f6f8db8590 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -714,25 +714,31 @@ static bool smlIsNchar(const char *pVal, uint16_t len) {
 
 static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) {
   char *endPtr = NULL;
-  double ts = (double)strtoll(value, &endPtr, 10);
+  int64_t tsInt64 = strtoll(value, &endPtr, 10);
   if(value + len != endPtr){
     return -1;
   }
+  double ts = tsInt64;
   switch (type) {
     case TSDB_TIME_PRECISION_HOURS:
       ts *= (3600 * 1e9);
+      tsInt64 *= (3600 * 1e9);
       break;
     case TSDB_TIME_PRECISION_MINUTES:
       ts *= (60 * 1e9);
+      tsInt64 *= (60 * 1e9);
       break;
     case TSDB_TIME_PRECISION_SECONDS:
       ts *= (1e9);
+      tsInt64 *= (1e9);
       break;
     case TSDB_TIME_PRECISION_MILLI:
       ts *= (1e6);
+      tsInt64 *= (1e6);
       break;
     case TSDB_TIME_PRECISION_MICRO:
       ts *= (1e3);
+      tsInt64 *= (1e3);
       break;
     case TSDB_TIME_PRECISION_NANO:
       break;
@@ -743,7 +749,7 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) {
     return -1;
   }
 
-  return (int64_t)ts;
+  return tsInt64;
 }
 
 static int64_t smlGetTimeNow(int8_t precision) {
diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp
index eeab844712..d9a81ad3e6 100644
--- a/source/client/test/smlTest.cpp
+++ b/source/client/test/smlTest.cpp
@@ -1188,3 +1188,36 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) {
   destroyRequest(request);
   smlDestroyInfo(info);
 }
+
+TEST(testCase, sml_TD15662_Test) {
+  TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  ASSERT_NE(taos, nullptr);
+
+  TAOS_RES *pRes = taos_query(taos, "create database if not exists db_15662 precision 'ns'");
+  taos_free_result(pRes);
+
+  pRes = taos_query(taos, "use db_15662");
+  taos_free_result(pRes);
+
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT);
+  ASSERT_NE(request, nullptr);
+
+  SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
+  ASSERT_NE(info, nullptr);
+
+  const char *sql[] = {
+      "iyyyje,id=iyyyje_41943_1303,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000",
+  };
+  int ret = smlProcess(info, (char **)sql, sizeof(sql) / sizeof(sql[0]));
+  ASSERT_EQ(ret, 0);
+
+  // case 1
+  TAOS_RES *res = taos_query(taos, "select * from t_a5615048edae55218a22a149edebdc82");
+  ASSERT_NE(res, nullptr);
+
+  TAOS_ROW row = taos_fetch_row(res);
+  int64_t ts = *(int64_t*)row[0];
+  ASSERT_EQ(ts, 1626006833639000000);
+
+  taos_free_result(res);
+}
\ No newline at end of file
diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c
index c2892c070f..c121b2d83d 100644
--- a/source/common/src/ttime.c
+++ b/source/common/src/ttime.c
@@ -374,39 +374,133 @@ char getPrecisionUnit(int32_t precision) {
 }
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) {
-  assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
+  assert(fromPrecision == TSDB_TIME_PRECISION_MILLI ||
+         fromPrecision == TSDB_TIME_PRECISION_MICRO ||
          fromPrecision == TSDB_TIME_PRECISION_NANO);
-  assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO ||
+  assert(toPrecision == TSDB_TIME_PRECISION_MILLI ||
+         toPrecision == TSDB_TIME_PRECISION_MICRO ||
          toPrecision == TSDB_TIME_PRECISION_NANO);
-  static double factors[3][3] = {{1., 1000., 1000000.}, {1.0 / 1000, 1., 1000.}, {1.0 / 1000000, 1.0 / 1000, 1.}};
-  return (int64_t)((double)time * factors[fromPrecision][toPrecision]);
+  double tempResult = (double)time;
+  switch(fromPrecision) {
+    case TSDB_TIME_PRECISION_MILLI: {
+      switch (toPrecision) {
+        case TSDB_TIME_PRECISION_MILLI:
+          return time;
+        case TSDB_TIME_PRECISION_MICRO:
+          tempResult *= 1000;
+          time *= 1000;
+          goto end_;
+        case TSDB_TIME_PRECISION_NANO:
+          tempResult *= 1000000;
+          time *= 1000000;
+          goto end_;
+      }
+    } // end from milli
+    case TSDB_TIME_PRECISION_MICRO: {
+      switch (toPrecision) {
+        case TSDB_TIME_PRECISION_MILLI:
+          return time / 1000;
+        case TSDB_TIME_PRECISION_MICRO:
+          return time;
+        case TSDB_TIME_PRECISION_NANO:
+          tempResult *= 1000;
+          time *= 1000;
+          goto end_;
+      }
+    } //end from micro
+    case TSDB_TIME_PRECISION_NANO: {
+      switch (toPrecision) {
+        case TSDB_TIME_PRECISION_MILLI:
+          return time / 1000000;
+        case TSDB_TIME_PRECISION_MICRO:
+          return time / 1000;
+        case TSDB_TIME_PRECISION_NANO:
+          return time;
+      }
+    } //end from nano
+    default: {
+      assert(0);
+      return time;  // only to pass windows compilation
+    }
+  } //end switch fromPrecision
+end_:
+  if (tempResult >= (double)INT64_MAX) return INT64_MAX;
+  if (tempResult <= (double)INT64_MIN) return INT64_MIN;  // INT64_MIN means NULL
+  return time;
 }
 
+// !!!!notice:there are precision problems, double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = 1626006833631000064
+//int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) {
+//  assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
+//         fromPrecision == TSDB_TIME_PRECISION_NANO);
+//  assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO ||
+//         toPrecision == TSDB_TIME_PRECISION_NANO);
+//  static double factors[3][3] = {{1., 1000., 1000000.}, {1.0 / 1000, 1., 1000.}, {1.0 / 1000000, 1.0 / 1000, 1.}};
+//  ((double)time * factors[fromPrecision][toPrecision]);
+//}
+
 int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) {
   assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
          fromPrecision == TSDB_TIME_PRECISION_NANO);
-  static double factors[3] = {1000000., 1000., 1.};
+  int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
+  double tmp = time;
   switch (toUnit) {
-    case 's':
-      return time * factors[fromPrecision] / NANOSECOND_PER_SEC;
+    case 's':{
+      tmp *= (NANOSECOND_PER_SEC/factors[fromPrecision]);     // the result of division is an integer
+      time *= (NANOSECOND_PER_SEC/factors[fromPrecision]);
+      break;
+    }
     case 'm':
-      return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE;
+      tmp *= (NANOSECOND_PER_MINUTE/factors[fromPrecision]);  // the result of division is an integer
+      time *= (NANOSECOND_PER_MINUTE/factors[fromPrecision]);
+      break;
     case 'h':
-      return time * factors[fromPrecision] / NANOSECOND_PER_HOUR;
+      tmp *= (NANOSECOND_PER_HOUR/factors[fromPrecision]);    // the result of division is an integer
+      time *= (NANOSECOND_PER_HOUR/factors[fromPrecision]);
+      break;
     case 'd':
-      return time * factors[fromPrecision] / NANOSECOND_PER_DAY;
+      tmp *= (NANOSECOND_PER_DAY/factors[fromPrecision]);     // the result of division is an integer
+      time *= (NANOSECOND_PER_DAY/factors[fromPrecision]);
+      break;
     case 'w':
-      return time * factors[fromPrecision] / NANOSECOND_PER_WEEK;
+      tmp *= (NANOSECOND_PER_WEEK/factors[fromPrecision]);    // the result of division is an integer
+      time *= (NANOSECOND_PER_WEEK/factors[fromPrecision]);
+      break;
     case 'a':
-      return time * factors[fromPrecision] / NANOSECOND_PER_MSEC;
+      tmp *= (NANOSECOND_PER_MSEC/factors[fromPrecision]);    // the result of division is an integer
+      time *= (NANOSECOND_PER_MSEC/factors[fromPrecision]);
+      break;
     case 'u':
-      return time * factors[fromPrecision] / NANOSECOND_PER_USEC;
+      // the result of (NANOSECOND_PER_USEC/(double)factors[fromPrecision]) maybe a double
+      switch (fromPrecision) {
+        case TSDB_TIME_PRECISION_MILLI:{
+          tmp /= 1000;
+          time /= 1000;
+          break;
+        }
+        case TSDB_TIME_PRECISION_MICRO:{
+          tmp /= 1;
+          time /= 1;
+          break;
+        }
+        case TSDB_TIME_PRECISION_NANO:{
+          tmp *= 1000;
+          time *= 1000;
+          break;
+        }
+      }
+      break;
     case 'b':
-      return time * factors[fromPrecision];
+      tmp *= factors[fromPrecision];
+      time *= factors[fromPrecision];
+      break;
     default: {
       return -1;
     }
   }
+  if (tmp >= (double)INT64_MAX) return INT64_MAX;
+  if (tmp <= (double)INT64_MIN) return INT64_MIN;
+  return time;
 }
 
 int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) {

From 485a1aedd5a46d69e30bcd0bb736c06586a78edb Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 17 May 2022 22:27:46 +0800
Subject: [PATCH 17/25] fix(query): fix elapsed parameter number

---
 source/libs/function/src/builtinsimpl.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index b98fd0f243..4ab1a52897 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -2516,7 +2516,7 @@ bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo
   pInfo->min = MAX_TS_KEY;
   pInfo->max = 0;
 
-  if (pCtx->numOfParams == 3) {
+  if (pCtx->numOfParams == 2) {
     pInfo->timeUnit = pCtx->param[1].param.i;
   } else {
     pInfo->timeUnit = 1;
@@ -2540,7 +2540,6 @@ int32_t elapsedFunction(SqlFunctionCtx *pCtx) {
   }
 
   if (pInput->colDataAggIsSet) {
-
     if (pInfo->min == MAX_TS_KEY) {
       pInfo->min = GET_INT64_VAL(&pAgg->min);
       pInfo->max = GET_INT64_VAL(&pAgg->max);

From 715aa50f4d4aaecf9c210ab5ac6d11f6952a1328 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 18 May 2022 12:03:08 +0800
Subject: [PATCH 18/25] fix:precision problems in time conversion

---
 source/common/src/ttime.c                   | 26 +++++++++++----------
 tests/script/tsim/query/interval-offset.sim | 24 +------------------
 2 files changed, 15 insertions(+), 35 deletions(-)

diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c
index c121b2d83d..d6332fbbc5 100644
--- a/source/common/src/ttime.c
+++ b/source/common/src/ttime.c
@@ -439,6 +439,8 @@ end_:
 //  ((double)time * factors[fromPrecision][toPrecision]);
 //}
 
+
+// !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = 1626006833631000064
 int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) {
   assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
          fromPrecision == TSDB_TIME_PRECISION_NANO);
@@ -446,29 +448,29 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
   double tmp = time;
   switch (toUnit) {
     case 's':{
-      tmp *= (NANOSECOND_PER_SEC/factors[fromPrecision]);     // the result of division is an integer
-      time *= (NANOSECOND_PER_SEC/factors[fromPrecision]);
+      tmp /= (NANOSECOND_PER_SEC/factors[fromPrecision]);     // the result of division is an integer
+      time /= (NANOSECOND_PER_SEC/factors[fromPrecision]);
       break;
     }
     case 'm':
-      tmp *= (NANOSECOND_PER_MINUTE/factors[fromPrecision]);  // the result of division is an integer
-      time *= (NANOSECOND_PER_MINUTE/factors[fromPrecision]);
+      tmp /= (NANOSECOND_PER_MINUTE/factors[fromPrecision]);  // the result of division is an integer
+      time /= (NANOSECOND_PER_MINUTE/factors[fromPrecision]);
       break;
     case 'h':
-      tmp *= (NANOSECOND_PER_HOUR/factors[fromPrecision]);    // the result of division is an integer
-      time *= (NANOSECOND_PER_HOUR/factors[fromPrecision]);
+      tmp /= (NANOSECOND_PER_HOUR/factors[fromPrecision]);    // the result of division is an integer
+      time /= (NANOSECOND_PER_HOUR/factors[fromPrecision]);
       break;
     case 'd':
-      tmp *= (NANOSECOND_PER_DAY/factors[fromPrecision]);     // the result of division is an integer
-      time *= (NANOSECOND_PER_DAY/factors[fromPrecision]);
+      tmp /= (NANOSECOND_PER_DAY/factors[fromPrecision]);     // the result of division is an integer
+      time /= (NANOSECOND_PER_DAY/factors[fromPrecision]);
       break;
     case 'w':
-      tmp *= (NANOSECOND_PER_WEEK/factors[fromPrecision]);    // the result of division is an integer
-      time *= (NANOSECOND_PER_WEEK/factors[fromPrecision]);
+      tmp /= (NANOSECOND_PER_WEEK/factors[fromPrecision]);    // the result of division is an integer
+      time /= (NANOSECOND_PER_WEEK/factors[fromPrecision]);
       break;
     case 'a':
-      tmp *= (NANOSECOND_PER_MSEC/factors[fromPrecision]);    // the result of division is an integer
-      time *= (NANOSECOND_PER_MSEC/factors[fromPrecision]);
+      tmp /= (NANOSECOND_PER_MSEC/factors[fromPrecision]);    // the result of division is an integer
+      time /= (NANOSECOND_PER_MSEC/factors[fromPrecision]);
       break;
     case 'u':
       // the result of (NANOSECOND_PER_USEC/(double)factors[fromPrecision]) maybe a double
diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim
index 68860dc2cb..5ce3d9382f 100644
--- a/tests/script/tsim/query/interval-offset.sim
+++ b/tests/script/tsim/query/interval-offset.sim
@@ -177,29 +177,7 @@ if $data70 != 1 then
   return -1
 endi
 
-sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
-print ===> rows: $rows
-print ===> rows0: $data00 $data01 $data02 $data03 $data04
-print ===> rows1: $data10 $data11 $data12 $data13 $data14
-print ===> rows2: $data20 $data21 $data22 $data23 $data24
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
-if $rows != 4 then
-  return -1
-endi
-if $data00 != @21-12-08 00:00:00.000@ then
-  return -1
-endi
-if $data31 != 1 then
-  return -1
-endi
-if $data34 != $data31 then
-  return -1
-endi
-if $data02 != 2678400000 then
-  return -1
-endi
-
+sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w)
 sql_error select  _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(2w)
 sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*)  from ct3 interval(1n, 1w) sliding(4w)
 

From 5656c59261f1597ff30ab78e483db371b2252b8e Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 18 May 2022 12:06:13 +0800
Subject: [PATCH 19/25] fix:precision problems in time conversion

---
 source/common/src/ttime.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c
index d6332fbbc5..62a7179626 100644
--- a/source/common/src/ttime.c
+++ b/source/common/src/ttime.c
@@ -476,8 +476,8 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
       // the result of (NANOSECOND_PER_USEC/(double)factors[fromPrecision]) maybe a double
       switch (fromPrecision) {
         case TSDB_TIME_PRECISION_MILLI:{
-          tmp /= 1000;
-          time /= 1000;
+          tmp *= 1000;
+          time *= 1000;
           break;
         }
         case TSDB_TIME_PRECISION_MICRO:{
@@ -486,8 +486,8 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
           break;
         }
         case TSDB_TIME_PRECISION_NANO:{
-          tmp *= 1000;
-          time *= 1000;
+          tmp /= 1000;
+          time /= 1000;
           break;
         }
       }

From 0b8a198074868b5db6bd2018d8c79c2cbe563c29 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 18 May 2022 12:50:53 +0800
Subject: [PATCH 20/25] fix:precision problems in time conversion

---
 tests/script/tsim/query/interval-offset.sim | 24 ++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim
index 5ce3d9382f..68860dc2cb 100644
--- a/tests/script/tsim/query/interval-offset.sim
+++ b/tests/script/tsim/query/interval-offset.sim
@@ -177,7 +177,29 @@ if $data70 != 1 then
   return -1
 endi
 
-sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w)
+sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w)
+print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
+print ===> rows: $rows
+print ===> rows0: $data00 $data01 $data02 $data03 $data04
+print ===> rows1: $data10 $data11 $data12 $data13 $data14
+print ===> rows2: $data20 $data21 $data22 $data23 $data24
+print ===> rows3: $data30 $data31 $data32 $data33 $data34
+if $rows != 4 then
+  return -1
+endi
+if $data00 != @21-12-08 00:00:00.000@ then
+  return -1
+endi
+if $data31 != 1 then
+  return -1
+endi
+if $data34 != $data31 then
+  return -1
+endi
+if $data02 != 2678400000 then
+  return -1
+endi
+
 sql_error select  _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(2w)
 sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*)  from ct3 interval(1n, 1w) sliding(4w)
 

From 3cbf4ad4d2c356d22a0ebd8698ab2ebd1d11d1b2 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 18 May 2022 14:11:23 +0800
Subject: [PATCH 21/25] fix: avoid invalid read/write

---
 source/dnode/mnode/sdb/src/sdbRaw.c  |  7 +++---
 source/libs/index/src/indexTfile.c   | 13 +++++-----
 source/libs/index/test/jsonUT.cc     | 37 ++++++++++++++++++++++++++++
 source/libs/transport/src/transCli.c | 11 ++++-----
 4 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c
index d09198f66f..fd2f20c242 100644
--- a/source/dnode/mnode/sdb/src/sdbRaw.c
+++ b/source/dnode/mnode/sdb/src/sdbRaw.c
@@ -213,8 +213,9 @@ int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valL
     terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
     return -1;
   }
-
-  memcpy(pVal, pRaw->pData + dataPos, valLen);
+  if (pVal != NULL) {
+    memcpy(pVal, pRaw->pData + dataPos, valLen);
+  }
   return 0;
 }
 
@@ -235,4 +236,4 @@ int32_t sdbGetRawTotalSize(SSdbRaw *pRaw) {
   }
 
   return sizeof(SSdbRaw) + pRaw->dataLen;
-}
\ No newline at end of file
+}
diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c
index 9533d3429e..163bb53163 100644
--- a/source/libs/index/src/indexTfile.c
+++ b/source/libs/index/src/indexTfile.c
@@ -473,17 +473,16 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
 
     int32_t sz = 0;
     char*   ch = (char*)fstSliceData(s, &sz);
-    // char*   tmp = taosMemoryCalloc(1, sz + 1);
-    // memcpy(tmp, ch, sz);
+    char*   tmp = taosMemoryCalloc(1, sz + 1);
+    memcpy(tmp, ch, sz);
 
-    if (0 != strncmp(ch, p, skip)) {
+    if (0 != strncmp(tmp, p, skip)) {
       swsResultDestroy(rt);
-      // taosMemoryFree(tmp);
+      taosMemoryFree(tmp);
       break;
     }
 
-    TExeCond cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
-
+    TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
     if (MATCH == cond) {
       tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
     } else if (CONTINUE == cond) {
@@ -491,7 +490,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
       swsResultDestroy(rt);
       break;
     }
-    // taosMemoryFree(tmp);
+    taosMemoryFree(tmp);
     swsResultDestroy(rt);
   }
   streamWithStateDestroy(st);
diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc
index 135ae61e83..f0c6455442 100644
--- a/source/libs/index/test/jsonUT.cc
+++ b/source/libs/index/test/jsonUT.cc
@@ -578,3 +578,40 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
     EXPECT_EQ(1000, taosArrayGetSize(res));
   }
 }
+TEST_F(JsonEnv, testWriteJsonTfileAndCache_DOUBLE) {
+  {
+    double      val = 10.0;
+    std::string colName("test1");
+    for (int i = 0; i < 1000; i++) {
+      WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i);
+    }
+  }
+  {
+    double      val = 2.0;
+    std::string colName("test1");
+    for (int i = 0; i < 1000; i++) {
+      WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000);
+    }
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test1");
+    double      val = 1.9;
+    Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(2000, taosArrayGetSize(res));
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test1");
+    double      val = 2.1;
+    Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(1000, taosArrayGetSize(res));
+  }
+  {
+    std::string colName("test1");
+    SArray*     res = NULL;
+    double      val = 2.1;
+    Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(1000, taosArrayGetSize(res));
+  }
+}
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index d2d38bd9bc..92c5e9faf7 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -145,9 +145,9 @@ static void cliWalkCb(uv_handle_t* handle, void* arg);
   } while (0)
 
 #define CONN_HOST_THREAD_INDEX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1)
-#define CONN_PERSIST_TIME(para) (para * 1000 * 10)
-#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
-#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label)
+#define CONN_PERSIST_TIME(para)      (para * 1000 * 10)
+#define CONN_GET_HOST_THREAD(conn)   (conn ? ((SCliConn*)conn)->hostThrd : NULL)
+#define CONN_GET_INST_LABEL(conn)    (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label)
 #define CONN_SHOULD_RELEASE(conn, head)                                                  \
   do {                                                                                   \
     if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) {                       \
@@ -227,7 +227,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg);
 #define REQUEST_PERSIS_HANDLE(msg)   ((msg)->info.persistHandle == 1)
 #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release)
 
-#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn)
+#define EPSET_GET_INUSE_IP(epSet)   ((epSet)->eps[(epSet)->inUse].fqdn)
 #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port)
 
 static void* cliWorkThread(void* arg);
@@ -924,8 +924,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
         arg->param1 = pMsg;
         arg->param2 = pThrd;
         transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
-
-        cliDestroy((uv_handle_t*)pConn->stream);
+        cliDestroyConn(pConn, true);
         return -1;
       }
     } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) {

From 77575d1e70d999b8b96a2476d61e752f017a6f3e Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 18 May 2022 14:20:33 +0800
Subject: [PATCH 22/25] refactor: shm queue in multi process mode

---
 include/common/tmsgcb.h                       |   8 +-
 source/common/src/tmsgcb.c                    |  12 +-
 source/dnode/mgmt/mgmt_dnode/src/dmWorker.c   |   2 +
 source/dnode/mgmt/mgmt_vnode/src/vmInt.c      |   1 +
 source/dnode/mgmt/node_mgmt/inc/dmMgmt.h      |   8 +-
 source/dnode/mgmt/node_mgmt/src/dmProc.c      | 237 +++++++-----------
 source/dnode/mgmt/node_mgmt/src/dmTransport.c |  26 +-
 source/dnode/mnode/impl/src/mnode.c           |   2 +-
 source/dnode/mnode/impl/test/trans/trans2.cpp |   3 +-
 source/libs/sync/src/syncIO.c                 |   1 +
 source/libs/wal/src/walMgmt.c                 |   1 +
 source/util/src/tlog.c                        |   1 +
 source/util/src/tsched.c                      |   1 +
 source/util/src/tworker.c                     |   2 +
 tests/test/c/create_table.c                   |   1 +
 tests/test/c/tmqSim.c                         |   1 +
 tests/tsim/src/simSystem.c                    |   1 +
 tools/shell/src/shellEngine.c                 |   1 +
 18 files changed, 135 insertions(+), 174 deletions(-)

diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h
index 679e3ba775..7ba6e5044c 100644
--- a/include/common/tmsgcb.h
+++ b/include/common/tmsgcb.h
@@ -41,8 +41,8 @@ typedef enum {
 typedef int32_t (*PutToQueueFp)(void* pMgmt, SRpcMsg* pMsg);
 typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
 typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
-typedef void (*SendRspFp)(const SRpcMsg* pMsg);
-typedef void (*SendRedirectRspFp)(const SRpcMsg* pMsg, const SEpSet* pNewEpSet);
+typedef void (*SendRspFp)(SRpcMsg* pMsg);
+typedef void (*SendRedirectRspFp)(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
 typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg);
 typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
 typedef void (*ReportStartup)(const char* name, const char* desc);
@@ -64,8 +64,8 @@ void    tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb);
 int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg);
 int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
 int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg);
-void    tmsgSendRsp(const SRpcMsg* pMsg);
-void    tmsgSendRedirectRsp(const SRpcMsg* pMsg, const SEpSet* pNewEpSet);
+void    tmsgSendRsp(SRpcMsg* pMsg);
+void    tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
 void    tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg);
 void    tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type);
 void    tmsgReportStartup(const char* name, const char* desc);
diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c
index d28e2c675d..f69fb65f04 100644
--- a/source/common/src/tmsgcb.c
+++ b/source/common/src/tmsgcb.c
@@ -21,9 +21,9 @@ static SMsgCb tsDefaultMsgCb;
 
 void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; }
 
-int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) {
+int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg) {
   PutToQueueFp fp = pMsgCb->queueFps[qtype];
-  return (*fp)(pMsgCb->mgmt, pReq);
+  return (*fp)(pMsgCb->mgmt, pMsg);
 }
 
 int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) {
@@ -31,17 +31,17 @@ int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) {
   return (*fp)(pMsgCb->mgmt, vgId, qtype);
 }
 
-int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pReq) {
+int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) {
   SendReqFp fp = tsDefaultMsgCb.sendReqFp;
-  return (*fp)(epSet, pReq);
+  return (*fp)(epSet, pMsg);
 }
 
-void tmsgSendRsp(const SRpcMsg* pMsg) {
+void tmsgSendRsp(SRpcMsg* pMsg) {
   SendRspFp fp = tsDefaultMsgCb.sendRspFp;
   return (*fp)(pMsg);
 }
 
-void tmsgSendRedirectRsp(const SRpcMsg* pMsg, const SEpSet* pNewEpSet) {
+void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) {
   SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp;
   (*fp)(pMsg, pNewEpSet);
 }
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
index 7daf25bb8a..787c6c98c8 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
@@ -75,6 +75,7 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) {
 void dmStopStatusThread(SDnodeMgmt *pMgmt) {
   if (taosCheckPthreadValid(pMgmt->statusThread)) {
     taosThreadJoin(pMgmt->statusThread, NULL);
+    taosThreadClear(&pMgmt->statusThread);
   }
 }
 
@@ -95,6 +96,7 @@ int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
 void dmStopMonitorThread(SDnodeMgmt *pMgmt) {
   if (taosCheckPthreadValid(pMgmt->monitorThread)) {
     taosThreadJoin(pMgmt->monitorThread, NULL);
+    taosThreadClear(&pMgmt->monitorThread);
   }
 }
 
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
index ab41ee5df3..8c3b8576a8 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
@@ -196,6 +196,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
     SVnodeThread *pThread = &threads[t];
     if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) {
       taosThreadJoin(pThread->thread, NULL);
+      taosThreadClear(&pThread->thread);
     }
     taosMemoryFree(pThread->pCfgs);
   }
diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
index 2d03267fe8..7484c1e18f 100644
--- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
+++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
@@ -151,12 +151,10 @@ int32_t dmInitProc(struct SMgmtWrapper *pWrapper);
 void    dmCleanupProc(struct SMgmtWrapper *pWrapper);
 int32_t dmRunProc(SProc *proc);
 void    dmStopProc(SProc *proc);
-int64_t dmRemoveProcRpcHandle(SProc *proc, void *handle);
+void    dmRemoveProcRpcHandle(SProc *proc, void *handle);
 void    dmCloseProcRpcHandles(SProc *proc);
-int32_t dmPutToProcCQueue(SProc *proc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                          void *handle, int64_t handleRef, EProcFuncType ftype);
-void    dmPutToProcPQueue(SProc *proc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                          EProcFuncType ftype);
+int32_t dmPutToProcCQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype);
+void    dmPutToProcPQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype);
 
 // dmTransport.c
 int32_t dmInitServer(SDnode *pDnode);
diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c
index f9875f7182..2e24e3fa1c 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmProc.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c
@@ -16,10 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "dmMgmt.h"
 
-static inline int32_t CEIL8(int32_t v) {
-  const int32_t c = ceil((float)(v) / 8) * 8;
-  return c < 8 ? 8 : c;
-}
+static inline int32_t CEIL8(int32_t v) { return ceil((float)(v) / 8) * 8; }
 
 static int32_t dmInitProcMutex(SProcQueue *queue) {
   TdThreadMutexAttr mattr = {0};
@@ -87,42 +84,17 @@ static SProcQueue *dmInitProcQueue(SProc *proc, char *ptr, int32_t size) {
   return queue;
 }
 
-#if 0
-static void dmDestroyProcQueue(SProcQueue *queue) {
-  if (queue->mutex != NULL) {
-    taosThreadMutexDestroy(queue->mutex);
-    queue->mutex = NULL;
-  }
-}
+static void dmCleanupProcQueue(SProcQueue *queue) {}
 
-static void dmDestroyProcSem(SProcQueue *queue) {
-  if (queue->sem != NULL) {
-    tsem_destroy(queue->sem);
-    queue->sem = NULL;
-  }
-}
-#endif
-
-static void dmCleanupProcQueue(SProcQueue *queue) {
-#if 0  
-  if (queue != NULL) {
-    dmDestroyProcQueue(queue);
-    dmDestroyProcSem(queue);
-  }
-#endif
-}
-
-static int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, const char *pHead, int16_t rawHeadLen,
-                                 const char *pBody, int32_t rawBodyLen, int64_t handle, int64_t handleRef,
-                                 EProcFuncType ftype) {
-  if (rawHeadLen == 0 || pHead == NULL) {
-    terrno = TSDB_CODE_INVALID_PARA;
-    return -1;
-  }
-
-  const int32_t headLen = CEIL8(rawHeadLen);
+static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg *pMsg, EProcFuncType ftype) {
+  const void   *pHead = pMsg;
+  const void   *pBody = pMsg->pCont;
+  const int16_t rawHeadLen = sizeof(SRpcMsg);
+  const int32_t rawBodyLen = pMsg->contLen;
+  const int16_t headLen = CEIL8(rawHeadLen);
   const int32_t bodyLen = CEIL8(rawBodyLen);
   const int32_t fullLen = headLen + bodyLen + 8;
+  const int64_t handle = (int64_t)pMsg->info.handle;
 
   taosThreadMutexLock(&queue->mutex);
   if (fullLen > queue->avail) {
@@ -131,8 +103,8 @@ static int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, const char *pHe
     return -1;
   }
 
-  if (handle != 0 && ftype == DND_FUNC_REQ) {
-    if (taosHashPut(proc->hash, &handle, sizeof(int64_t), &handleRef, sizeof(int64_t)) != 0) {
+  if (ftype == DND_FUNC_REQ && IsReq(pMsg) && pMsg->code == 0 && handle != 0) {
+    if (taosHashPut(proc->hash, &handle, sizeof(int64_t), &pMsg->info, sizeof(SRpcConnInfo)) != 0) {
       taosThreadMutexUnlock(&queue->mutex);
       return -1;
     }
@@ -151,31 +123,31 @@ static int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, const char *pHe
 
   if (queue->tail < queue->head) {
     memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen);
-    memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, rawBodyLen);
+    if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, rawBodyLen);
     queue->tail = queue->tail + 8 + headLen + bodyLen;
   } else {
     int32_t remain = queue->total - queue->tail;
     if (remain == 0) {
       memcpy(queue->pBuffer + 8, pHead, rawHeadLen);
-      memcpy(queue->pBuffer + 8 + headLen, pBody, rawBodyLen);
+      if (rawBodyLen > 0) memcpy(queue->pBuffer + 8 + headLen, pBody, rawBodyLen);
       queue->tail = 8 + headLen + bodyLen;
     } else if (remain == 8) {
       memcpy(queue->pBuffer, pHead, rawHeadLen);
-      memcpy(queue->pBuffer + headLen, pBody, rawBodyLen);
+      if (rawBodyLen > 0) memcpy(queue->pBuffer + headLen, pBody, rawBodyLen);
       queue->tail = headLen + bodyLen;
     } else if (remain < 8 + headLen) {
       memcpy(queue->pBuffer + queue->tail + 8, pHead, remain - 8);
       memcpy(queue->pBuffer, pHead + remain - 8, rawHeadLen - (remain - 8));
-      memcpy(queue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen);
+      if (rawBodyLen > 0) memcpy(queue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen);
       queue->tail = headLen - (remain - 8) + bodyLen;
     } else if (remain < 8 + headLen + bodyLen) {
       memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen);
-      memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, remain - 8 - headLen);
-      memcpy(queue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen));
+      if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, remain - 8 - headLen);
+      if (rawBodyLen > 0) memcpy(queue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen));
       queue->tail = bodyLen - (remain - 8 - headLen);
     } else {
       memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen);
-      memcpy(queue->pBuffer + queue->tail + headLen + 8, pBody, rawBodyLen);
+      if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + headLen + 8, pBody, rawBodyLen);
       queue->tail = queue->tail + headLen + bodyLen + 8;
     }
   }
@@ -185,13 +157,12 @@ static int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, const char *pHe
   taosThreadMutexUnlock(&queue->mutex);
   tsem_post(&queue->sem);
 
-  dTrace("node:%s, push %s msg:%p:%d cont:%p:%d, pos:%d remain:%d", queue->name, dmFuncStr(ftype), pHead, headLen,
-         pBody, bodyLen, pos, queue->items);
+  dTrace("node:%s, push %s msg:%p type:%d handle:%p len:%d code:0x%x, pos:%d remain:%d", queue->name, dmFuncStr(ftype),
+         pMsg, pMsg->msgType, pMsg->info.handle, pMsg->contLen, pMsg->code, pos, queue->items);
   return 0;
 }
 
-static int32_t dmPopFromProcQueue(SProcQueue *queue, void **ppHead, int16_t *pHeadLen, void **ppBody, int32_t *pBodyLen,
-                                  EProcFuncType *pFuncType) {
+static int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) {
   tsem_wait(&queue->sem);
 
   taosThreadMutexLock(&queue->mutex);
@@ -217,8 +188,9 @@ static int32_t dmPopFromProcQueue(SProcQueue *queue, void **ppHead, int16_t *pHe
   int32_t bodyLen = CEIL8(rawBodyLen);
 
   void *pHead = taosAllocateQitem(headLen, DEF_QITEM);
-  void *pBody = rpcMallocCont(bodyLen);
-  if (pHead == NULL || pBody == NULL) {
+  void *pBody = NULL;
+  if (bodyLen > 0) pBody = rpcMallocCont(bodyLen);
+  if (pHead == NULL || (bodyLen > 0 && pBody == NULL)) {
     taosThreadMutexUnlock(&queue->mutex);
     tsem_post(&queue->sem);
     taosFreeQitem(pHead);
@@ -230,31 +202,31 @@ static int32_t dmPopFromProcQueue(SProcQueue *queue, void **ppHead, int16_t *pHe
   const int32_t pos = queue->head;
   if (queue->head < queue->tail) {
     memcpy(pHead, queue->pBuffer + queue->head + 8, headLen);
-    memcpy(pBody, queue->pBuffer + queue->head + 8 + headLen, bodyLen);
+    if (bodyLen > 0) memcpy(pBody, queue->pBuffer + queue->head + 8 + headLen, bodyLen);
     queue->head = queue->head + 8 + headLen + bodyLen;
   } else {
     int32_t remain = queue->total - queue->head;
     if (remain == 0) {
       memcpy(pHead, queue->pBuffer + 8, headLen);
-      memcpy(pBody, queue->pBuffer + 8 + headLen, bodyLen);
+      if (bodyLen > 0) memcpy(pBody, queue->pBuffer + 8 + headLen, bodyLen);
       queue->head = 8 + headLen + bodyLen;
     } else if (remain == 8) {
       memcpy(pHead, queue->pBuffer, headLen);
-      memcpy(pBody, queue->pBuffer + headLen, bodyLen);
+      if (bodyLen > 0) memcpy(pBody, queue->pBuffer + headLen, bodyLen);
       queue->head = headLen + bodyLen;
     } else if (remain < 8 + headLen) {
       memcpy(pHead, queue->pBuffer + queue->head + 8, remain - 8);
       memcpy((char *)pHead + remain - 8, queue->pBuffer, headLen - (remain - 8));
-      memcpy(pBody, queue->pBuffer + headLen - (remain - 8), bodyLen);
+      if (bodyLen > 0) memcpy(pBody, queue->pBuffer + headLen - (remain - 8), bodyLen);
       queue->head = headLen - (remain - 8) + bodyLen;
     } else if (remain < 8 + headLen + bodyLen) {
       memcpy(pHead, queue->pBuffer + queue->head + 8, headLen);
-      memcpy(pBody, queue->pBuffer + queue->head + 8 + headLen, remain - 8 - headLen);
-      memcpy((char *)pBody + remain - 8 - headLen, queue->pBuffer, bodyLen - (remain - 8 - headLen));
+      if (bodyLen > 0) memcpy(pBody, queue->pBuffer + queue->head + 8 + headLen, remain - 8 - headLen);
+      if (bodyLen > 0) memcpy((char *)pBody + remain - 8 - headLen, queue->pBuffer, bodyLen - (remain - 8 - headLen));
       queue->head = bodyLen - (remain - 8 - headLen);
     } else {
       memcpy(pHead, queue->pBuffer + queue->head + 8, headLen);
-      memcpy(pBody, queue->pBuffer + queue->head + headLen + 8, bodyLen);
+      if (bodyLen > 0) memcpy(pBody, queue->pBuffer + queue->head + headLen + 8, bodyLen);
       queue->head = queue->head + headLen + bodyLen + 8;
     }
   }
@@ -263,14 +235,12 @@ static int32_t dmPopFromProcQueue(SProcQueue *queue, void **ppHead, int16_t *pHe
   queue->items--;
   taosThreadMutexUnlock(&queue->mutex);
 
-  *ppHead = pHead;
-  *ppBody = pBody;
-  *pHeadLen = rawHeadLen;
-  *pBodyLen = rawBodyLen;
+  *ppMsg = pHead;
+  (*ppMsg)->pCont = pBody;
   *pFuncType = (EProcFuncType)ftype;
 
-  dTrace("node:%s, pop %s msg:%p:%d cont:%p:%d, pos:%d remain:%d", queue->name, dmFuncStr(ftype), pHead, headLen, pBody,
-         bodyLen, pos, queue->items);
+  dTrace("node:%s, pop %s msg:%p type:%d handle:%p len:%d code:0x%x, pos:%d remain:%d", queue->name, dmFuncStr(ftype),
+         (*ppMsg), (*ppMsg)->msgType, (*ppMsg)->info.handle, (*ppMsg)->contLen, (*ppMsg)->code, pos, queue->items);
   return 1;
 }
 
@@ -308,18 +278,14 @@ static void *dmConsumChildQueue(void *param) {
   SProc        *proc = param;
   SMgmtWrapper *pWrapper = proc->wrapper;
   SProcQueue   *queue = proc->cqueue;
-  void         *pHead = NULL;
-  void         *pBody = NULL;
-  int16_t       headLen = 0;
-  int32_t       bodyLen = 0;
   int32_t       numOfMsgs = 0;
   int32_t       code = 0;
   EProcFuncType ftype = DND_FUNC_REQ;
-  SRpcMsg      *pReq = NULL;
+  SRpcMsg      *pMsg = NULL;
 
   dDebug("node:%s, start to consume from cqueue", proc->name);
   do {
-    numOfMsgs = dmPopFromProcQueue(queue, &pHead, &headLen, &pBody, &bodyLen, &ftype);
+    numOfMsgs = dmPopFromProcQueue(queue, &pMsg, &ftype);
     if (numOfMsgs == 0) {
       dDebug("node:%s, get no msg from cqueue and exit thread", proc->name);
       break;
@@ -332,25 +298,24 @@ static void *dmConsumChildQueue(void *param) {
     }
 
     if (ftype != DND_FUNC_REQ) {
-      dFatal("node:%s, get msg:%p from cqueue, invalid ftype:%d", proc->name, pHead, ftype);
-      taosFreeQitem(pHead);
-      rpcFreeCont(pBody);
-    } else {
-      pReq = pHead;
-      pReq->pCont = pBody;
-      code = dmProcessNodeMsg(pWrapper, pReq);
-      if (code != 0) {
-        dError("node:%s, failed to process msg:%p since %s, put into pqueue", proc->name, pReq, terrstr());
-        SRpcMsg rspMsg = {
-            .info = pReq->info,
-            .pCont = pReq->info.rsp,
-            .contLen = pReq->info.rspLen,
-        };
-        dmPutToProcPQueue(proc, &rspMsg, sizeof(SRpcMsg), rspMsg.pCont, rspMsg.contLen, DND_FUNC_RSP);
-        taosFreeQitem(pHead);
-        rpcFreeCont(pBody);
-        rpcFreeCont(rspMsg.pCont);
-      }
+      dError("node:%s, invalid ftype:%d from cqueue", proc->name, ftype);
+      rpcFreeCont(pMsg->pCont);
+      taosFreeQitem(pMsg);
+      continue;
+    }
+
+    code = dmProcessNodeMsg(pWrapper, pMsg);
+    if (code != 0) {
+      dError("node:%s, failed to process msg:%p since %s, put into pqueue", proc->name, pMsg, terrstr());
+      SRpcMsg rsp = {
+          .code = (terrno != 0 ? terrno : code),
+          .pCont = pMsg->info.rsp,
+          .contLen = pMsg->info.rspLen,
+          .info = pMsg->info,
+      };
+      dmPutToProcPQueue(proc, &rsp, DND_FUNC_RSP);
+      rpcFreeCont(pMsg->pCont);
+      taosFreeQitem(pMsg);
     }
   } while (1);
 
@@ -361,18 +326,14 @@ static void *dmConsumParentQueue(void *param) {
   SProc        *proc = param;
   SMgmtWrapper *pWrapper = proc->wrapper;
   SProcQueue   *queue = proc->pqueue;
-  void         *pHead = NULL;
-  void         *pBody = NULL;
-  int16_t       headLen = 0;
-  int32_t       bodyLen = 0;
   int32_t       numOfMsgs = 0;
   int32_t       code = 0;
   EProcFuncType ftype = DND_FUNC_REQ;
-  SRpcMsg      *pRsp = NULL;
+  SRpcMsg      *pMsg = NULL;
 
   dDebug("node:%s, start to consume from pqueue", proc->name);
   do {
-    numOfMsgs = dmPopFromProcQueue(queue, &pHead, &headLen, &pBody, &bodyLen, &ftype);
+    numOfMsgs = dmPopFromProcQueue(queue, &pMsg, &ftype);
     if (numOfMsgs == 0) {
       dDebug("node:%s, get no msg from pqueue and exit thread", proc->name);
       break;
@@ -385,31 +346,19 @@ static void *dmConsumParentQueue(void *param) {
     }
 
     if (ftype == DND_FUNC_RSP) {
-      pRsp = pHead;
-      pRsp->pCont = pBody;
-      dTrace("node:%s, get rsp msg:%p from pqueue, code:0x%04x handle:%p", proc->name, pRsp, code, pRsp->info.handle);
-      dmRemoveProcRpcHandle(proc, pRsp->info.handle);
-      rpcSendResponse(pRsp);
+      dmRemoveProcRpcHandle(proc, pMsg->info.handle);
+      rpcSendResponse(pMsg);
     } else if (ftype == DND_FUNC_REGIST) {
-      pRsp = pHead;
-      pRsp->pCont = pBody;
-      dTrace("node:%s, get regist msg:%p from pqueue, code:0x%04x handle:%p", proc->name, pRsp, code,
-             pRsp->info.handle);
-      rpcRegisterBrokenLinkArg(pRsp);
+      rpcRegisterBrokenLinkArg(pMsg);
     } else if (ftype == DND_FUNC_RELEASE) {
-      pRsp = pHead;
-      pRsp->pCont = NULL;
-      dTrace("node:%s, get release msg:%p from pqueue, code:0x%04x handle:%p", proc->name, pRsp, code,
-             pRsp->info.handle);
-      dmRemoveProcRpcHandle(proc, pRsp->info.handle);
-      rpcReleaseHandle(pRsp->info.handle, (int8_t)pRsp->code);
-      rpcFreeCont(pBody);
+      dmRemoveProcRpcHandle(proc, pMsg->info.handle);
+      rpcReleaseHandle(pMsg->info.handle, (int8_t)pMsg->code);
     } else {
-      dFatal("node:%s, get msg:%p from pqueue, invalid ftype:%d", proc->name, pHead, ftype);
-      rpcFreeCont(pBody);
+      dError("node:%s, invalid ftype:%d from pqueue", proc->name, ftype);
+      rpcFreeCont(pMsg->pCont);
     }
 
-    taosFreeQitem(pHead);
+    taosFreeQitem(pMsg);
   } while (1);
 
   return NULL;
@@ -468,51 +417,55 @@ void dmCleanupProc(struct SMgmtWrapper *pWrapper) {
   dmCleanupProcQueue(proc->cqueue);
   dmCleanupProcQueue(proc->pqueue);
   taosHashCleanup(proc->hash);
+  proc->hash = NULL;
   dDebug("node:%s, proc is cleaned up", pWrapper->name);
 }
 
-int64_t dmRemoveProcRpcHandle(SProc *proc, void *handle) {
+void dmRemoveProcRpcHandle(SProc *proc, void *handle) {
   int64_t h = (int64_t)handle;
   taosThreadMutexLock(&proc->cqueue->mutex);
-
-  int64_t *pRef = taosHashGet(proc->hash, &h, sizeof(int64_t));
-  int64_t  ref = 0;
-  if (pRef != NULL) {
-    ref = *pRef;
-  }
-
   taosHashRemove(proc->hash, &h, sizeof(int64_t));
   taosThreadMutexUnlock(&proc->cqueue->mutex);
-
-  return ref;
 }
 
 void dmCloseProcRpcHandles(SProc *proc) {
   taosThreadMutexLock(&proc->cqueue->mutex);
-  void *h = taosHashIterate(proc->hash, NULL);
-  while (h != NULL) {
-    void *handle = *((void **)h);
-    h = taosHashIterate(proc->hash, h);
-
-    dError("node:%s, the child process dies and send an offline rsp to handle:%p", proc->name, handle);
-    SRpcMsg rpcMsg = {.info.handle = handle, .code = TSDB_CODE_NODE_OFFLINE};
+  SRpcHandleInfo *pInfo = taosHashIterate(proc->hash, NULL);
+  while (pInfo != NULL) {
+    dError("node:%s, the child process dies and send an offline rsp to handle:%p", proc->name, pInfo->handle);
+    SRpcMsg rpcMsg = {.info = *pInfo, .code = TSDB_CODE_NODE_OFFLINE};
     rpcSendResponse(&rpcMsg);
+    pInfo = taosHashIterate(proc->hash, pInfo);
   }
   taosHashClear(proc->hash);
   taosThreadMutexUnlock(&proc->cqueue->mutex);
 }
 
-void dmPutToProcPQueue(SProc *proc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                       EProcFuncType ftype) {
+void dmPutToProcPQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype) {
   int32_t retry = 0;
-  while (dmPushToProcQueue(proc, proc->pqueue, pHead, headLen, pBody, bodyLen, 0, 0, ftype) != 0) {
-    dWarn("node:%s, failed to put msg:%p to pqueue since %s, retry:%d", proc->name, pHead, terrstr(), retry);
-    retry++;
-    taosMsleep(retry);
+  while (1) {
+    if (dmPushToProcQueue(proc, proc->pqueue, pMsg, ftype) == 0) {
+      break;
+    }
+
+    if (retry == 10) {
+      pMsg->code = terrno;
+      if (pMsg->contLen > 0) {
+        rpcFreeCont(pMsg->pCont);
+        pMsg->pCont = NULL;
+        pMsg->contLen = 0;
+      }
+      dError("node:%s, failed to push %s msg:%p type:%d handle:%p then discard data and return error", proc->name,
+             dmFuncStr(ftype), pMsg, pMsg->msgType, pMsg->info.handle);
+    } else {
+      dError("node:%s, failed to push %s msg:%p type:%d handle:%p len:%d since %s, retry:%d", proc->name,
+             dmFuncStr(ftype), pMsg, pMsg->msgType, pMsg->info.handle, pMsg->contLen, terrstr(), retry);
+      retry++;
+      taosMsleep(retry);
+    }
   }
 }
 
-int32_t dmPutToProcCQueue(SProc *proc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                          void *handle, int64_t ref, EProcFuncType ftype) {
-  return dmPushToProcQueue(proc, proc->cqueue, pHead, headLen, pBody, bodyLen, (int64_t)handle, ref, ftype);
+int32_t dmPutToProcCQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype) {
+  return dmPushToProcQueue(proc, proc->cqueue, pMsg, ftype);
 }
diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 2efa341704..c9100aab9d 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -128,9 +128,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
   }
 
   if (InParentProc(pWrapper)) {
-    code = dmPutToProcCQueue(&pWrapper->proc, pMsg, sizeof(SRpcMsg), pRpc->pCont, pRpc->contLen,
-                             (IsReq(pRpc) && (pRpc->code == 0)) ? pRpc->info.handle : NULL, pRpc->info.refId,
-                             DND_FUNC_REQ);
+    code = dmPutToProcCQueue(&pWrapper->proc, pMsg, DND_FUNC_REQ);
   } else {
     code = dmProcessNodeMsg(pWrapper, pMsg);
   }
@@ -255,23 +253,23 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pReq) {
   }
 }
 
-static inline void dmSendRsp(const SRpcMsg *pMsg) {
+static inline void dmSendRsp(SRpcMsg *pMsg) {
   SMgmtWrapper *pWrapper = pMsg->info.wrapper;
-  if (InChildProc(pWrapper)) {
-    dmPutToProcPQueue(&pWrapper->proc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, DND_FUNC_RSP);
+  if (pMsg->code == TSDB_CODE_NODE_REDIRECT) {
+    dmSendRpcRedirectRsp(pMsg);
   } else {
-    if (pMsg->code == TSDB_CODE_NODE_REDIRECT) {
-      dmSendRpcRedirectRsp(pMsg);
+    if (InChildProc(pWrapper)) {
+      dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP);
     } else {
       rpcSendResponse(pMsg);
     }
   }
 }
 
-static inline void dmSendRedirectRsp(const SRpcMsg *pRsp, const SEpSet *pNewEpSet) {
-  SMgmtWrapper *pWrapper = pRsp->info.wrapper;
+static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) {
+  SMgmtWrapper *pWrapper = pMsg->info.wrapper;
   if (InChildProc(pWrapper)) {
-    dmPutToProcPQueue(&pWrapper->proc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, DND_FUNC_RSP);
+    dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP);
   } else {
     SRpcMsg rsp = {0};
     SMEpSet msg = {.epSet = *pNewEpSet};
@@ -281,7 +279,7 @@ static inline void dmSendRedirectRsp(const SRpcMsg *pRsp, const SEpSet *pNewEpSe
     tSerializeSMEpSet(rsp.pCont, len, &msg);
 
     rsp.code = TSDB_CODE_RPC_REDIRECT;
-    rsp.info = pRsp->info;
+    rsp.info = pMsg->info;
     rpcSendResponse(&rsp);
   }
 }
@@ -289,7 +287,7 @@ static inline void dmSendRedirectRsp(const SRpcMsg *pRsp, const SEpSet *pNewEpSe
 static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) {
   SMgmtWrapper *pWrapper = pMsg->info.wrapper;
   if (InChildProc(pWrapper)) {
-    dmPutToProcPQueue(&pWrapper->proc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, DND_FUNC_REGIST);
+    dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_REGIST);
   } else {
     rpcRegisterBrokenLinkArg(pMsg);
   }
@@ -299,7 +297,7 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) {
   SMgmtWrapper *pWrapper = pHandle->wrapper;
   if (InChildProc(pWrapper)) {
     SRpcMsg msg = {.code = type, .info = *pHandle};
-    dmPutToProcPQueue(&pWrapper->proc, &msg, sizeof(SRpcMsg), NULL, 0, DND_FUNC_RELEASE);
+    dmPutToProcPQueue(&pWrapper->proc, &msg, DND_FUNC_RELEASE);
   } else {
     rpcReleaseHandle(pHandle->handle, type);
   }
diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c
index 2756d2203e..3dfba4eca7 100644
--- a/source/dnode/mnode/impl/src/mnode.c
+++ b/source/dnode/mnode/impl/src/mnode.c
@@ -122,7 +122,7 @@ static void mndCleanupTimer(SMnode *pMnode) {
   pMnode->stopped = true;
   if (taosCheckPthreadValid(pMnode->thread)) {
     taosThreadJoin(pMnode->thread, NULL);
-    memset(&pMnode->thread, 0, sizeof(pMnode->thread));
+    taosThreadClear(&pMnode->thread);
   }
 }
 
diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp
index efaa8bb7d3..82acd7fddc 100644
--- a/source/dnode/mnode/impl/test/trans/trans2.cpp
+++ b/source/dnode/mnode/impl/test/trans/trans2.cpp
@@ -16,10 +16,9 @@
 #include "tcache.h"
 
 void reportStartup(const char *name, const char *desc) {}
-void sendRsp(const SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); }
+void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); }
 
 int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
-  // rpcFreeCont(pMsg->pCont);
   terrno = TSDB_CODE_INVALID_PTR;
   return -1;
 }
diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c
index 1117528b53..203a8a1e62 100644
--- a/source/libs/sync/src/syncIO.c
+++ b/source/libs/sync/src/syncIO.c
@@ -237,6 +237,7 @@ static int32_t syncIOStopInternal(SSyncIO *io) {
   int32_t ret = 0;
   atomic_store_8(&io->isStart, 0);
   taosThreadJoin(io->consumerTid, NULL);
+  taosThreadClear(&io->consumerTid);
   taosTmrCleanUp(io->timerMgr);
   return ret;
 }
diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c
index cf40c998de..ada1f599f2 100644
--- a/source/libs/wal/src/walMgmt.c
+++ b/source/libs/wal/src/walMgmt.c
@@ -244,6 +244,7 @@ static void walStopThread() {
 
   if (taosCheckPthreadValid(tsWal.thread)) {
     taosThreadJoin(tsWal.thread, NULL);
+    taosThreadClear(&tsWal.thread);
   }
 
   wDebug("wal thread is stopped");
diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c
index 46403833f1..c1fc2c48c0 100644
--- a/source/util/src/tlog.c
+++ b/source/util/src/tlog.c
@@ -145,6 +145,7 @@ void taosCloseLog() {
     taosStopLog();
     if (tsLogObj.logHandle != NULL && taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) {
       taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL);
+      taosThreadClear(&tsLogObj.logHandle->asyncThread);
     }
     tsLogInited = 0;
 
diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c
index 2deba5077b..ee1f418561 100644
--- a/source/util/src/tsched.c
+++ b/source/util/src/tsched.c
@@ -209,6 +209,7 @@ void taosCleanUpScheduler(void *param) {
   for (int32_t i = 0; i < pSched->numOfThreads; ++i) {
     if (taosCheckPthreadValid(pSched->qthread[i])) {
       taosThreadJoin(pSched->qthread[i], NULL);
+      taosThreadClear(&pSched->qthread[i]);
     }
   }
 
diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c
index 7fc390e70b..dc48fc3f8d 100644
--- a/source/util/src/tworker.c
+++ b/source/util/src/tworker.c
@@ -57,6 +57,7 @@ void tQWorkerCleanup(SQWorkerPool *pool) {
     if (worker == NULL) continue;
     if (taosCheckPthreadValid(worker->thread)) {
       taosThreadJoin(worker->thread, NULL);
+      taosThreadClear(&worker->thread);
     }
   }
 
@@ -179,6 +180,7 @@ void tWWorkerCleanup(SWWorkerPool *pool) {
     SWWorker *worker = pool->workers + i;
     if (taosCheckPthreadValid(worker->thread)) {
       taosThreadJoin(worker->thread, NULL);
+      taosThreadClear(&worker->thread);
       taosFreeQall(worker->qall);
       taosCloseQset(worker->qset);
     }
diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c
index 4fca7d6245..c53ae0136c 100644
--- a/tests/test/c/create_table.c
+++ b/tests/test/c/create_table.c
@@ -436,6 +436,7 @@ int32_t main(int32_t argc, char *argv[]) {
   taosMsleep(300);
   for (int32_t i = 0; i < numOfThreads; i++) {
     taosThreadJoin(pInfo[i].thread, NULL);
+    taosThreadClear(&pInfo[i].thread);
   }
 
   int64_t maxDelay = 0;
diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index 55aa7a8d31..cf113369bc 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -537,6 +537,7 @@ int main(int32_t argc, char* argv[]) {
 
   for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
     taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL);
+    taosThreadClear(&g_stConfInfo.stThreads[i].thread);
   }
 
   // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt);
diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c
index 69f15a164e..969332ba5f 100644
--- a/tests/tsim/src/simSystem.c
+++ b/tests/tsim/src/simSystem.c
@@ -56,6 +56,7 @@ void simFreeScript(SScript *script) {
       bgScript->killed = true;
       if (taosCheckPthreadValid(bgScript->bgPid)) {
         taosThreadJoin(bgScript->bgPid, NULL);
+        taosThreadClear(&bgScript->bgPid);
       }
 
       simDebug("script:%s, background thread joined", bgScript->fileName);
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 4825aae699..9f9c8821b0 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -985,6 +985,7 @@ int32_t shellExecute() {
   while (1) {
     taosThreadCreate(&shell.pid, NULL, shellThreadLoop, shell.conn);
     taosThreadJoin(shell.pid, NULL);
+    taosThreadClear(&shell.pid);
   }
 
   return 0;

From bf44937dca41e55e44bed19749c7138c09561eac Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Wed, 18 May 2022 14:21:04 +0800
Subject: [PATCH 23/25] fix(os): win compile error

---
 contrib/CMakeLists.txt            | 6 +++---
 source/libs/index/src/indexComm.c | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 97bfcfb8c0..aba955ff3b 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -226,10 +226,10 @@ endif(${BUILD_WITH_NURAFT})
 if(${BUILD_PTHREAD})
     set(CMAKE_BUILD_TYPE release)
     add_definitions(-DPTW32_STATIC_LIB)
-    add_subdirectory(pthread)
+    add_subdirectory(pthread EXCLUDE_FROM_ALL)
     set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
-    add_library(pthread STATIC IMPORTED GLOBAL)
-    SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
+    add_library(pthread INTERFACE)
+    target_link_libraries(pthread INTERFACE libpthreadVC3)
 endif()
 
 # iconv
diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c
index 74e2861037..8763a8ab3f 100644
--- a/source/libs/index/src/indexComm.c
+++ b/source/libs/index/src/indexComm.c
@@ -361,7 +361,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
       tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
-      *dst = *dst - tlen;
+      *dst = (char*) * dst - tlen;
       break;
     }
     case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY

From 292e9afd3da70583a26fde3fdc9923e9c6d54e5f Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 18 May 2022 15:14:23 +0800
Subject: [PATCH 24/25] fix: restart killed child process

---
 source/dnode/mgmt/node_mgmt/src/dmNodes.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c
index ff9d4089cd..f23db5da16 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c
@@ -58,6 +58,7 @@ static int32_t dmNewProc(SMgmtWrapper *pWrapper, EDndNodeType ntype) {
     return -1;
   }
 
+  taosIgnSignal(SIGCHLD);
   pWrapper->proc.pid = pid;
   dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid);
   return 0;
@@ -254,7 +255,7 @@ static void dmWatchNodes(SDnode *pDnode) {
     if (!OnlyInParentProc(pWrapper)) continue;
 
     if (proc->pid <= 0 || !taosProcExist(proc->pid)) {
-      dWarn("node:%s, process:%d is killed and needs to restart", pWrapper->name, proc->pid);
+      dError("node:%s, process:%d is killed and needs to restart", pWrapper->name, proc->pid);
       dmCloseProcRpcHandles(&pWrapper->proc);
       dmNewProc(pWrapper, ntype);
     }

From 2f773114f8b042efa2d1433e36dc53ad248b6172 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Wed, 18 May 2022 15:56:17 +0800
Subject: [PATCH 25/25] fix(os): compile ptr error

---
 source/dnode/mgmt/node_mgmt/src/dmProc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c
index 2e24e3fa1c..0a8b5135aa 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmProc.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c
@@ -137,13 +137,13 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg
       queue->tail = headLen + bodyLen;
     } else if (remain < 8 + headLen) {
       memcpy(queue->pBuffer + queue->tail + 8, pHead, remain - 8);
-      memcpy(queue->pBuffer, pHead + remain - 8, rawHeadLen - (remain - 8));
+      memcpy(queue->pBuffer, (char*)pHead + remain - 8, rawHeadLen - (remain - 8));
       if (rawBodyLen > 0) memcpy(queue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen);
       queue->tail = headLen - (remain - 8) + bodyLen;
     } else if (remain < 8 + headLen + bodyLen) {
       memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen);
       if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, remain - 8 - headLen);
-      if (rawBodyLen > 0) memcpy(queue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen));
+      if (rawBodyLen > 0) memcpy(queue->pBuffer, (char*)pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen));
       queue->tail = bodyLen - (remain - 8 - headLen);
     } else {
       memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen);