From cccd5dbd8bda0ad56d736add278419754a30aefa Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 05:28:08 +0000
Subject: [PATCH 01/15] refact data

---
 include/common/tdataformat.h    |   3 +
 include/common/tmsg.h           |  13 ++-
 source/common/src/tdataformat.c | 165 ++++++++++++++++++++++++--------
 3 files changed, 138 insertions(+), 43 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 2e8a214a9d..b01b7ac240 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -38,6 +38,8 @@ typedef struct SKVIdx        SKVIdx;
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
+                  int8_t *flags);
 
 // STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
@@ -86,6 +88,7 @@ struct STSRowBuilder {
   uint8_t  *pKVBuf;
   int32_t   szTPBuf;
   uint8_t  *pTPBuf;
+  uint8_t  *pBitBuf;
   int32_t   nCols;
   int32_t   kvVLen;
   int32_t   tpVLen;
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 7f17c1673b..6294e14adb 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -271,12 +271,13 @@ typedef struct {
 
 int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
 int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
-void tFreeSSubmitRsp(SSubmitRsp *pRsp);
-
-#define COL_SMA_ON  ((int8_t)0x1)
-#define COL_IDX_ON  ((int8_t)0x2)
-#define COL_VAL_SET ((int8_t)0x4)
+void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
+#define COL_SMA_ON   ((int8_t)0x1)
+#define COL_IDX_ON   ((int8_t)0x2)
+#define COL_SET_VAL  ((int8_t)0x10)
+#define COL_SET_NONE ((int8_t)0x20)
+#define COL_SET_NULL ((int8_t)0x40)
 typedef struct SSchema {
   int8_t   type;
   int8_t   flags;
@@ -285,6 +286,8 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
+#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0)
+
 #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON)
 
 #define SSCHMEA_TYPE(s)  ((s)->type)
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index ad020fe7d9..d492d98b63 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -19,13 +19,17 @@
 #include "tdatablock.h"
 #include "tlog.h"
 
-#define TD_KV_ROW 0x1U
+#define TD_HAS_NONE 0x1U
+#define TD_HAS_NULL 0x2U
+#define TD_HAS_VAL  0x4U
+#define TD_KV_ROW   0x10U
 
 struct SKVIdx {
   int32_t cid;
   int32_t offset;
 };
 
+// STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
   if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
   if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1;
@@ -50,6 +54,59 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
   return 0;
 }
 
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
+                  int8_t *flags) {
+  if (cid == 0) {
+    *ppData = (uint8_t *)&pRow->ts;
+    *nData = sizeof(TSKEY);
+    *flags = 0;
+  } else {
+    uint32_t tflags = pRow->flags & 0xf;
+    *ppData = NULL;
+    *nData = 0;
+
+    switch (tflags) {
+      case TD_HAS_NONE:
+        *flags = -1;
+        break;
+      case TD_HAS_NULL:
+        *flags = 1;
+        break;
+      case TD_HAS_VAL:
+        *flags = 0;
+        // find the row
+        break;
+      case TD_HAS_NULL | TD_HAS_NONE:
+        // read bit map (todo)
+        if (0) {
+          *flags = 1;
+        } else {
+          *flags = -1;
+        }
+        break;
+      case TD_HAS_VAL | TD_HAS_NONE:
+      case TD_HAS_VAL | TD_HAS_NULL:
+        // read bitmap (todo)
+        if (0) {
+          if (tflags & TD_HAS_NONE) {
+            *flags = -1;
+          } else {
+            *flags = 1;
+          }
+        } else {
+          // get value (todo)
+        }
+        break;
+      case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
+        break;
+      default:
+        return -1;
+    }
+  }
+  return 0;
+}
+
+// STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) {
   *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols);
   if (*ppTSchema == NULL) {
@@ -87,6 +144,7 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema *
 
 void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); }
 
+// STSRowBuilder
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
   int32_t  kvBufLen;
   int32_t  tpBufLen;
@@ -131,7 +189,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
   for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) {
     pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
 
-    pBuilder->pTColumn->flags &= (~COL_VAL_SET);
+    pBuilder->pTColumn->flags &= 0xf;
   }
 
   pBuilder->nCols = 0;
@@ -144,7 +202,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   int32_t  iCol;
   uint8_t *p;
 
-  // search column
+  // search column (todo: make here faster)
   if (pBuilder->pTColumn->colId < cid) {
     iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
     for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
@@ -160,7 +218,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   }
 
   // check
-  if (pBuilder->pTColumn->colId != cid || pBuilder->pTColumn->flags & COL_VAL_SET) {
+  if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) {
     return -1;
   }
 
@@ -168,9 +226,12 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   if (cid == 0) {
     ASSERT(pData && nData == sizeof(TSKEY));
     pBuilder->row.ts = *(TSKEY *)pData;
+
+    pBuilder->pTColumn->flags |= COL_SET_VAL;
   } else {
     if (pData) {
-      // ASSERT(!IS_NULL(pData));
+      pBuilder->row.flags |= TD_HAS_VAL;
+      pBuilder->pTColumn->flags |= COL_SET_VAL;
 
       // set tuple data
       p = pBuilder->pTPBuf + pBuilder->pTColumn->offset;
@@ -197,51 +258,79 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
         pBuilder->kvVLen += nData;
       }
     } else {
-      // set NULL val
+      pBuilder->row.flags |= TD_HAS_NULL;
+      pBuilder->pTColumn->flags |= COL_SET_NULL;
+
+      p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
+      ((SKVIdx *)p)->cid = cid;
+      ((SKVIdx *)p)->offset = -1;
     }
+
+    pBuilder->nCols++;
   }
 
-  pBuilder->pTColumn->flags |= COL_VAL_SET;
-  pBuilder->nCols++;
   return 0;
 }
 
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
-  if ((pBuilder->pTSchema->columns[0].flags & COL_VAL_SET) == 0) {
+  int32_t  tpDataLen, kvDataLen;
+  uint32_t flags;
+
+  // error not set ts
+  if (!COL_IS_SET(pBuilder->pTSchema->columns)) {
     return -1;
   }
 
-  if (pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen < pBuilder->pTSchema->flen + pBuilder->tpVLen) {
-    // encode as TD_KV_ROW
-    pBuilder->row.flags |= TD_KV_ROW;
-    pBuilder->row.ncols = pBuilder->nCols;
-    pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen;
-    pBuilder->row.pData = pBuilder->pKVBuf;
+  if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
+    pBuilder->row.flags |= TD_HAS_NONE;
+  }
 
-    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) {
-      memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
-              pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen);
-    }
-  } else {
-    // encode as TD_TUPLE_ROW
-    pBuilder->row.flags &= (~TD_KV_ROW);
-    pBuilder->row.sver = pBuilder->pTSchema->version;
-    pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
-    pBuilder->row.pData = pBuilder->pTPBuf;
-
-    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) {
-      // set non-set cols as None
-      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
-        pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
-        if (pBuilder->pTColumn->flags & COL_VAL_SET) continue;
-
-        {
-          // set None (todo)
-        }
-
-        pBuilder->pTColumn->flags |= COL_VAL_SET;
+  flags = pBuilder->row.flags & 0xf;
+  switch (flags) {
+    case TD_HAS_NONE:
+    case TD_HAS_NULL:
+      pBuilder->row.sver = pBuilder->pTSchema->version;
+      pBuilder->row.nData = 0;
+      pBuilder->row.pData = NULL;
+      break;
+    case TD_HAS_VAL:
+      pBuilder->row.sver = pBuilder->pTSchema->version;
+      pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
+      pBuilder->row.pData = pBuilder->pTPBuf;
+      break;
+    case TD_HAS_NULL | TD_HAS_NONE:
+      pBuilder->row.sver = pBuilder->pTSchema->version;
+      // set bitmap (todo)
+      pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1;
+      pBuilder->row.pData = pBuilder->pBitBuf;
+      break;
+    case TD_HAS_VAL | TD_HAS_NONE:
+    case TD_HAS_VAL | TD_HAS_NULL:
+    case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
+      if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) {
+        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
+      } else {
+        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
       }
-    }
+      kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
+
+      if (kvDataLen < tpDataLen) {
+        pBuilder->row.flags |= TD_KV_ROW;
+        pBuilder->row.ncols = pBuilder->nCols;
+        pBuilder->row.nData = kvDataLen;
+        pBuilder->row.pData = pBuilder->pKVBuf;
+        // memmove(); todo
+        // qsort
+      } else {
+        pBuilder->row.sver = pBuilder->pTSchema->numOfCols;
+        // set bitmap etc (todo)
+        pBuilder->row.nData = tpDataLen;
+        pBuilder->row.pData = pBuilder->pTPBuf;
+      }
+      break;
+    default:
+      ASSERT(0);
+      return -1;
   }
 
   *ppRow = &pBuilder->row;

From 793b49dbf61c29fe6802e7d7658223e7442b94d9 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 10:38:35 +0000
Subject: [PATCH 02/15] feat: data format

---
 include/common/tdataformat.h    |  12 +-
 include/common/tmsg.h           |   8 +-
 source/common/src/tdataformat.c | 194 +++++++++++++++++++-------------
 3 files changed, 126 insertions(+), 88 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index b01b7ac240..de7c2a49f6 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -33,9 +33,9 @@ typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
 typedef struct SKVIdx        SKVIdx;
 
-// STSchema
-
 // STSRow2
+#define TSROW_SVER(r) (((r)->flags & TSROW_KV_ROW) ? -1 : (r)->sver)
+
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
@@ -70,9 +70,13 @@ struct STSchema {
   STColumn columns[];
 };
 
+#define TSROW_HAS_NONE ((uint8_t)0x1)
+#define TSROW_HAS_NULL ((uint8_t)0x2U)
+#define TSROW_HAS_VAL  ((uint8_t)0x4U)
+#define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY    ts;
-  uint32_t flags;
+  TSKEY   ts;
+  uint8_t flags;
   union {
     int32_t sver;
     int32_t ncols;
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 8eb9ae62d8..d860ce5b99 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -275,9 +275,8 @@ void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
 #define COL_SMA_ON   ((int8_t)0x1)
 #define COL_IDX_ON   ((int8_t)0x2)
-#define COL_SET_VAL  ((int8_t)0x10)
-#define COL_SET_NONE ((int8_t)0x20)
-#define COL_SET_NULL ((int8_t)0x40)
+#define COL_SET_NULL ((int8_t)0x10)
+#define COL_SET_VAL  ((int8_t)0x20)
 typedef struct SSchema {
   int8_t   type;
   int8_t   flags;
@@ -286,7 +285,8 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
-#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0)
+#define COL_IS_SET(FLG)  ((FLG) & (COL_SET_VAL | COL_SET_NULL) != 0)
+#define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL)))
 
 #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON)
 
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index d492d98b63..0fed7c86fe 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -19,38 +19,66 @@
 #include "tdatablock.h"
 #include "tlog.h"
 
-#define TD_HAS_NONE 0x1U
-#define TD_HAS_NULL 0x2U
-#define TD_HAS_VAL  0x4U
-#define TD_KV_ROW   0x10U
-
 struct SKVIdx {
   int32_t cid;
   int32_t offset;
 };
 
+#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
+
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
   if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
-  if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1;
-  if (pRow->flags & TD_KV_ROW) {
-    if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1;
-  } else {
-    if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
+  if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1;
+
+  ASSERT(pRow->flags & 0xf != 0);
+
+  switch (pRow->flags & 0xf) {
+    case TSROW_HAS_NONE:
+    case TSROW_HAS_NULL:
+      ASSERT(TSROW_IS_KV_ROW(pRow) && pRow->nData == 0 && pRow->pData == NULL);
+      return 0;
+    case TSROW_HAS_VAL:
+      ASSERT(!TSROW_IS_KV_ROW(pRow));
+    default:
+      ASSERT(pRow->nData && pRow->pData);
+      if (TSROW_IS_KV_ROW(pRow)) {
+        if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1;
+      } else {
+        if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
+      }
+      if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1;
+      break;
   }
-  if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData) < 0) return -1;
+
   return 0;
 }
 
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
   if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1;
-  if (tDecodeU32v(pDecoder, &pRow->flags) < 0) return -1;
-  if (pRow->flags & TD_KV_ROW) {
-    if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1;
-  } else {
-    if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
+  if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1;
+
+  ASSERT(pRow->flags & 0xf != 0);
+
+  switch (pRow->flags & 0xf) {
+    case TSROW_HAS_NONE:
+    case TSROW_HAS_NULL:
+      ASSERT(TSROW_IS_KV_ROW(pRow));
+      pRow->nData = 0;
+      pRow->pData = NULL;
+      return 0;
+    case TSROW_HAS_VAL:
+      ASSERT(!TSROW_IS_KV_ROW(pRow));
+    default:
+      if (TSROW_IS_KV_ROW(pRow)) {
+        if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1;
+      } else {
+        if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
+      }
+      if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1;
+      break;
   }
-  if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData) < 0) return -1;
+
   return 0;
 }
 
@@ -66,17 +94,17 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui
     *nData = 0;
 
     switch (tflags) {
-      case TD_HAS_NONE:
+      case TSROW_HAS_NONE:
         *flags = -1;
         break;
-      case TD_HAS_NULL:
+      case TSROW_HAS_NULL:
         *flags = 1;
         break;
-      case TD_HAS_VAL:
+      case TSROW_HAS_VAL:
         *flags = 0;
         // find the row
         break;
-      case TD_HAS_NULL | TD_HAS_NONE:
+      case TSROW_HAS_NULL | TSROW_HAS_NONE:
         // read bit map (todo)
         if (0) {
           *flags = 1;
@@ -84,11 +112,11 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui
           *flags = -1;
         }
         break;
-      case TD_HAS_VAL | TD_HAS_NONE:
-      case TD_HAS_VAL | TD_HAS_NULL:
+      case TSROW_HAS_VAL | TSROW_HAS_NONE:
+      case TSROW_HAS_VAL | TSROW_HAS_NULL:
         // read bitmap (todo)
         if (0) {
-          if (tflags & TD_HAS_NONE) {
+          if (tflags & TSROW_HAS_NONE) {
             *flags = -1;
           } else {
             *flags = 1;
@@ -97,7 +125,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui
           // get value (todo)
         }
         break;
-      case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
+      case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
         break;
       default:
         return -1;
@@ -199,18 +227,15 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
 }
 
 int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) {
-  int32_t  iCol;
-  uint8_t *p;
-
-  // search column (todo: make here faster)
+  // search column (TODO: bsearch with interp)
   if (pBuilder->pTColumn->colId < cid) {
-    iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
+    int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
     for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
       pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
       if (pBuilder->pTColumn->colId == cid) break;
     }
   } else if (pBuilder->pTColumn->colId > cid) {
-    iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1;
+    int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1;
     for (; iCol >= 0; iCol--) {
       pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
       if (pBuilder->pTColumn->colId == cid) break;
@@ -218,19 +243,20 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   }
 
   // check
-  if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) {
+  if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn->flags)) {
     return -1;
   }
 
   // set value
+  uint8_t *p;
   if (cid == 0) {
     ASSERT(pData && nData == sizeof(TSKEY));
     pBuilder->row.ts = *(TSKEY *)pData;
 
     pBuilder->pTColumn->flags |= COL_SET_VAL;
   } else {
-    if (pData) {
-      pBuilder->row.flags |= TD_HAS_VAL;
+    if (pData) {  // set val
+      pBuilder->row.flags |= TSROW_HAS_VAL;
       pBuilder->pTColumn->flags |= COL_SET_VAL;
 
       // set tuple data
@@ -250,20 +276,20 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
       ((SKVIdx *)p)->cid = cid;
       ((SKVIdx *)p)->offset = pBuilder->kvVLen;
 
-      p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols + pBuilder->kvVLen;
+      p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->kvVLen;
       if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) {
         pBuilder->kvVLen += tPutBinary(p, pData, nData);
       } else {
         memcpy(p, pData, nData);
         pBuilder->kvVLen += nData;
       }
-    } else {
-      pBuilder->row.flags |= TD_HAS_NULL;
+    } else {  // set NULL
+      pBuilder->row.flags |= TSROW_HAS_NULL;
       pBuilder->pTColumn->flags |= COL_SET_NULL;
 
       p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
       ((SKVIdx *)p)->cid = cid;
-      ((SKVIdx *)p)->offset = -1;
+      ((SKVIdx *)p)->offset = -1;  // for TSROW_KV_ROW, offset -1 means NULL
     }
 
     pBuilder->nCols++;
@@ -272,68 +298,76 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   return 0;
 }
 
+static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) {
+  SKVIdx *pKVIdx1 = (SKVIdx *)p1;
+  SKVIdx *pKVIdx2 = (SKVIdx *)p2;
+  if (pKVIdx1->cid > pKVIdx2->cid) {
+    return 1;
+  } else if (pKVIdx1->cid < pKVIdx2->cid) {
+    return -1;
+  }
+  return 0;
+}
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   int32_t  tpDataLen, kvDataLen;
   uint32_t flags;
 
   // error not set ts
-  if (!COL_IS_SET(pBuilder->pTSchema->columns)) {
+  if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) {
     return -1;
   }
 
+  ASSERT(pBuilder->nCols < pBuilder->pTSchema->numOfCols);
   if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
-    pBuilder->row.flags |= TD_HAS_NONE;
+    pBuilder->row.flags |= TSROW_HAS_NONE;
   }
 
-  flags = pBuilder->row.flags & 0xf;
-  switch (flags) {
-    case TD_HAS_NONE:
-    case TD_HAS_NULL:
-      pBuilder->row.sver = pBuilder->pTSchema->version;
+  ASSERT(pBuilder->row.flags & 0xf != 0);
+  *(ppRow) = &pBuilder->row;
+  switch (pBuilder->row.flags & 0xf) {
+    case TSROW_HAS_NONE:
+    case TSROW_HAS_NULL:
+      pBuilder->row.flags |= TSROW_KV_ROW;
       pBuilder->row.nData = 0;
       pBuilder->row.pData = NULL;
+      return 0;
+    case TSROW_HAS_NULL | TSROW_HAS_NONE:
+      tpDataLen = (pBuilder->pTSchema->numOfCols - 1) / 8;
       break;
-    case TD_HAS_VAL:
-      pBuilder->row.sver = pBuilder->pTSchema->version;
-      pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
-      pBuilder->row.pData = pBuilder->pTPBuf;
+    case TSROW_HAS_VAL:
+      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen;
       break;
-    case TD_HAS_NULL | TD_HAS_NONE:
-      pBuilder->row.sver = pBuilder->pTSchema->version;
-      // set bitmap (todo)
-      pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1;
-      pBuilder->row.pData = pBuilder->pBitBuf;
+    case TSROW_HAS_VAL | TSROW_HAS_NONE:
+    case TSROW_HAS_VAL | TSROW_HAS_NULL:
+      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 8;
       break;
-    case TD_HAS_VAL | TD_HAS_NONE:
-    case TD_HAS_VAL | TD_HAS_NULL:
-    case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
-      if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) {
-        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
-      } else {
-        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
-      }
-      kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
-
-      if (kvDataLen < tpDataLen) {
-        pBuilder->row.flags |= TD_KV_ROW;
-        pBuilder->row.ncols = pBuilder->nCols;
-        pBuilder->row.nData = kvDataLen;
-        pBuilder->row.pData = pBuilder->pKVBuf;
-        // memmove(); todo
-        // qsort
-      } else {
-        pBuilder->row.sver = pBuilder->pTSchema->numOfCols;
-        // set bitmap etc (todo)
-        pBuilder->row.nData = tpDataLen;
-        pBuilder->row.pData = pBuilder->pTPBuf;
-      }
+    case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
+      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 4;
       break;
     default:
-      ASSERT(0);
-      return -1;
+      // decide chose tuple or kv
+      kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
+      break;
+  }
+
+  if (kvDataLen < tpDataLen) {
+    pBuilder->row.flags |= TSROW_KV_ROW;
+    pBuilder->row.ncols = pBuilder->nCols;
+
+    pBuilder->row.nData = kvDataLen;
+    pBuilder->row.pData = pBuilder->pKVBuf;
+    qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
+    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
+      memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
+              pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->kvVLen);
+    }
+  } else {
+    pBuilder->row.sver = pBuilder->pTSchema->version;
+
+    pBuilder->row.nData = tpDataLen;
+    pBuilder->row.pData
   }
 
-  *ppRow = &pBuilder->row;
   return 0;
 }
 

From 6972b1d6d14ac119589434b64a3cdd43beececa0 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 10:41:29 +0000
Subject: [PATCH 03/15] make compile

---
 source/common/src/tdataformat.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 0fed7c86fe..efc8819435 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -365,7 +365,6 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
     pBuilder->row.sver = pBuilder->pTSchema->version;
 
     pBuilder->row.nData = tpDataLen;
-    pBuilder->row.pData
   }
 
   return 0;

From 038e9558bbf40a030deb95bc25e18f975d1e5c70 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 13 May 2022 01:19:34 +0000
Subject: [PATCH 04/15] more data

---
 include/common/tdataformat.h    |   9 +-
 source/common/src/tdataformat.c | 222 +++++++++++++++++++-------------
 2 files changed, 140 insertions(+), 91 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index aed7f6ddc6..ad47b82e7f 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -86,16 +86,17 @@ struct STSRow2 {
 };
 
 struct STSRowBuilder {
-  STColumn *pTColumn;
   STSchema *pTSchema;
+  int32_t   szBitMap1;
+  int32_t   szBitMap2;
   int32_t   szKVBuf;
   uint8_t  *pKVBuf;
   int32_t   szTPBuf;
   uint8_t  *pTPBuf;
-  uint8_t  *pBitBuf;
+  int32_t   iCol;
   int32_t   nCols;
-  int32_t   kvVLen;
-  int32_t   tpVLen;
+  int32_t   vlenKV;
+  int32_t   vlenTP;
   STSRow2   row;
 };
 
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index efc8819435..a951efbe20 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -25,6 +25,8 @@ struct SKVIdx {
 };
 
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
+#define SET_BIT1(p, i, v)
+#define SET_BIT2(p, i, v)
 
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
@@ -170,126 +172,135 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema *
   return 0;
 }
 
-void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); }
+void tTSchemaDestroy(STSchema *pTSchema) {
+  if (pTSchema) taosMemoryFree(pTSchema);
+}
 
 // STSRowBuilder
-int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
-  int32_t  kvBufLen;
-  int32_t  tpBufLen;
-  uint8_t *p;
+static int32_t tTSRowBitMapLen1(int32_t nCols) { return nCols / 8 + ((nCols % 8) == 0 ? 0 : 1); }
+static int32_t tTSRowBitMapLen2(int32_t nCols) { return nCols / 4 + ((nCols % 4) == 0 ? 0 : 1); }
 
+int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
   if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1;
 
-  kvBufLen = sizeof(SKVIdx) * nCols + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
-  tpBufLen = pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
-
-  if (pBuilder->szKVBuf < kvBufLen) {
-    p = taosMemoryRealloc(pBuilder->pKVBuf, kvBufLen);
-    if (p == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-    pBuilder->pKVBuf = p;
-    pBuilder->szKVBuf = kvBufLen;
+  pBuilder->szBitMap1 = tTSRowBitMapLen1(nCols - 1);
+  pBuilder->szBitMap2 = tTSRowBitMapLen2(nCols - 1);
+  pBuilder->szKVBuf = sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
+  pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
+  pBuilder->pKVBuf = taosMemoryMalloc(pBuilder->szKVBuf);
+  if (pBuilder->pKVBuf == NULL) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return -1;
   }
-
-  if (pBuilder->szTPBuf < tpBufLen) {
-    p = taosMemoryRealloc(pBuilder->pTPBuf, tpBufLen);
-    if (p == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-    pBuilder->pTPBuf = p;
-    pBuilder->szTPBuf = tpBufLen;
+  pBuilder->pTPBuf = taosMemoryMalloc(pBuilder->szTPBuf);
+  if (pBuilder->pTPBuf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
   }
 
-  tTSRowBuilderReset(pBuilder);
-
   return 0;
 }
 
 void tTSRowBuilderClear(STSRowBuilder *pBuilder) {
-  taosMemoryFree(pBuilder->pKVBuf);
-  taosMemoryFree(pBuilder->pTPBuf);
+  tTSchemaDestroy(pBuilder->pTSchema);
+  pBuilder->pTSchema = NULL;
+  if (pBuilder->pKVBuf) {
+    taosMemoryFree(pBuilder->pKVBuf);
+    pBuilder->pKVBuf = NULL;
+  }
+  if (pBuilder->pTPBuf) {
+    taosMemoryFree(pBuilder->pTPBuf);
+    pBuilder->pTPBuf = NULL;
+  }
 }
 
 void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
   for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) {
-    pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
-
-    pBuilder->pTColumn->flags &= 0xf;
+    STColumn *pTColumn = &pBuilder->pTSchema->columns[iCol];
+    COL_CLR_SET(pTColumn->flags);
   }
 
+  pBuilder->iCol = 0;
   pBuilder->nCols = 0;
-  pBuilder->kvVLen = 0;
-  pBuilder->tpVLen = 0;
+  pBuilder->vlenKV = 0;
+  pBuilder->vlenTP = 0;
   pBuilder->row.flags = 0;
 }
 
 int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) {
-  // search column (TODO: bsearch with interp)
-  if (pBuilder->pTColumn->colId < cid) {
-    int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
-    for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
-      pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
-      if (pBuilder->pTColumn->colId == cid) break;
+  STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol];
+  uint8_t  *p;
+  int32_t   iCol;
+
+  // use interp search (todo)
+  if (pTColumn->colId > cid) {
+    for (iCol = pBuilder->iCol + 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
+      pTColumn = &pBuilder->pTSchema->columns[iCol];
+      if (pTColumn->colId == cid) break;
     }
-  } else if (pBuilder->pTColumn->colId > cid) {
-    int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1;
-    for (; iCol >= 0; iCol--) {
-      pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
-      if (pBuilder->pTColumn->colId == cid) break;
+  } else if (pTColumn->colId < cid) {
+    for (iCol = pBuilder->iCol - 1; iCol >= 0; iCol--) {
+      pTColumn = &pBuilder->pTSchema->columns[iCol];
+      if (pTColumn->colId == cid) break;
     }
   }
 
-  // check
-  if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn->flags)) {
+  if (pTColumn->colId != cid || COL_IS_SET(pTColumn->flags)) {
     return -1;
   }
 
+  pBuilder->iCol = iCol;
+
   // set value
-  uint8_t *p;
   if (cid == 0) {
-    ASSERT(pData && nData == sizeof(TSKEY));
+    ASSERT(pData && nData == sizeof(TSKEY) && iCol == 0);
     pBuilder->row.ts = *(TSKEY *)pData;
-
-    pBuilder->pTColumn->flags |= COL_SET_VAL;
+    pTColumn->flags |= COL_SET_VAL;
   } else {
-    if (pData) {  // set val
+    if (pData) {
+      // set VAL
+
       pBuilder->row.flags |= TSROW_HAS_VAL;
-      pBuilder->pTColumn->flags |= COL_SET_VAL;
+      pTColumn->flags |= COL_SET_VAL;
 
-      // set tuple data
-      p = pBuilder->pTPBuf + pBuilder->pTColumn->offset;
-      if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) {
-        *(int32_t *)p = pBuilder->tpVLen;
+      /* KV */
+      if (1) {  // avoid KV at some threshold (todo)
+        p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
+        ((SKVIdx *)p)->cid = cid;
+        ((SKVIdx *)p)->offset = pBuilder->vlenKV;
 
-        // encode the variant-length data
-        p = pBuilder->pTPBuf + pBuilder->pTSchema->flen + pBuilder->tpVLen;
-        pBuilder->tpVLen += tPutBinary(p, pData, nData);
-      } else {
-        memcpy(p, pData, nData);
+        p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->vlenKV;
+        if (IS_VAR_DATA_TYPE(pTColumn->type)) {
+          ASSERT(nData <= pTColumn->bytes);
+          pBuilder->vlenKV += tPutBinary(p, pData, nData);
+        } else {
+          ASSERT(nData == pTColumn->bytes);
+          memcpy(p, pData, nData);
+          pBuilder->vlenKV += nData;
+        }
       }
 
-      // set kv data
-      p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
-      ((SKVIdx *)p)->cid = cid;
-      ((SKVIdx *)p)->offset = pBuilder->kvVLen;
+      /* TUPLE */
+      p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pTColumn->offset;
+      if (IS_VAR_DATA_TYPE(pTColumn->type)) {
+        ASSERT(nData <= pTColumn->bytes);
+        *(int32_t *)p = pBuilder->vlenTP;
 
-      p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->kvVLen;
-      if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) {
-        pBuilder->kvVLen += tPutBinary(p, pData, nData);
+        p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen;
+        pBuilder->vlenTP += tPutBinary(p, pData, nData);
       } else {
+        ASSERT(nData == pTColumn->bytes);
         memcpy(p, pData, nData);
-        pBuilder->kvVLen += nData;
       }
-    } else {  // set NULL
+    } else {
+      // set NULL
+
       pBuilder->row.flags |= TSROW_HAS_NULL;
-      pBuilder->pTColumn->flags |= COL_SET_NULL;
+      pTColumn->flags |= COL_SET_NULL;
 
       p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
       ((SKVIdx *)p)->cid = cid;
-      ((SKVIdx *)p)->offset = -1;  // for TSROW_KV_ROW, offset -1 means NULL
+      ((SKVIdx *)p)->offset = -1;
     }
 
     pBuilder->nCols++;
@@ -309,7 +320,7 @@ static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) {
   return 0;
 }
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
-  int32_t  tpDataLen, kvDataLen;
+  int32_t  nDataTP, nDataKV;
   uint32_t flags;
 
   // error not set ts
@@ -332,39 +343,76 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
       pBuilder->row.pData = NULL;
       return 0;
     case TSROW_HAS_NULL | TSROW_HAS_NONE:
-      tpDataLen = (pBuilder->pTSchema->numOfCols - 1) / 8;
+      nDataTP = pBuilder->szBitMap1;
       break;
     case TSROW_HAS_VAL:
-      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen;
+      nDataTP = pBuilder->pTSchema->flen + pBuilder->vlenTP;
       break;
     case TSROW_HAS_VAL | TSROW_HAS_NONE:
     case TSROW_HAS_VAL | TSROW_HAS_NULL:
-      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 8;
+      nDataTP = pBuilder->szBitMap1 + pBuilder->pTSchema->flen + pBuilder->vlenTP;
       break;
     case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
-      tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 4;
+      nDataTP = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP;
       break;
     default:
-      // decide chose tuple or kv
-      kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
-      break;
+      ASSERT(0);
   }
 
-  if (kvDataLen < tpDataLen) {
+  nDataKV = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->vlenKV;
+  ASSERT(pBuilder->row.flags & TSROW_KV_ROW == 0);
+  if (nDataKV < nDataTP) {
+    // generate KV row
+
     pBuilder->row.flags |= TSROW_KV_ROW;
     pBuilder->row.ncols = pBuilder->nCols;
-
-    pBuilder->row.nData = kvDataLen;
+    pBuilder->row.nData = nDataKV;
     pBuilder->row.pData = pBuilder->pKVBuf;
+
     qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
     if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
       memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
-              pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->kvVLen);
+              pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->vlenKV);
     }
   } else {
-    pBuilder->row.sver = pBuilder->pTSchema->version;
+    // generate TUPLE row
 
-    pBuilder->row.nData = tpDataLen;
+    uint8_t  *p;
+    STColumn *pTColumn;
+    pBuilder->row.sver = pBuilder->pTSchema->version;
+    pBuilder->row.nData = nDataTP;
+    if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL) {
+      // no bitmap
+      pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2;
+    } else if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) {
+      // bitmap2
+      p = pBuilder->pTPBuf;
+
+      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
+        pTColumn = &pBuilder->pTSchema->columns[iCol];
+
+        if (pTColumn->flags & COL_SET_VAL) {
+          SET_BIT2(p, iCol - 1, 0x2);
+        } else if (pTColumn->flags & COL_SET_VAL) {
+          SET_BIT2(p, iCol - 1, 0x1);
+        }
+      }
+
+      pBuilder->row.pData = p;
+    } else {
+      // bitmap1
+      p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
+
+      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
+        pTColumn = &pBuilder->pTSchema->columns[iCol];
+
+        if (1) {
+          SET_BIT1(p, iCol - 1, 0x1);
+        }
+      }
+
+      pBuilder->row.pData = p;
+    }
   }
 
   return 0;

From e9ffb086a5443082e2f768dce2dac95352838c81 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 13 May 2022 15:36:26 +0000
Subject: [PATCH 05/15] more data format

---
 include/common/tdataformat.h    |  12 +--
 source/common/src/tdataformat.c | 171 +++++++++++++++++---------------
 2 files changed, 96 insertions(+), 87 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index ad47b82e7f..b8a4f6ac9d 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -34,8 +34,6 @@ typedef struct STSRowBuilder STSRowBuilder;
 typedef struct SKVIdx        SKVIdx;
 
 // STSRow2
-#define TSROW_SVER(r) (((r)->flags & TSROW_KV_ROW) ? -1 : (r)->sver)
-
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
@@ -75,12 +73,9 @@ struct STSchema {
 #define TSROW_HAS_VAL  ((uint8_t)0x4U)
 #define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY   ts;
-  uint8_t flags;
-  union {
-    int32_t sver;
-    int32_t ncols;
-  };
+  TSKEY          ts;
+  uint8_t        flags;
+  int32_t        sver;
   uint32_t       nData;
   const uint8_t *pData;
 };
@@ -94,7 +89,6 @@ struct STSRowBuilder {
   int32_t   szTPBuf;
   uint8_t  *pTPBuf;
   int32_t   iCol;
-  int32_t   nCols;
   int32_t   vlenKV;
   int32_t   vlenTP;
   STSRow2   row;
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index a951efbe20..055eb9df80 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -24,31 +24,33 @@ struct SKVIdx {
   int32_t offset;
 };
 
+#pragma pack(push, 1)
+typedef struct {
+  int16_t nCols;
+  SKVIdx  idx[];
+} STSKVRow;
+#pragma pack(pop)
+
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
-#define SET_BIT1(p, i, v)
-#define SET_BIT2(p, i, v)
+#define SET_BIT1(p, i, v)  ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8)))
+#define SET_BIT2(p, i, v)  ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4)))
 
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
   if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
   if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1;
+  if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
 
   ASSERT(pRow->flags & 0xf != 0);
 
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      ASSERT(TSROW_IS_KV_ROW(pRow) && pRow->nData == 0 && pRow->pData == NULL);
       return 0;
     case TSROW_HAS_VAL:
       ASSERT(!TSROW_IS_KV_ROW(pRow));
     default:
       ASSERT(pRow->nData && pRow->pData);
-      if (TSROW_IS_KV_ROW(pRow)) {
-        if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1;
-      } else {
-        if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
-      }
       if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1;
       break;
   }
@@ -59,24 +61,19 @@ int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
   if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1;
   if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
 
   ASSERT(pRow->flags & 0xf != 0);
 
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      ASSERT(TSROW_IS_KV_ROW(pRow));
       pRow->nData = 0;
       pRow->pData = NULL;
       return 0;
     case TSROW_HAS_VAL:
       ASSERT(!TSROW_IS_KV_ROW(pRow));
     default:
-      if (TSROW_IS_KV_ROW(pRow)) {
-        if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1;
-      } else {
-        if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
-      }
       if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1;
       break;
   }
@@ -86,6 +83,7 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
 
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
                   int8_t *flags) {
+#if 0
   if (cid == 0) {
     *ppData = (uint8_t *)&pRow->ts;
     *nData = sizeof(TSKEY);
@@ -133,6 +131,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui
         return -1;
     }
   }
+#endif
   return 0;
 }
 
@@ -177,24 +176,25 @@ void tTSchemaDestroy(STSchema *pTSchema) {
 }
 
 // STSRowBuilder
-static int32_t tTSRowBitMapLen1(int32_t nCols) { return nCols / 8 + ((nCols % 8) == 0 ? 0 : 1); }
-static int32_t tTSRowBitMapLen2(int32_t nCols) { return nCols / 4 + ((nCols % 4) == 0 ? 0 : 1); }
-
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
   if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1;
 
-  pBuilder->szBitMap1 = tTSRowBitMapLen1(nCols - 1);
-  pBuilder->szBitMap2 = tTSRowBitMapLen2(nCols - 1);
-  pBuilder->szKVBuf = sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
+  pBuilder->szBitMap1 = (nCols - 2) / 8 + 1;
+  pBuilder->szBitMap2 = (nCols - 2) / 4 + 1;
+  pBuilder->szKVBuf =
+      sizeof(STSKVRow) + sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
   pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
   pBuilder->pKVBuf = taosMemoryMalloc(pBuilder->szKVBuf);
   if (pBuilder->pKVBuf == NULL) {
-    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    tTSchemaDestroy(pBuilder->pTSchema);
     return -1;
   }
   pBuilder->pTPBuf = taosMemoryMalloc(pBuilder->szTPBuf);
   if (pBuilder->pTPBuf == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(pBuilder->pKVBuf);
+    tTSchemaDestroy(pBuilder->pTSchema);
     return -1;
   }
 
@@ -202,16 +202,16 @@ int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchem
 }
 
 void tTSRowBuilderClear(STSRowBuilder *pBuilder) {
-  tTSchemaDestroy(pBuilder->pTSchema);
-  pBuilder->pTSchema = NULL;
-  if (pBuilder->pKVBuf) {
-    taosMemoryFree(pBuilder->pKVBuf);
-    pBuilder->pKVBuf = NULL;
-  }
   if (pBuilder->pTPBuf) {
     taosMemoryFree(pBuilder->pTPBuf);
     pBuilder->pTPBuf = NULL;
   }
+  if (pBuilder->pKVBuf) {
+    taosMemoryFree(pBuilder->pKVBuf);
+    pBuilder->pKVBuf = NULL;
+  }
+  tTSchemaDestroy(pBuilder->pTSchema);
+  pBuilder->pTSchema = NULL;
 }
 
 void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
@@ -221,7 +221,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
   }
 
   pBuilder->iCol = 0;
-  pBuilder->nCols = 0;
+  ((STSKVRow *)pBuilder->pKVBuf)->nCols = 0;
   pBuilder->vlenKV = 0;
   pBuilder->vlenTP = 0;
   pBuilder->row.flags = 0;
@@ -231,6 +231,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol];
   uint8_t  *p;
   int32_t   iCol;
+  STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf;
 
   // use interp search (todo)
   if (pTColumn->colId > cid) {
@@ -265,11 +266,11 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
 
       /* KV */
       if (1) {  // avoid KV at some threshold (todo)
-        p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
-        ((SKVIdx *)p)->cid = cid;
-        ((SKVIdx *)p)->offset = pBuilder->vlenKV;
+        pTSKVRow->idx[pTSKVRow->nCols].cid = cid;
+        pTSKVRow->idx[pTSKVRow->nCols].offset = pBuilder->vlenKV;
 
-        p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->vlenKV;
+        p = pBuilder->pKVBuf + sizeof(STSKVRow) + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) +
+            pBuilder->vlenKV;
         if (IS_VAR_DATA_TYPE(pTColumn->type)) {
           ASSERT(nData <= pTColumn->bytes);
           pBuilder->vlenKV += tPutBinary(p, pData, nData);
@@ -286,7 +287,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
         ASSERT(nData <= pTColumn->bytes);
         *(int32_t *)p = pBuilder->vlenTP;
 
-        p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen;
+        p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP;
         pBuilder->vlenTP += tPutBinary(p, pData, nData);
       } else {
         ASSERT(nData == pTColumn->bytes);
@@ -298,12 +299,11 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
       pBuilder->row.flags |= TSROW_HAS_NULL;
       pTColumn->flags |= COL_SET_NULL;
 
-      p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
-      ((SKVIdx *)p)->cid = cid;
-      ((SKVIdx *)p)->offset = -1;
+      pTSKVRow->idx[pTSKVRow->nCols].cid = cid;
+      pTSKVRow->idx[pTSKVRow->nCols].offset = -1;
     }
 
-    pBuilder->nCols++;
+    pTSKVRow->nCols++;
   }
 
   return 0;
@@ -319,17 +319,54 @@ static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) {
   }
   return 0;
 }
+static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) {
+  int32_t   bidx;
+  STColumn *pTColumn;
+
+  for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
+    pTColumn = &pTSchema->columns[iCol];
+    bidx = iCol - 1;
+
+    switch (flags) {
+      case TSROW_HAS_NULL | TSROW_HAS_NONE:
+        if (pTColumn->flags & COL_SET_NULL) {
+          SET_BIT1(p, bidx, (uint8_t)1);
+        } else {
+          SET_BIT1(p, bidx, (uint8_t)0);
+        }
+        break;
+      case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
+        if (pTColumn->flags & COL_SET_NULL) {
+          SET_BIT2(p, bidx, (uint8_t)1);
+        } else if (pTColumn->flags & COL_SET_NULL) {
+          SET_BIT2(p, bidx, (uint8_t)2);
+        } else {
+          SET_BIT2(p, bidx, (uint8_t)0);
+        }
+        break;
+      default:
+        if (pTColumn->flags & COL_SET_VAL) {
+          SET_BIT1(p, bidx, (uint8_t)1);
+        } else {
+          SET_BIT1(p, bidx, (uint8_t)0);
+        }
+
+        break;
+    }
+  }
+}
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
-  int32_t  nDataTP, nDataKV;
-  uint32_t flags;
+  int32_t   nDataTP, nDataKV;
+  uint32_t  flags;
+  STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf;
 
   // error not set ts
   if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) {
     return -1;
   }
 
-  ASSERT(pBuilder->nCols < pBuilder->pTSchema->numOfCols);
-  if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
+  ASSERT(pTSKVRow->nCols < pBuilder->pTSchema->numOfCols);
+  if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) {
     pBuilder->row.flags |= TSROW_HAS_NONE;
   }
 
@@ -338,7 +375,6 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   switch (pBuilder->row.flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      pBuilder->row.flags |= TSROW_KV_ROW;
       pBuilder->row.nData = 0;
       pBuilder->row.pData = NULL;
       return 0;
@@ -359,58 +395,37 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
       ASSERT(0);
   }
 
-  nDataKV = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->vlenKV;
-  ASSERT(pBuilder->row.flags & TSROW_KV_ROW == 0);
+  nDataKV = sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pBuilder->vlenKV;
+  pBuilder->row.sver = pBuilder->pTSchema->version;
   if (nDataKV < nDataTP) {
     // generate KV row
 
     pBuilder->row.flags |= TSROW_KV_ROW;
-    pBuilder->row.ncols = pBuilder->nCols;
     pBuilder->row.nData = nDataKV;
     pBuilder->row.pData = pBuilder->pKVBuf;
 
-    qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
-    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
-      memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
-              pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->vlenKV);
+    qsort(pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
+    if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) {
+      memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[pBuilder->pTSchema->numOfCols - 1], pBuilder->vlenKV);
     }
   } else {
     // generate TUPLE row
 
-    uint8_t  *p;
-    STColumn *pTColumn;
-    pBuilder->row.sver = pBuilder->pTSchema->version;
     pBuilder->row.nData = nDataTP;
-    if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL) {
-      // no bitmap
+
+    uint8_t *p;
+    uint8_t  flags = pBuilder->row.flags & 0xf;
+
+    if (flags == TSROW_HAS_VAL) {
       pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2;
-    } else if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) {
-      // bitmap2
-      p = pBuilder->pTPBuf;
-
-      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
-        pTColumn = &pBuilder->pTSchema->columns[iCol];
-
-        if (pTColumn->flags & COL_SET_VAL) {
-          SET_BIT2(p, iCol - 1, 0x2);
-        } else if (pTColumn->flags & COL_SET_VAL) {
-          SET_BIT2(p, iCol - 1, 0x1);
-        }
-      }
-
-      pBuilder->row.pData = p;
     } else {
-      // bitmap1
-      p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
-
-      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
-        pTColumn = &pBuilder->pTSchema->columns[iCol];
-
-        if (1) {
-          SET_BIT1(p, iCol - 1, 0x1);
-        }
+      if (flags == TSROW_HAS_VAL) {
+        p = pBuilder->pTPBuf;
+      } else {
+        p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
       }
 
+      setBitMap(p, pBuilder->pTSchema, flags);
       pBuilder->row.pData = p;
     }
   }

From 34a219b5fc5589ed45bbc6434d0138edb5e35233 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 14 May 2022 06:18:07 +0000
Subject: [PATCH 06/15] feat: impl tsrow read

---
 include/common/tdataformat.h    |  52 +++++++---
 source/common/src/tdataformat.c | 163 +++++++++++++++++++++-----------
 2 files changed, 150 insertions(+), 65 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index b8a4f6ac9d..ab0415d5d8 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -29,20 +29,41 @@ extern "C" {
 typedef struct SSchema       SSchema;
 typedef struct STColumn      STColumn;
 typedef struct STSchema      STSchema;
+typedef struct SColVal       SColVal;
 typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
-typedef struct SKVIdx        SKVIdx;
-
-// STSRow2
-int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
-int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
-int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
-                  int8_t *flags);
 
 // STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
 void    tTSchemaDestroy(STSchema *pTSchema);
 
+// SColVal
+#define COL_VAL_SET_NONE(CV) \
+  do {                       \
+    (CV)->type = COL_NONE;   \
+    (CV)->nData = 0;         \
+    (CV)->pData = NULL;      \
+  } while (0)
+
+#define COL_VAL_SET_NULL(CV) \
+  do {                       \
+    (CV)->type = COL_NULL;   \
+    (CV)->nData = 0;         \
+    (CV)->pData = NULL;      \
+  } while (0)
+
+#define COL_VAL_SET_VAL(CV, PDATA, NDATA) \
+  do {                                    \
+    (CV)->type = COL_VAL;                 \
+    (CV)->nData = (NDATA);                \
+    (CV)->pData = (PDATA);                \
+  } while (0)
+
+// STSRow2
+int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
+int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
+
 // STSRowBuilder
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols);
 void    tTSRowBuilderClear(STSRowBuilder *pBuilder);
@@ -73,11 +94,11 @@ struct STSchema {
 #define TSROW_HAS_VAL  ((uint8_t)0x4U)
 #define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY          ts;
-  uint8_t        flags;
-  int32_t        sver;
-  uint32_t       nData;
-  const uint8_t *pData;
+  TSKEY    ts;
+  uint8_t  flags;
+  int32_t  sver;
+  uint32_t nData;
+  uint8_t *pData;
 };
 
 struct STSRowBuilder {
@@ -94,6 +115,13 @@ struct STSRowBuilder {
   STSRow2   row;
 };
 
+typedef enum { COL_VAL = 0, COL_NONE = 1, COL_NULL = 2 } EColValT;
+struct SColVal {
+  EColValT type;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
 #if 1  //====================================
 // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
 #define TD_SUPPORT_BITMAP
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 055eb9df80..4b1df237da 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -19,10 +19,10 @@
 #include "tdatablock.h"
 #include "tlog.h"
 
-struct SKVIdx {
+typedef struct SKVIdx {
   int32_t cid;
   int32_t offset;
-};
+} SKVIdx;
 
 #pragma pack(push, 1)
 typedef struct {
@@ -34,6 +34,8 @@ typedef struct {
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
 #define SET_BIT1(p, i, v)  ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8)))
 #define SET_BIT2(p, i, v)  ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4)))
+#define GET_BIT1(p, i)     (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
+#define GET_BIT2(p, i)     (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3))
 
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
@@ -81,57 +83,112 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
   return 0;
 }
 
-int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
-                  int8_t *flags) {
-#if 0
-  if (cid == 0) {
-    *ppData = (uint8_t *)&pRow->ts;
-    *nData = sizeof(TSKEY);
-    *flags = 0;
-  } else {
-    uint32_t tflags = pRow->flags & 0xf;
-    *ppData = NULL;
-    *nData = 0;
+static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
+  col_id_t cid = *(col_id_t *)p1;
+  SKVIdx  *pKVIdx = (SKVIdx *)p2;
 
-    switch (tflags) {
-      case TSROW_HAS_NONE:
-        *flags = -1;
-        break;
-      case TSROW_HAS_NULL:
-        *flags = 1;
-        break;
-      case TSROW_HAS_VAL:
-        *flags = 0;
-        // find the row
-        break;
-      case TSROW_HAS_NULL | TSROW_HAS_NONE:
-        // read bit map (todo)
-        if (0) {
-          *flags = 1;
-        } else {
-          *flags = -1;
-        }
-        break;
-      case TSROW_HAS_VAL | TSROW_HAS_NONE:
-      case TSROW_HAS_VAL | TSROW_HAS_NULL:
-        // read bitmap (todo)
-        if (0) {
-          if (tflags & TSROW_HAS_NONE) {
-            *flags = -1;
-          } else {
-            *flags = 1;
-          }
-        } else {
-          // get value (todo)
-        }
-        break;
-      case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
-        break;
-      default:
-        return -1;
-    }
+  if (cid < pKVIdx->cid) {
+    return -1;
+  } else if (cid > pKVIdx->cid) {
+    return 1;
   }
-#endif
+  return 0;
+}
+
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
+  uint32_t  n;
+  uint8_t  *p;
+  uint8_t   v;
+  int32_t   bidx = iCol - 1;
+  STColumn *pTColumn = &pTSchema->columns[iCol];
+  STSKVRow *pTSKVRow;
+  SKVIdx   *pKVIdx;
+
+  ASSERT(pTColumn->colId != 0);
+
+  ASSERT(pRow->flags & 0xf != 0);
+  switch (pRow->flags & 0xf) {
+    case TSROW_HAS_NONE:
+      COL_VAL_SET_NONE(pColVal);
+      return 0;
+    case TSROW_HAS_NULL:
+      COL_VAL_SET_NULL(pColVal);
+      return 0;
+  }
+
+  if (TSROW_IS_KV_ROW(pRow)) {
+    ASSERT((pRow->flags & 0xf) != TSROW_HAS_VAL);
+
+    pTSKVRow = (STSKVRow *)pRow->pData;
+    pKVIdx = bsearch(&pTColumn->colId, pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), kvRowCmprFn);
+    if (pKVIdx == NULL) {
+      COL_VAL_SET_NONE(pColVal);
+    } else if (pKVIdx->offset < 0) {
+      COL_VAL_SET_NULL(pColVal);
+    } else {
+      p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset;
+      tGetBinary(p, &p, &n);
+      COL_VAL_SET_VAL(pColVal, p, n);
+    }
+  } else {
+    // get bitmap
+    switch (pRow->flags & 0xf) {
+      p = pRow->pData;
+      case TSROW_HAS_NULL | TSROW_HAS_NONE:
+        v = GET_BIT1(p, bidx);
+        if (v == 0) {
+          COL_VAL_SET_NONE(pColVal);
+        } else {
+          COL_VAL_SET_NULL(pColVal);
+        }
+        return 0;
+      case TSROW_HAS_VAL | TSROW_HAS_NONE:
+        v = GET_BIT1(p, bidx);
+        if (v == 1) {
+          p = p + (pTSchema->numOfCols - 2) / 8 + 1;
+          break;
+        } else {
+          COL_VAL_SET_NONE(pColVal);
+          return 0;
+        }
+      case TSROW_HAS_VAL | TSROW_HAS_NULL:
+        v = GET_BIT1(p, bidx);
+        if (v == 1) {
+          p = p + (pTSchema->numOfCols - 2) / 8 + 1;
+          break;
+        } else {
+          COL_VAL_SET_NULL(pColVal);
+          return 0;
+        }
+      case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
+        v = GET_BIT2(p, bidx);
+        if (v == 0) {
+          COL_VAL_SET_NONE(pColVal);
+          return 0;
+        } else if (v == 1) {
+          COL_VAL_SET_NULL(pColVal);
+          return 0;
+        } else if (v == 2) {
+          p = p + (pTSchema->numOfCols - 2) / 4 + 1;
+          break;
+        } else {
+          ASSERT(0);
+        }
+      default:
+        break;
+    }
+
+    // get real value
+    p = p + pTColumn->offset;
+    if (IS_VAR_DATA_TYPE(pTColumn->type)) {
+      p = pTSchema->flen + *(int32_t *)p;
+      tGetBinary(p, &p, &n);
+    } else {
+      n = pTColumn->bytes;
+    }
+    COL_VAL_SET_VAL(pColVal, p, n);
+  }
+
   return 0;
 }
 
@@ -445,8 +502,8 @@ int         tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   spaceNeeded += (int)nBitmapBytes;
   // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more
   // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered.
-  // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove
-  // the additional space
+  // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus
+  // remove the additional space
 #endif
 
   if (pCol->spaceSize < spaceNeeded) {

From acf1b14b21e1612097b4f3fd294f9cc7842ed218 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 16 May 2022 01:47:35 +0000
Subject: [PATCH 07/15] fix compile problem

---
 include/common/tdataformat.h    | 16 ++++++++--------
 source/common/src/tdataformat.c | 19 +++++++++----------
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index ab0415d5d8..1a31b384d8 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -94,11 +94,11 @@ struct STSchema {
 #define TSROW_HAS_VAL  ((uint8_t)0x4U)
 #define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY    ts;
-  uint8_t  flags;
-  int32_t  sver;
-  uint32_t nData;
-  uint8_t *pData;
+  TSKEY          ts;
+  uint8_t        flags;
+  int32_t        sver;
+  uint32_t       nData;
+  const uint8_t *pData;
 };
 
 struct STSRowBuilder {
@@ -117,9 +117,9 @@ struct STSRowBuilder {
 
 typedef enum { COL_VAL = 0, COL_NONE = 1, COL_NULL = 2 } EColValT;
 struct SColVal {
-  EColValT type;
-  uint32_t nData;
-  uint8_t *pData;
+  EColValT       type;
+  uint32_t       nData;
+  const uint8_t *pData;
 };
 
 #if 1  //====================================
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 4b1df237da..d421c452ca 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -96,13 +96,13 @@ static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
 }
 
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
-  uint32_t  n;
-  uint8_t  *p;
-  uint8_t   v;
-  int32_t   bidx = iCol - 1;
-  STColumn *pTColumn = &pTSchema->columns[iCol];
-  STSKVRow *pTSKVRow;
-  SKVIdx   *pKVIdx;
+  uint32_t       n;
+  const uint8_t *p;
+  uint8_t        v;
+  int32_t        bidx = iCol - 1;
+  STColumn      *pTColumn = &pTSchema->columns[iCol];
+  STSKVRow      *pTSKVRow;
+  SKVIdx        *pKVIdx;
 
   ASSERT(pTColumn->colId != 0);
 
@@ -132,8 +132,8 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     }
   } else {
     // get bitmap
+    p = pRow->pData;
     switch (pRow->flags & 0xf) {
-      p = pRow->pData;
       case TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT1(p, bidx);
         if (v == 0) {
@@ -181,8 +181,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     // get real value
     p = p + pTColumn->offset;
     if (IS_VAR_DATA_TYPE(pTColumn->type)) {
-      p = pTSchema->flen + *(int32_t *)p;
-      tGetBinary(p, &p, &n);
+      tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n);
     } else {
       n = pTColumn->bytes;
     }

From de5828e4856c8d02251ea1b6c8598cd2113d7c11 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 16 May 2022 11:35:37 +0000
Subject: [PATCH 08/15] refact more

---
 source/common/test/dataformatTest.cpp |  0
 source/common/type/type.c             | 14 --------------
 source/common/type/typeBigint.c       | 14 --------------
 source/common/type/typeBinary.c       | 14 --------------
 source/common/type/typeBlob.c         | 14 --------------
 source/common/type/typeBool.c         | 14 --------------
 source/common/type/typeDecimal.c      | 14 --------------
 source/common/type/typeDouble.c       | 14 --------------
 source/common/type/typeFloat.c        | 14 --------------
 source/common/type/typeInt.c          | 14 --------------
 source/common/type/typeJson.c         | 14 --------------
 source/common/type/typeLongblob.c     | 14 --------------
 source/common/type/typeNchar.c        | 14 --------------
 source/common/type/typeNull.c         | 14 --------------
 source/common/type/typeSmallint.c     | 14 --------------
 source/common/type/typeTimestamp.c    | 14 --------------
 source/common/type/typeTinyint.c      | 14 --------------
 source/common/type/typeUBigint.c      | 14 --------------
 source/common/type/typeUSmallint.c    | 14 --------------
 source/common/type/typeUTinyint.c     | 14 --------------
 source/common/type/typeUint.c         | 14 --------------
 source/common/type/typeVarchar.c      | 14 --------------
 22 files changed, 294 deletions(-)
 create mode 100644 source/common/test/dataformatTest.cpp
 delete mode 100644 source/common/type/type.c
 delete mode 100644 source/common/type/typeBigint.c
 delete mode 100644 source/common/type/typeBinary.c
 delete mode 100644 source/common/type/typeBlob.c
 delete mode 100644 source/common/type/typeBool.c
 delete mode 100644 source/common/type/typeDecimal.c
 delete mode 100644 source/common/type/typeDouble.c
 delete mode 100644 source/common/type/typeFloat.c
 delete mode 100644 source/common/type/typeInt.c
 delete mode 100644 source/common/type/typeJson.c
 delete mode 100644 source/common/type/typeLongblob.c
 delete mode 100644 source/common/type/typeNchar.c
 delete mode 100644 source/common/type/typeNull.c
 delete mode 100644 source/common/type/typeSmallint.c
 delete mode 100644 source/common/type/typeTimestamp.c
 delete mode 100644 source/common/type/typeTinyint.c
 delete mode 100644 source/common/type/typeUBigint.c
 delete mode 100644 source/common/type/typeUSmallint.c
 delete mode 100644 source/common/type/typeUTinyint.c
 delete mode 100644 source/common/type/typeUint.c
 delete mode 100644 source/common/type/typeVarchar.c

diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/source/common/type/type.c b/source/common/type/type.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/type.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeBigint.c b/source/common/type/typeBigint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeBigint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeBinary.c b/source/common/type/typeBinary.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeBinary.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeBlob.c b/source/common/type/typeBlob.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeBlob.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeBool.c b/source/common/type/typeBool.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeBool.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeDecimal.c b/source/common/type/typeDecimal.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeDecimal.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeDouble.c b/source/common/type/typeDouble.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeDouble.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeFloat.c b/source/common/type/typeFloat.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeFloat.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeInt.c b/source/common/type/typeInt.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeInt.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeJson.c b/source/common/type/typeJson.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeJson.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeLongblob.c b/source/common/type/typeLongblob.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeLongblob.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeNchar.c b/source/common/type/typeNchar.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeNchar.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeNull.c b/source/common/type/typeNull.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeNull.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeSmallint.c b/source/common/type/typeSmallint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeSmallint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeTimestamp.c b/source/common/type/typeTimestamp.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeTimestamp.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeTinyint.c b/source/common/type/typeTinyint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeTinyint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeUBigint.c b/source/common/type/typeUBigint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeUBigint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeUSmallint.c b/source/common/type/typeUSmallint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeUSmallint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeUTinyint.c b/source/common/type/typeUTinyint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeUTinyint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeUint.c b/source/common/type/typeUint.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeUint.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/common/type/typeVarchar.c b/source/common/type/typeVarchar.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/common/type/typeVarchar.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file

From 96eecc4d3ee21be96f8641b6fd9454acdcd178ed Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 16 May 2022 11:43:48 +0000
Subject: [PATCH 09/15] more refact

---
 source/common/test/CMakeLists.txt     |  9 +++++++++
 source/common/test/dataformatTest.cpp |  1 +
 source/common/test/trowTest.cpp       | 23 -----------------------
 3 files changed, 10 insertions(+), 23 deletions(-)
 delete mode 100644 source/common/test/trowTest.cpp

diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt
index a0406e099c..0535b08be7 100644
--- a/source/common/test/CMakeLists.txt
+++ b/source/common/test/CMakeLists.txt
@@ -17,6 +17,15 @@ TARGET_INCLUDE_DIRECTORIES(
         PRIVATE "${TD_SOURCE_DIR}/source/libs/common/inc"
 )
 
+# dataformatTest.cpp 
+add_executable(dataformatTest "")
+target_sources(
+    dataformatTest
+    PRIVATE
+    "dataformatTest.cpp"
+)
+target_link_libraries(dataformatTest gtest gtest_main util)
+
 # tmsg test
 # add_executable(tmsgTest "")
 # target_sources(tmsgTest 
diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp
index e69de29bb2..3497014c22 100644
--- a/source/common/test/dataformatTest.cpp
+++ b/source/common/test/dataformatTest.cpp
@@ -0,0 +1 @@
+#include "gtest/gtest.h"
\ No newline at end of file
diff --git a/source/common/test/trowTest.cpp b/source/common/test/trowTest.cpp
deleted file mode 100644
index d7f0783d4a..0000000000
--- a/source/common/test/trowTest.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-#include <gtest/gtest.h>
-
-#include "trow.h"
-
-TEST(td_row_test, build_row_to_target) {
-#if 0
-  char        dst[1024];
-  SRow*       pRow = (SRow*)dst;
-  int         ncols = 10;
-  col_id_t    cid;
-  void*       pData;
-  SRowBuilder rb = trbInit(TD_OR_ROW_BUILDER, NULL, 0, pRow, 1024);
-
-  trbSetRowInfo(&rb, false, 0);
-  trbSetRowTS(&rb, 1637550210000);
-  for (int c = 0; c < ncols; c++) {
-    cid = c;
-    if (trbWriteCol(&rb, pData, cid) < 0) {
-      // TODO
-    }
-  }
-#endif
-}
\ No newline at end of file

From 93c095b1a56e515be082a7b5ff1b6fed4aa9f58d Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 05:46:22 +0000
Subject: [PATCH 10/15] make compile

---
 include/common/tdataformat.h    |   4 +-
 include/util/tencode.h          | 205 ++++++++++++++++++++++++++------
 source/common/src/tdataformat.c |   6 +-
 3 files changed, 177 insertions(+), 38 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 1a31b384d8..1097f62798 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -62,13 +62,15 @@ void    tTSchemaDestroy(STSchema *pTSchema);
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
+int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
+int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 
 // STSRowBuilder
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols);
 void    tTSRowBuilderClear(STSRowBuilder *pBuilder);
 void    tTSRowBuilderReset(STSRowBuilder *pBuilder);
-int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData);
+int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData);
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow);
 
 // STRUCT =================
diff --git a/include/util/tencode.h b/include/util/tencode.h
index e49429f865..484c9443f7 100644
--- a/include/util/tencode.h
+++ b/include/util/tencode.h
@@ -456,52 +456,189 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) {
   return p;
 }
 
-static FORCE_INLINE int32_t tPutBinary(uint8_t* p, const uint8_t* pData, uint32_t nData) {
-  int      n = 0;
-  uint32_t v = nData;
+// ===========================================
+#define tPutV(p, v)                    \
+  do {                                 \
+    int32_t n = 0;                     \
+    for (;;) {                         \
+      if (v <= 0x7f) {                 \
+        if (p) p[n] = v;               \
+        n++;                           \
+        break;                         \
+      }                                \
+      if (p) p[n] = (v & 0x7f) | 0x80; \
+      n++;                             \
+      v >>= 7;                         \
+    }                                  \
+    return n;                          \
+  } while (0)
 
-  for (;;) {
-    if (v <= 0x7f) {
-      if (p) p[n] = v;
-      n++;
-      break;
-    }
+#define tGetV(p, v)                              \
+  do {                                           \
+    int32_t n = 0;                               \
+    if (v) *v = 0;                               \
+    for (;;) {                                   \
+      if (p[n] <= 0x7f) {                        \
+        if (v) (*v) |= (p[n] << (7 * n));        \
+        n++;                                     \
+        break;                                   \
+      }                                          \
+      if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \
+      n++;                                       \
+    }                                            \
+    return n;                                    \
+  } while (0)
 
-    if (p) p[n] = (v & 0x7f) | 0x80;
-    n++;
-    v >>= 7;
-  }
+// PUT
+static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) {
+  if (p) ((uint8_t*)p)[0] = v;
+  return sizeof(uint8_t);
+}
 
-  if (p) {
-    memcpy(p + n, pData, nData);
-  }
+static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) {
+  if (p) ((int8_t*)p)[0] = v;
+  return sizeof(int8_t);
+}
+
+static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) {
+  if (p) ((uint16_t*)p)[0] = v;
+  return sizeof(uint16_t);
+}
+
+static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) {
+  if (p) ((int16_t*)p)[0] = v;
+  return sizeof(int16_t);
+}
+
+static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) {
+  if (p) ((uint32_t*)p)[0] = v;
+  return sizeof(uint32_t);
+}
+
+static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) {
+  if (p) ((int32_t*)p)[0] = v;
+  return sizeof(int32_t);
+}
+
+static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) {
+  if (p) ((uint64_t*)p)[0] = v;
+  return sizeof(uint64_t);
+}
+
+static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) {
+  if (p) ((int64_t*)p)[0] = v;
+  return sizeof(int64_t);
+}
+
+static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); }
+
+static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); }
+
+static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); }
+
+// GET
+static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) {
+  if (v) *v = ((uint8_t*)p)[0];
+  return sizeof(uint8_t);
+}
+
+static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) {
+  if (v) *v = ((int8_t*)p)[0];
+  return sizeof(int8_t);
+}
+
+static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) {
+  if (v) *v = ((uint16_t*)p)[0];
+  return sizeof(uint16_t);
+}
+
+static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) {
+  if (v) *v = ((int16_t*)p)[0];
+  return sizeof(int16_t);
+}
+
+static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) {
+  if (v) *v = ((uint32_t*)p)[0];
+  return sizeof(uint32_t);
+}
+
+static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) {
+  if (v) *v = ((int32_t*)p)[0];
+  return sizeof(int32_t);
+}
+
+static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) {
+  if (v) *v = ((uint64_t*)p)[0];
+  return sizeof(uint64_t);
+}
+
+static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) {
+  if (v) *v = ((int64_t*)p)[0];
+  return sizeof(int64_t);
+}
+
+static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) {
+  int32_t  n;
+  uint16_t tv;
+
+  n = tGetU16v(p, &tv);
+  if (v) *v = ZIGZAGD(int16_t, tv);
+
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) {
+  int32_t  n;
+  uint32_t tv;
+
+  n = tGetU32v(p, &tv);
+  if (v) *v = ZIGZAGD(int32_t, tv);
+
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) {
+  int32_t  n;
+  uint64_t tv;
+
+  n = tGetU64v(p, &tv);
+  if (v) *v = ZIGZAGD(int64_t, tv);
+
+  return n;
+}
+
+// =====================
+static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) {
+  int n = 0;
+
+  n += tPutU32v(p ? p + n : p, nData);
+  if (p) memcpy(p + n, pData, nData);
   n += nData;
 
   return n;
 }
 
-static FORCE_INLINE int32_t tGetBinary(const uint8_t* p, const uint8_t** ppData, uint32_t* nData) {
+static FORCE_INLINE int32_t tGetBinary(uint8_t* p, const uint8_t** ppData, uint32_t* nData) {
   int32_t  n = 0;
-  uint32_t tv = 0;
-  uint32_t t;
+  uint32_t nt;
 
-  for (;;) {
-    if (p[n] <= 0x7f) {
-      t = p[n];
-      tv |= (t << (7 * n));
-      n++;
-      break;
-    }
-
-    t = p[n] & 0x7f;
-    tv |= (t << (7 * n));
-    n++;
-  }
-
-  if (nData) *nData = n;
+  n += tGetU32v(p, &nt);
+  if (nData) *nData = nt;
   if (ppData) *ppData = p + n;
+  n += nt;
 
-  n += tv;
   return n;
 }
 
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index d421c452ca..a37d8931ad 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -127,7 +127,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
       COL_VAL_SET_NULL(pColVal);
     } else {
       p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset;
-      tGetBinary(p, &p, &n);
+      // tGetBinary(p, &p, &n); (todo)
       COL_VAL_SET_VAL(pColVal, p, n);
     }
   } else {
@@ -181,7 +181,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     // get real value
     p = p + pTColumn->offset;
     if (IS_VAR_DATA_TYPE(pTColumn->type)) {
-      tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n);
+      // tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n); (todo)
     } else {
       n = pTColumn->bytes;
     }
@@ -283,7 +283,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
   pBuilder->row.flags = 0;
 }
 
-int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) {
+int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData) {
   STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol];
   uint8_t  *p;
   int32_t   iCol;

From 4568c672d643e5ff72fed338d91c9d9d5bac3456 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 06:40:37 +0000
Subject: [PATCH 11/15] feat: tdata

---
 include/common/tdataformat.h    | 43 ++++++--------------
 include/util/tencode.h          |  2 +-
 source/common/src/tdataformat.c | 69 ++++++++++++++++-----------------
 3 files changed, 46 insertions(+), 68 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 1097f62798..bf2ddc0920 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -38,30 +38,11 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema *
 void    tTSchemaDestroy(STSchema *pTSchema);
 
 // SColVal
-#define COL_VAL_SET_NONE(CV) \
-  do {                       \
-    (CV)->type = COL_NONE;   \
-    (CV)->nData = 0;         \
-    (CV)->pData = NULL;      \
-  } while (0)
-
-#define COL_VAL_SET_NULL(CV) \
-  do {                       \
-    (CV)->type = COL_NULL;   \
-    (CV)->nData = 0;         \
-    (CV)->pData = NULL;      \
-  } while (0)
-
-#define COL_VAL_SET_VAL(CV, PDATA, NDATA) \
-  do {                                    \
-    (CV)->type = COL_VAL;                 \
-    (CV)->nData = (NDATA);                \
-    (CV)->pData = (PDATA);                \
-  } while (0)
+#define ColValNONE               ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL})
+#define ColValNULL               ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL})
+#define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)})
 
 // STSRow2
-int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
-int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
@@ -96,11 +77,11 @@ struct STSchema {
 #define TSROW_HAS_VAL  ((uint8_t)0x4U)
 #define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
-  TSKEY          ts;
-  uint8_t        flags;
-  int32_t        sver;
-  uint32_t       nData;
-  const uint8_t *pData;
+  TSKEY    ts;
+  uint8_t  flags;
+  int32_t  sver;
+  uint32_t nData;
+  uint8_t *pData;
 };
 
 struct STSRowBuilder {
@@ -117,11 +98,11 @@ struct STSRowBuilder {
   STSRow2   row;
 };
 
-typedef enum { COL_VAL = 0, COL_NONE = 1, COL_NULL = 2 } EColValT;
+typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_Data = 2 } EColValT;
 struct SColVal {
-  EColValT       type;
-  uint32_t       nData;
-  const uint8_t *pData;
+  EColValT type;
+  uint32_t nData;
+  uint8_t *pData;
 };
 
 #if 1  //====================================
diff --git a/include/util/tencode.h b/include/util/tencode.h
index 484c9443f7..938e3018a8 100644
--- a/include/util/tencode.h
+++ b/include/util/tencode.h
@@ -630,7 +630,7 @@ static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nDat
   return n;
 }
 
-static FORCE_INLINE int32_t tGetBinary(uint8_t* p, const uint8_t** ppData, uint32_t* nData) {
+static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData) {
   int32_t  n = 0;
   uint32_t nt;
 
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index a37d8931ad..fbc2c2306d 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -38,49 +38,46 @@ typedef struct {
 #define GET_BIT2(p, i)     (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3))
 
 // STSRow2
-int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
-  if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
-  if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1;
-  if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1;
+int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) {
+  int32_t n = 0;
 
-  ASSERT(pRow->flags & 0xf != 0);
+  n += tPutI64(p ? p + n : p, pRow->ts);
+  n += tPutI8(p ? p + n : p, pRow->flags);
+  n += tPutI32v(p ? p + n : p, pRow->sver);
+
+  ASSERT(pRow->flags & 0xf);
 
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      return 0;
-    case TSROW_HAS_VAL:
-      ASSERT(!TSROW_IS_KV_ROW(pRow));
+      break;
     default:
-      ASSERT(pRow->nData && pRow->pData);
-      if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1;
+      n += tPutBinary(p ? p + n : p, pRow->pData, pRow->nData);
       break;
   }
 
-  return 0;
+  return n;
 }
 
-int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
-  if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1;
-  if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1;
+int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) {
+  int32_t n = 0;
+  uint8_t flags;
 
-  ASSERT(pRow->flags & 0xf != 0);
+  n += tGetI64(p + n, pRow ? &pRow->ts : NULL);
+  n += tGetI8(p + n, pRow ? &pRow->flags : &flags);
+  n += tGetI32v(p + n, pRow ? &pRow->sver : NULL);
 
-  switch (pRow->flags & 0xf) {
+  if (pRow) flags = pRow->flags;
+  switch (flags & 0xf) {
     case TSROW_HAS_NONE:
     case TSROW_HAS_NULL:
-      pRow->nData = 0;
-      pRow->pData = NULL;
-      return 0;
-    case TSROW_HAS_VAL:
-      ASSERT(!TSROW_IS_KV_ROW(pRow));
+      break;
     default:
-      if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1;
+      n += tGetBinary(p + n, pRow ? &pRow->pData : NULL, pRow ? &pRow->nData : NULL);
       break;
   }
 
-  return 0;
+  return n;
 }
 
 static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
@@ -109,10 +106,10 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
   ASSERT(pRow->flags & 0xf != 0);
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
-      COL_VAL_SET_NONE(pColVal);
+      // COL_VAL_SET_NONE(pColVal);
       return 0;
     case TSROW_HAS_NULL:
-      COL_VAL_SET_NULL(pColVal);
+      // COL_VAL_SET_NULL(pColVal);
       return 0;
   }
 
@@ -122,13 +119,13 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     pTSKVRow = (STSKVRow *)pRow->pData;
     pKVIdx = bsearch(&pTColumn->colId, pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), kvRowCmprFn);
     if (pKVIdx == NULL) {
-      COL_VAL_SET_NONE(pColVal);
+      // COL_VAL_SET_NONE(pColVal);
     } else if (pKVIdx->offset < 0) {
-      COL_VAL_SET_NULL(pColVal);
+      // COL_VAL_SET_NULL(pColVal);
     } else {
       p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset;
       // tGetBinary(p, &p, &n); (todo)
-      COL_VAL_SET_VAL(pColVal, p, n);
+      // COL_VAL_SET_VAL(pColVal, p, n);
     }
   } else {
     // get bitmap
@@ -137,9 +134,9 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
       case TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT1(p, bidx);
         if (v == 0) {
-          COL_VAL_SET_NONE(pColVal);
+          // COL_VAL_SET_NONE(pColVal);
         } else {
-          COL_VAL_SET_NULL(pColVal);
+          // COL_VAL_SET_NULL(pColVal);
         }
         return 0;
       case TSROW_HAS_VAL | TSROW_HAS_NONE:
@@ -148,7 +145,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
           p = p + (pTSchema->numOfCols - 2) / 8 + 1;
           break;
         } else {
-          COL_VAL_SET_NONE(pColVal);
+          // COL_VAL_SET_NONE(pColVal);
           return 0;
         }
       case TSROW_HAS_VAL | TSROW_HAS_NULL:
@@ -157,16 +154,16 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
           p = p + (pTSchema->numOfCols - 2) / 8 + 1;
           break;
         } else {
-          COL_VAL_SET_NULL(pColVal);
+          // COL_VAL_SET_NULL(pColVal);
           return 0;
         }
       case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT2(p, bidx);
         if (v == 0) {
-          COL_VAL_SET_NONE(pColVal);
+          // COL_VAL_SET_NONE(pColVal);
           return 0;
         } else if (v == 1) {
-          COL_VAL_SET_NULL(pColVal);
+          // COL_VAL_SET_NULL(pColVal);
           return 0;
         } else if (v == 2) {
           p = p + (pTSchema->numOfCols - 2) / 4 + 1;
@@ -185,7 +182,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     } else {
       n = pTColumn->bytes;
     }
-    COL_VAL_SET_VAL(pColVal, p, n);
+    // COL_VAL_SET_VAL(pColVal, p, n);
   }
 
   return 0;

From af8ac6e1b08704e77c0d558b02bf4a85529951b5 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 07:54:38 +0000
Subject: [PATCH 12/15] more tdata

---
 include/common/tdataformat.h    |  4 +-
 source/common/src/tdataformat.c | 65 ++++++++++++++++++++++++---------
 2 files changed, 50 insertions(+), 19 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index bf2ddc0920..af035d0d1f 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -45,10 +45,12 @@ void    tTSchemaDestroy(STSchema *pTSchema);
 // STSRow2
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
+int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow);
+void    tTSRowFree(STSRow2 *pRow);
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 
 // STSRowBuilder
-int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols);
+int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema);
 void    tTSRowBuilderClear(STSRowBuilder *pBuilder);
 void    tTSRowBuilderReset(STSRowBuilder *pBuilder);
 int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData);
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index fbc2c2306d..3f179ac58a 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -32,6 +32,8 @@ typedef struct {
 #pragma pack(pop)
 
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
+#define BIT1_SIZE(n)       (((n)-1) / 8 + 1)
+#define BIT2_SIZE(n)       (((n)-1) / 4 + 1)
 #define SET_BIT1(p, i, v)  ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8)))
 #define SET_BIT2(p, i, v)  ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4)))
 #define GET_BIT1(p, i)     (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
@@ -80,6 +82,31 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) {
   return n;
 }
 
+int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow) {
+  (*ppRow) = taosMemoryMalloc(sizeof(*pRow) + pRow->nData);
+  if (*ppRow == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  (*ppRow)->ts = pRow->ts;
+  (*ppRow)->flags = pRow->flags;
+  (*ppRow)->sver = pRow->sver;
+  (*ppRow)->nData = pRow->nData;
+  if (pRow->nData) {
+    (*ppRow)->pData = (uint8_t *)(&(*ppRow)[1]);
+    memcpy((*ppRow)->pData, pRow->pData, pRow->nData);
+  } else {
+    (*ppRow)->pData = NULL;
+  }
+
+  return 0;
+}
+
+void tTSRowFree(STSRow2 *pRow) {
+  if (pRow) taosMemoryFree(pRow);
+}
+
 static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
   col_id_t cid = *(col_id_t *)p1;
   SKVIdx  *pKVIdx = (SKVIdx *)p2;
@@ -229,11 +256,11 @@ void tTSchemaDestroy(STSchema *pTSchema) {
 }
 
 // STSRowBuilder
-int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
+int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema) {
   if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1;
 
-  pBuilder->szBitMap1 = (nCols - 2) / 8 + 1;
-  pBuilder->szBitMap2 = (nCols - 2) / 4 + 1;
+  pBuilder->szBitMap1 = BIT1_SIZE(nCols - 1);
+  pBuilder->szBitMap2 = BIT2_SIZE(nCols - 1);
   pBuilder->szKVBuf =
       sizeof(STSKVRow) + sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
   pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen;
@@ -286,16 +313,16 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, u
   int32_t   iCol;
   STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf;
 
-  // use interp search (todo)
-  if (pTColumn->colId > cid) {
+  // use interp search
+  if (pTColumn->colId < cid) {  // right search
     for (iCol = pBuilder->iCol + 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
       pTColumn = &pBuilder->pTSchema->columns[iCol];
-      if (pTColumn->colId == cid) break;
+      if (pTColumn->colId >= cid) break;
     }
-  } else if (pTColumn->colId < cid) {
+  } else if (pTColumn->colId > cid) {  // left search
     for (iCol = pBuilder->iCol - 1; iCol >= 0; iCol--) {
       pTColumn = &pBuilder->pTSchema->columns[iCol];
-      if (pTColumn->colId == cid) break;
+      if (pTColumn->colId <= cid) break;
     }
   }
 
@@ -391,7 +418,7 @@ static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) {
       case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
         if (pTColumn->flags & COL_SET_NULL) {
           SET_BIT2(p, bidx, (uint8_t)1);
-        } else if (pTColumn->flags & COL_SET_NULL) {
+        } else if (pTColumn->flags & COL_SET_VAL) {
           SET_BIT2(p, bidx, (uint8_t)2);
         } else {
           SET_BIT2(p, bidx, (uint8_t)0);
@@ -412,14 +439,15 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   int32_t   nDataTP, nDataKV;
   uint32_t  flags;
   STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf;
+  int32_t   nCols = pBuilder->pTSchema->numOfCols;
 
   // error not set ts
   if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) {
     return -1;
   }
 
-  ASSERT(pTSKVRow->nCols < pBuilder->pTSchema->numOfCols);
-  if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) {
+  ASSERT(pTSKVRow->nCols < nCols);
+  if (pTSKVRow->nCols < nCols - 1) {
     pBuilder->row.flags |= TSROW_HAS_NONE;
   }
 
@@ -453,13 +481,15 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   if (nDataKV < nDataTP) {
     // generate KV row
 
+    ASSERT(pBuilder->row.flags & 0xf != TSROW_HAS_VAL);
+
     pBuilder->row.flags |= TSROW_KV_ROW;
     pBuilder->row.nData = nDataKV;
     pBuilder->row.pData = pBuilder->pKVBuf;
 
     qsort(pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
-    if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) {
-      memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[pBuilder->pTSchema->numOfCols - 1], pBuilder->vlenKV);
+    if (pTSKVRow->nCols < nCols - 1) {
+      memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[nCols - 1], pBuilder->vlenKV);
     }
   } else {
     // generate TUPLE row
@@ -472,14 +502,13 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
     if (flags == TSROW_HAS_VAL) {
       pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2;
     } else {
-      if (flags == TSROW_HAS_VAL) {
-        p = pBuilder->pTPBuf;
+      if (flags == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) {
+        pBuilder->row.pData = pBuilder->pTPBuf;
       } else {
-        p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
+        pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1;
       }
 
-      setBitMap(p, pBuilder->pTSchema, flags);
-      pBuilder->row.pData = p;
+      setBitMap(pBuilder->row.pData, pBuilder->pTSchema, flags);
     }
   }
 

From aaf78935b9a1468b6e093f126c0aed4c07a5b781 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 08:13:58 +0000
Subject: [PATCH 13/15] more tdata

---
 include/common/tdataformat.h    |  2 +-
 source/common/src/tdataformat.c | 55 ++++++++++++++-------------------
 2 files changed, 25 insertions(+), 32 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index af035d0d1f..7e411cb301 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -100,7 +100,7 @@ struct STSRowBuilder {
   STSRow2   row;
 };
 
-typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_Data = 2 } EColValT;
+typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT;
 struct SColVal {
   EColValT type;
   uint32_t nData;
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 3f179ac58a..a5447c016f 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -39,6 +39,8 @@ typedef struct {
 #define GET_BIT1(p, i)     (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
 #define GET_BIT2(p, i)     (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3))
 
+static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2);
+
 // STSRow2
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) {
   int32_t n = 0;
@@ -107,18 +109,6 @@ void tTSRowFree(STSRow2 *pRow) {
   if (pRow) taosMemoryFree(pRow);
 }
 
-static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) {
-  col_id_t cid = *(col_id_t *)p1;
-  SKVIdx  *pKVIdx = (SKVIdx *)p2;
-
-  if (cid < pKVIdx->cid) {
-    return -1;
-  } else if (cid > pKVIdx->cid) {
-    return 1;
-  }
-  return 0;
-}
-
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
   uint32_t       n;
   const uint8_t *p;
@@ -128,15 +118,16 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
   STSKVRow      *pTSKVRow;
   SKVIdx        *pKVIdx;
 
+  ASSERT(iCol != 0);
   ASSERT(pTColumn->colId != 0);
 
   ASSERT(pRow->flags & 0xf != 0);
   switch (pRow->flags & 0xf) {
     case TSROW_HAS_NONE:
-      // COL_VAL_SET_NONE(pColVal);
+      *pColVal = ColValNONE;
       return 0;
     case TSROW_HAS_NULL:
-      // COL_VAL_SET_NULL(pColVal);
+      *pColVal = ColValNULL;
       return 0;
   }
 
@@ -144,15 +135,16 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
     ASSERT((pRow->flags & 0xf) != TSROW_HAS_VAL);
 
     pTSKVRow = (STSKVRow *)pRow->pData;
-    pKVIdx = bsearch(&pTColumn->colId, pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), kvRowCmprFn);
+    pKVIdx =
+        bsearch(&((SKVIdx){.cid = pTColumn->colId}), pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn);
     if (pKVIdx == NULL) {
-      // COL_VAL_SET_NONE(pColVal);
+      *pColVal = ColValNONE;
     } else if (pKVIdx->offset < 0) {
-      // COL_VAL_SET_NULL(pColVal);
+      *pColVal = ColValNULL;
     } else {
       p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset;
-      // tGetBinary(p, &p, &n); (todo)
-      // COL_VAL_SET_VAL(pColVal, p, n);
+      pColVal->type = COL_VAL_DATA;
+      tGetBinary(p, &pColVal->pData, &pColVal->nData);
     }
   } else {
     // get bitmap
@@ -161,39 +153,39 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
       case TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT1(p, bidx);
         if (v == 0) {
-          // COL_VAL_SET_NONE(pColVal);
+          *pColVal = ColValNONE;
         } else {
-          // COL_VAL_SET_NULL(pColVal);
+          *pColVal = ColValNULL;
         }
         return 0;
       case TSROW_HAS_VAL | TSROW_HAS_NONE:
         v = GET_BIT1(p, bidx);
         if (v == 1) {
-          p = p + (pTSchema->numOfCols - 2) / 8 + 1;
+          p = p + BIT1_SIZE(pTSchema->numOfCols - 1);
           break;
         } else {
-          // COL_VAL_SET_NONE(pColVal);
+          *pColVal = ColValNONE;
           return 0;
         }
       case TSROW_HAS_VAL | TSROW_HAS_NULL:
         v = GET_BIT1(p, bidx);
         if (v == 1) {
-          p = p + (pTSchema->numOfCols - 2) / 8 + 1;
+          p = p + BIT1_SIZE(pTSchema->numOfCols - 1);
           break;
         } else {
-          // COL_VAL_SET_NULL(pColVal);
+          *pColVal = ColValNULL;
           return 0;
         }
       case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE:
         v = GET_BIT2(p, bidx);
         if (v == 0) {
-          // COL_VAL_SET_NONE(pColVal);
+          *pColVal = ColValNONE;
           return 0;
         } else if (v == 1) {
-          // COL_VAL_SET_NULL(pColVal);
+          *pColVal = ColValNULL;
           return 0;
         } else if (v == 2) {
-          p = p + (pTSchema->numOfCols - 2) / 4 + 1;
+          p = p + BIT2_SIZE(pTSchema->numOfCols - 1);
           break;
         } else {
           ASSERT(0);
@@ -204,12 +196,13 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal
 
     // get real value
     p = p + pTColumn->offset;
+    pColVal->type = COL_VAL_DATA;
     if (IS_VAR_DATA_TYPE(pTColumn->type)) {
-      // tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n); (todo)
+      tGetBinary(p + pTSchema->flen + *(int32_t *)p, &pColVal->pData, &pColVal->nData);
     } else {
-      n = pTColumn->bytes;
+      pColVal->pData = p;
+      pColVal->nData = pTColumn->bytes;
     }
-    // COL_VAL_SET_VAL(pColVal, p, n);
   }
 
   return 0;

From d6521a0a52a47c3a4257dd7957e483316877bce8 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 08:16:23 +0000
Subject: [PATCH 14/15] more tdata

---
 source/common/src/tdataformat.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index a5447c016f..4b9531aa6b 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -110,13 +110,13 @@ void tTSRowFree(STSRow2 *pRow) {
 }
 
 int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
-  uint32_t       n;
-  const uint8_t *p;
-  uint8_t        v;
-  int32_t        bidx = iCol - 1;
-  STColumn      *pTColumn = &pTSchema->columns[iCol];
-  STSKVRow      *pTSKVRow;
-  SKVIdx        *pKVIdx;
+  uint32_t  n;
+  uint8_t  *p;
+  uint8_t   v;
+  int32_t   bidx = iCol - 1;
+  STColumn *pTColumn = &pTSchema->columns[iCol];
+  STSKVRow *pTSKVRow;
+  SKVIdx   *pKVIdx;
 
   ASSERT(iCol != 0);
   ASSERT(pTColumn->colId != 0);

From 3a851ca9f29b09cc1d2e2b37e045f8b7cad61558 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 17 May 2022 09:50:21 +0000
Subject: [PATCH 15/15] more tdata

---
 include/common/tdataformat.h    |  18 +++++-
 source/common/src/tdataformat.c | 104 +++++++++++++++++++++++++++++++-
 2 files changed, 120 insertions(+), 2 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 7e411cb301..e13705d403 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -32,6 +32,8 @@ typedef struct STSchema      STSchema;
 typedef struct SColVal       SColVal;
 typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
+typedef struct STagVal       STagVal;
+typedef struct STag          STag;
 
 // STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
@@ -56,6 +58,13 @@ void    tTSRowBuilderReset(STSRowBuilder *pBuilder);
 int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData);
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow);
 
+// STag
+int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag);
+void    tTagFree(STag *pTag);
+void    tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData);
+int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag);
+int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag);
+
 // STRUCT =================
 struct STColumn {
   col_id_t colId;
@@ -107,7 +116,14 @@ struct SColVal {
   uint8_t *pData;
 };
 
-#if 1  //====================================
+struct STagVal {
+  int16_t  cid;
+  int8_t   type;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
+#if 1  //================================================================================================================================================
 // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
 #define TD_SUPPORT_BITMAP
 #define TD_SUPPORT_READ2
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 4b9531aa6b..8aa8ed2f14 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -31,6 +31,19 @@ typedef struct {
 } STSKVRow;
 #pragma pack(pop)
 
+typedef struct STagIdx {
+  int16_t  cid;
+  uint16_t offset;
+} STagIdx;
+
+#pragma pack(push, 1)
+struct STag {
+  uint16_t len;
+  uint16_t nTag;
+  STagIdx  idx[];
+};
+#pragma pack(pop)
+
 #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
 #define BIT1_SIZE(n)       (((n)-1) / 8 + 1)
 #define BIT2_SIZE(n)       (((n)-1) / 4 + 1)
@@ -508,7 +521,96 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
   return 0;
 }
 
-#if 1  // ====================
+static FORCE_INLINE int tTagIdxCmprFn(const void *p1, const void *p2) {
+  STagIdx *pTagIdx1 = (STagIdx *)p1;
+  STagIdx *pTagIdx2 = (STagIdx *)p2;
+  if (pTagIdx1->cid < pTagIdx1->cid) {
+    return -1;
+  } else if (pTagIdx1->cid > pTagIdx1->cid) {
+    return 1;
+  }
+  return 0;
+}
+int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag) {
+  STagVal *pTagVal;
+  uint8_t *p;
+  int32_t  n;
+  uint16_t tsize = sizeof(STag) + sizeof(STagIdx) * nTag;
+
+  for (int16_t iTag = 0; iTag < nTag; iTag++) {
+    pTagVal = &pTagVals[iTag];
+
+    if (IS_VAR_DATA_TYPE(pTagVal->type)) {
+      tsize += tPutBinary(NULL, pTagVal->pData, pTagVal->nData);
+    } else {
+      ASSERT(pTagVal->nData == TYPE_BYTES[pTagVal->type]);
+      tsize += pTagVal->nData;
+    }
+  }
+
+  (*ppTag) = (STag *)taosMemoryMalloc(tsize);
+  if (*ppTag == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  p = (uint8_t *)&((*ppTag)->idx[nTag]);
+  n = 0;
+
+  (*ppTag)->len = tsize;
+  (*ppTag)->nTag = nTag;
+  for (int16_t iTag = 0; iTag < nTag; iTag++) {
+    pTagVal = &pTagVals[iTag];
+
+    (*ppTag)->idx[iTag].cid = pTagVal->cid;
+    (*ppTag)->idx[iTag].offset = n;
+
+    if (IS_VAR_DATA_TYPE(pTagVal->type)) {
+      n += tPutBinary(p + n, pTagVal->pData, pTagVal->nData);
+    } else {
+      memcpy(p + n, pTagVal->pData, pTagVal->nData);
+      n += pTagVal->nData;
+    }
+  }
+
+  qsort((*ppTag)->idx, (*ppTag)->nTag, sizeof(STagIdx), tTagIdxCmprFn);
+  return 0;
+}
+
+void tTagFree(STag *pTag) {
+  if (pTag) taosMemoryFree(pTag);
+}
+
+void tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData) {
+  STagIdx *pTagIdx = bsearch(&((STagIdx){.cid = cid}), pTag->idx, pTag->nTag, sizeof(STagIdx), tTagIdxCmprFn);
+  if (pTagIdx == NULL) {
+    *ppData = NULL;
+    *nData = 0;
+  } else {
+    uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag] + pTagIdx->offset;
+    if (IS_VAR_DATA_TYPE(type)) {
+      tGetBinary(p, ppData, nData);
+    } else {
+      *ppData = p;
+      *nData = TYPE_BYTES[type];
+    }
+  }
+}
+
+int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag) {
+  // return tEncodeBinary(pEncoder, (uint8_t *)pTag, pTag->len);
+  ASSERT(0);
+  return 0;
+}
+
+int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag) {
+  // uint32_t n;
+  // return tDecodeBinary(pDecoder, (const uint8_t **)ppTag, &n);
+  ASSERT(0);
+  return 0;
+}
+
+#if 1  // ===================================================================================================================
 static void dataColSetNEleNull(SDataCol *pCol, int nEle);
 int         tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   int spaceNeeded = pCol->bytes * maxPoints;