From ce9fb842212cd74f1cb54b6c523ed3ffaa197d3b Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 13 Sep 2022 14:21:43 +0800
Subject: [PATCH 01/11] refact

---
 include/common/tdataformat.h           |  43 ++++-
 source/common/src/tdataformat.c        | 175 +++++++++++++++++-
 source/dnode/vnode/src/inc/tsdb.h      |  26 ---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 244 +------------------------
 4 files changed, 210 insertions(+), 278 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index df16f4f0ab..2edecc6199 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -36,8 +36,13 @@ typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
 typedef struct STagVal       STagVal;
 typedef struct STag          STag;
+typedef struct SColData      SColData;
 
-// bitmap
+#define HAS_NONE  ((int8_t)0x1)
+#define HAS_NULL  ((int8_t)0x2)
+#define HAS_VALUE ((int8_t)0x4)
+
+// bitmap ================================
 const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0},
                                        {0b00000000, 0b00000100, 0b00001000, 2},
                                        {0b00000000, 0b00010000, 0b00100000, 4},
@@ -51,21 +56,21 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0},
 #define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)])
 #define GET_BIT2(p, i)    (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3))
 
-// STSchema
+// STSchema ================================
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
 void    tTSchemaDestroy(STSchema *pTSchema);
 
-// SValue
+// SValue ================================
 int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type);
 int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type);
 int     tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type);
 
-// SColVal
+// SColVal ================================
 #define COL_VAL_NONE(CID, TYPE)     ((SColVal){.cid = (CID), .type = (TYPE), .isNone = 1})
 #define COL_VAL_NULL(CID, TYPE)     ((SColVal){.cid = (CID), .type = (TYPE), .isNull = 1})
 #define COL_VAL_VALUE(CID, TYPE, V) ((SColVal){.cid = (CID), .type = (TYPE), .value = (V)})
 
-// STSRow2
+// STSRow2 ================================
 #define TSROW_LEN(PROW, V)  tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL)
 #define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL)
 
@@ -77,7 +82,7 @@ int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray);
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow);
 
-// STSRowBuilder
+// STSRowBuilder ================================
 #define tsRowBuilderInit() ((STSRowBuilder){0})
 #define tsRowBuilderClear(B)     \
   do {                           \
@@ -86,7 +91,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow);
     }                            \
   } while (0)
 
-// STag
+// STag ================================
 int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag);
 void    tTagFree(STag *pTag);
 bool    tTagIsJson(const void *pTag);
@@ -100,7 +105,17 @@ void    tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid);
 void    debugPrintSTag(STag *pTag, const char *tag, int32_t ln);  // TODO: remove
 int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf);
 
-// STRUCT =================
+// SColData ================================
+void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
+void    tColDataReset(SColData *pColData);
+void    tColDataClear(void *ph);
+int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
+int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
+int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
+int32_t tPutColData(uint8_t *p, SColData *pColData);
+int32_t tGetColData(uint8_t *p, SColData *pColData);
+
+// STRUCT ================================
 struct STColumn {
   col_id_t colId;
   int8_t   type;
@@ -166,6 +181,18 @@ struct SColVal {
   SValue  value;
 };
 
+struct SColData {
+  int16_t  cid;
+  int8_t   type;
+  int8_t   smaOn;
+  int32_t  nVal;
+  uint8_t  flag;
+  uint8_t *pBitMap;
+  int32_t *aOffset;
+  int32_t  nData;
+  uint8_t *pData;
+};
+
 #pragma pack(push, 1)
 struct STagVal {
   //  char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index b40f449a05..48fb0bb5f4 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "tdataformat.h"
+#include "tRealloc.h"
 #include "tcoding.h"
 #include "tdatablock.h"
 #include "tlog.h"
@@ -1172,4 +1173,176 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
   return pSchema;
 }
 
-#endif
\ No newline at end of file
+#endif
+
+// SColData ========================================
+void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
+  pColData->cid = cid;
+  pColData->type = type;
+  pColData->smaOn = smaOn;
+  tColDataReset(pColData);
+}
+
+void tColDataReset(SColData *pColData) {
+  pColData->nVal = 0;
+  pColData->flag = 0;
+  pColData->nData = 0;
+}
+
+void tColDataClear(void *ph) {
+  SColData *pColData = (SColData *)ph;
+
+  tFree(pColData->pBitMap);
+  tFree((uint8_t *)pColData->aOffset);
+  tFree(pColData->pData);
+}
+
+int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
+  int32_t code = 0;
+  int64_t size;
+  SValue  value = {0};
+  SValue *pValue = &value;
+
+  ASSERT(pColVal->cid == pColData->cid);
+  ASSERT(pColVal->type == pColData->type);
+
+  // realloc bitmap
+  size = BIT2_SIZE(pColData->nVal + 1);
+  code = tRealloc(&pColData->pBitMap, size);
+  if (code) goto _exit;
+  if ((pColData->nVal & 3) == 0) {
+    pColData->pBitMap[pColData->nVal >> 2] = 0;
+  }
+
+  // put value
+  if (pColVal->isNone) {
+    pColData->flag |= HAS_NONE;
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
+  } else if (pColVal->isNull) {
+    pColData->flag |= HAS_NULL;
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
+  } else {
+    pColData->flag |= HAS_VALUE;
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
+    pValue = &pColVal->value;
+  }
+
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    // offset
+    code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1));
+    if (code) goto _exit;
+    pColData->aOffset[pColData->nVal] = pColData->nData;
+
+    // value
+    if ((!pColVal->isNone) && (!pColVal->isNull)) {
+      code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
+      if (code) goto _exit;
+      memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
+      pColData->nData += pColVal->value.nData;
+    }
+  } else {
+    code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type));
+    if (code) goto _exit;
+    pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type);
+  }
+
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+
+int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
+  int32_t code = 0;
+  int32_t size;
+
+  ASSERT(pColDataSrc->nVal > 0);
+  ASSERT(pColDataDest->cid = pColDataSrc->cid);
+  ASSERT(pColDataDest->type = pColDataSrc->type);
+
+  pColDataDest->smaOn = pColDataSrc->smaOn;
+  pColDataDest->nVal = pColDataSrc->nVal;
+  pColDataDest->flag = pColDataSrc->flag;
+
+  // bitmap
+  if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
+    size = BIT2_SIZE(pColDataSrc->nVal);
+    code = tRealloc(&pColDataDest->pBitMap, size);
+    if (code) goto _exit;
+    memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
+  }
+
+  // offset
+  if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
+    size = sizeof(int32_t) * pColDataSrc->nVal;
+
+    code = tRealloc((uint8_t **)&pColDataDest->aOffset, size);
+    if (code) goto _exit;
+
+    memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size);
+  }
+
+  // value
+  pColDataDest->nData = pColDataSrc->nData;
+  code = tRealloc(&pColDataDest->pData, pColDataSrc->nData);
+  if (code) goto _exit;
+  memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);
+
+_exit:
+  return code;
+}
+
+int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
+  int32_t code = 0;
+
+  ASSERT(iVal < pColData->nVal);
+  ASSERT(pColData->flag);
+
+  if (pColData->flag == HAS_NONE) {
+    *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+    goto _exit;
+  } else if (pColData->flag == HAS_NULL) {
+    *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+    goto _exit;
+  } else if (pColData->flag != HAS_VALUE) {
+    uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
+    if (v == 0) {
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      goto _exit;
+    } else if (v == 1) {
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      goto _exit;
+    }
+  }
+
+  // get value
+  SValue value;
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    if (iVal + 1 < pColData->nVal) {
+      value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
+    } else {
+      value.nData = pColData->nData - pColData->aOffset[iVal];
+    }
+
+    value.pData = pColData->pData + pColData->aOffset[iVal];
+  } else {
+    tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type);
+  }
+  *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value);
+
+_exit:
+  return code;
+}
+
+static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) {
+  SColData *pColData1 = (SColData *)p1;
+  SColData *pColData2 = (SColData *)p2;
+
+  if (pColData1->cid < pColData2->cid) {
+    return -1;
+  } else if (pColData1->cid > pColData2->cid) {
+    return 1;
+  }
+
+  return 0;
+}
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index a836fa2bc5..063b33f3a9 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -44,7 +44,6 @@ typedef struct SMapData      SMapData;
 typedef struct SBlockIdx     SBlockIdx;
 typedef struct SDataBlk      SDataBlk;
 typedef struct SSttBlk       SSttBlk;
-typedef struct SColData      SColData;
 typedef struct SDiskDataHdr  SDiskDataHdr;
 typedef struct SBlockData    SBlockData;
 typedef struct SDelFile      SDelFile;
@@ -71,10 +70,6 @@ typedef struct SLDataIter    SLDataIter;
 #define TSDB_MAX_SUBBLOCKS 8
 #define TSDB_FHDR_SIZE     512
 
-#define HAS_NONE  ((int8_t)0x1)
-#define HAS_NULL  ((int8_t)0x2)
-#define HAS_VALUE ((int8_t)0x4)
-
 #define VERSION_MIN 0
 #define VERSION_MAX INT64_MAX
 
@@ -148,15 +143,6 @@ int32_t tPutBlockIdx(uint8_t *p, void *ph);
 int32_t tGetBlockIdx(uint8_t *p, void *ph);
 int32_t tCmprBlockIdx(void const *lhs, void const *rhs);
 int32_t tCmprBlockL(void const *lhs, void const *rhs);
-// SColdata
-void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
-void    tColDataReset(SColData *pColData);
-void    tColDataClear(void *ph);
-int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
-int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
-int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
-int32_t tPutColData(uint8_t *p, SColData *pColData);
-int32_t tGetColData(uint8_t *p, SColData *pColData);
 // SBlockData
 #define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0)
 #define tBlockDataLastRow(PBLOCKDATA)  tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
@@ -470,18 +456,6 @@ struct SSttBlk {
   SBlockInfo bInfo;
 };
 
-struct SColData {
-  int16_t  cid;
-  int8_t   type;
-  int8_t   smaOn;
-  int32_t  nVal;
-  uint8_t  flag;
-  uint8_t *pBitMap;
-  int32_t *aOffset;
-  int32_t  nData;
-  uint8_t *pData;
-};
-
 // (SBlockData){.suid = 0, .uid = 0}: block data not initialized
 // (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file
 // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index caeca45e01..1a109b2cf6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -909,248 +909,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
   return code;
 }
 
-// SColData ========================================
-void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
-  pColData->cid = cid;
-  pColData->type = type;
-  pColData->smaOn = smaOn;
-  tColDataReset(pColData);
-}
-
-void tColDataReset(SColData *pColData) {
-  pColData->nVal = 0;
-  pColData->flag = 0;
-  pColData->nData = 0;
-}
-
-void tColDataClear(void *ph) {
-  SColData *pColData = (SColData *)ph;
-
-  tFree(pColData->pBitMap);
-  tFree((uint8_t *)pColData->aOffset);
-  tFree(pColData->pData);
-}
-
-int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
-  int32_t code = 0;
-  int64_t size;
-  SValue  value = {0};
-  SValue *pValue = &value;
-
-  ASSERT(pColVal->cid == pColData->cid);
-  ASSERT(pColVal->type == pColData->type);
-
-  // realloc bitmap
-  size = BIT2_SIZE(pColData->nVal + 1);
-  code = tRealloc(&pColData->pBitMap, size);
-  if (code) goto _exit;
-  if ((pColData->nVal & 3) == 0) {
-    pColData->pBitMap[pColData->nVal >> 2] = 0;
-  }
-
-  // put value
-  if (pColVal->isNone) {
-    pColData->flag |= HAS_NONE;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
-  } else if (pColVal->isNull) {
-    pColData->flag |= HAS_NULL;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
-  } else {
-    pColData->flag |= HAS_VALUE;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
-    pValue = &pColVal->value;
-  }
-
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-    code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1));
-    if (code) goto _exit;
-    pColData->aOffset[pColData->nVal] = pColData->nData;
-
-    // value
-    if ((!pColVal->isNone) && (!pColVal->isNull)) {
-      code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
-      if (code) goto _exit;
-      memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
-      pColData->nData += pColVal->value.nData;
-    }
-  } else {
-    code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type));
-    if (code) goto _exit;
-    pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type);
-  }
-
-  pColData->nVal++;
-
-_exit:
-  return code;
-}
-
-int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
-  int32_t code = 0;
-  int32_t size;
-
-  ASSERT(pColDataSrc->nVal > 0);
-  ASSERT(pColDataDest->cid = pColDataSrc->cid);
-  ASSERT(pColDataDest->type = pColDataSrc->type);
-
-  pColDataDest->smaOn = pColDataSrc->smaOn;
-  pColDataDest->nVal = pColDataSrc->nVal;
-  pColDataDest->flag = pColDataSrc->flag;
-
-  // bitmap
-  if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
-    size = BIT2_SIZE(pColDataSrc->nVal);
-    code = tRealloc(&pColDataDest->pBitMap, size);
-    if (code) goto _exit;
-    memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
-  }
-
-  // offset
-  if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
-    size = sizeof(int32_t) * pColDataSrc->nVal;
-
-    code = tRealloc((uint8_t **)&pColDataDest->aOffset, size);
-    if (code) goto _exit;
-
-    memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size);
-  }
-
-  // value
-  pColDataDest->nData = pColDataSrc->nData;
-  code = tRealloc(&pColDataDest->pData, pColDataSrc->nData);
-  if (code) goto _exit;
-  memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);
-
-_exit:
-  return code;
-}
-
-int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
-  int32_t code = 0;
-
-  ASSERT(iVal < pColData->nVal);
-  ASSERT(pColData->flag);
-
-  if (pColData->flag == HAS_NONE) {
-    *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
-    goto _exit;
-  } else if (pColData->flag == HAS_NULL) {
-    *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
-    goto _exit;
-  } else if (pColData->flag != HAS_VALUE) {
-    uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
-    if (v == 0) {
-      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
-      goto _exit;
-    } else if (v == 1) {
-      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
-      goto _exit;
-    }
-  }
-
-  // get value
-  SValue value;
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    if (iVal + 1 < pColData->nVal) {
-      value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
-    } else {
-      value.nData = pColData->nData - pColData->aOffset[iVal];
-    }
-
-    value.pData = pColData->pData + pColData->aOffset[iVal];
-  } else {
-    tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type);
-  }
-  *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value);
-
-_exit:
-  return code;
-}
-
-int32_t tPutColData(uint8_t *p, SColData *pColData) {
-  int32_t n = 0;
-
-  n += tPutI16v(p ? p + n : p, pColData->cid);
-  n += tPutI8(p ? p + n : p, pColData->type);
-  n += tPutI8(p ? p + n : p, pColData->smaOn);
-  n += tPutI32v(p ? p + n : p, pColData->nVal);
-  n += tPutU8(p ? p + n : p, pColData->flag);
-
-  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
-  if (pColData->flag != HAS_VALUE) {
-    // bitmap
-
-    int32_t size = BIT2_SIZE(pColData->nVal);
-    if (p) {
-      memcpy(p + n, pColData->pBitMap, size);
-    }
-    n += size;
-  }
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-
-    int32_t size = sizeof(int32_t) * pColData->nVal;
-    if (p) {
-      memcpy(p + n, pColData->aOffset, size);
-    }
-    n += size;
-  }
-  n += tPutI32v(p ? p + n : p, pColData->nData);
-  if (p) {
-    memcpy(p + n, pColData->pData, pColData->nData);
-  }
-  n += pColData->nData;
-
-_exit:
-  return n;
-}
-
-int32_t tGetColData(uint8_t *p, SColData *pColData) {
-  int32_t n = 0;
-
-  n += tGetI16v(p + n, &pColData->cid);
-  n += tGetI8(p + n, &pColData->type);
-  n += tGetI8(p + n, &pColData->smaOn);
-  n += tGetI32v(p + n, &pColData->nVal);
-  n += tGetU8(p + n, &pColData->flag);
-
-  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
-  if (pColData->flag != HAS_VALUE) {
-    // bitmap
-
-    int32_t size = BIT2_SIZE(pColData->nVal);
-    pColData->pBitMap = p + n;
-    n += size;
-  }
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-
-    int32_t size = sizeof(int32_t) * pColData->nVal;
-    pColData->aOffset = (int32_t *)(p + n);
-    n += size;
-  }
-  n += tGetI32v(p + n, &pColData->nData);
-  pColData->pData = p + n;
-  n += pColData->nData;
-
-_exit:
-  return n;
-}
-
-static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) {
-  SColData *pColData1 = (SColData *)p1;
-  SColData *pColData2 = (SColData *)p2;
-
-  if (pColData1->cid < pColData2->cid) {
-    return -1;
-  } else if (pColData1->cid > pColData2->cid) {
-    return 1;
-  }
-
-  return 0;
-}
-
 // SBlockData ======================================================
 int32_t tBlockDataCreate(SBlockData *pBlockData) {
   int32_t code = 0;
@@ -1501,7 +1259,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD
   while (lidx <= ridx) {
     int32_t   midx = (lidx + ridx) / 2;
     SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx);
-    int32_t   c = tColDataCmprFn(pColData, &(SColData){.cid = cid});
+    int32_t   c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1);
 
     if (c == 0) {
       *ppColData = pColData;

From 0fc81c7d84f4f7bc96d477bc84b4f1811a2d9b52 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 13 Sep 2022 14:52:31 +0800
Subject: [PATCH 02/11] refact code

---
 include/common/tdataformat.h           |  2 +-
 source/common/src/tdataformat.c        | 22 ++++++++++------------
 source/dnode/vnode/src/tsdb/tsdbUtil.c |  2 +-
 3 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 2edecc6199..4e5ecd90dc 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -108,7 +108,7 @@ int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, voi
 // SColData ================================
 void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
 void    tColDataReset(SColData *pColData);
-void    tColDataClear(void *ph);
+void    tColDataDestroy(void *ph);
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
 int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
 int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 48fb0bb5f4..88f0777f57 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -681,7 +681,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) {
   return n;
 }
 
-// STSchema
+// STSchema ========================================
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) {
   *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols);
   if (*ppTSchema == NULL) {
@@ -721,9 +721,7 @@ void tTSchemaDestroy(STSchema *pTSchema) {
   if (pTSchema) taosMemoryFree(pTSchema);
 }
 
-// STSRowBuilder
-
-// STag
+// STag ========================================
 static int tTagValCmprFn(const void *p1, const void *p2) {
   if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) {
     return -1;
@@ -1176,6 +1174,14 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
 #endif
 
 // SColData ========================================
+void tColDataDestroy(void *ph) {
+  SColData *pColData = (SColData *)ph;
+
+  tFree(pColData->pBitMap);
+  tFree((uint8_t *)pColData->aOffset);
+  tFree(pColData->pData);
+}
+
 void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
   pColData->cid = cid;
   pColData->type = type;
@@ -1189,14 +1195,6 @@ void tColDataReset(SColData *pColData) {
   pColData->nData = 0;
 }
 
-void tColDataClear(void *ph) {
-  SColData *pColData = (SColData *)ph;
-
-  tFree(pColData->pBitMap);
-  tFree((uint8_t *)pColData->aOffset);
-  tFree(pColData->pData);
-}
-
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
   int32_t code = 0;
   int64_t size;
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 1a109b2cf6..0a78085db2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -940,7 +940,7 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) {
   tFree((uint8_t *)pBlockData->aVersion);
   tFree((uint8_t *)pBlockData->aTSKEY);
   taosArrayDestroy(pBlockData->aIdx);
-  taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL);
+  taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL);
   pBlockData->aUid = NULL;
   pBlockData->aVersion = NULL;
   pBlockData->aTSKEY = NULL;

From 9e5e8d9969940c1846975426804845ed5b6f465f Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 13 Sep 2022 15:23:45 +0800
Subject: [PATCH 03/11] more code

---
 source/common/src/tdataformat.c | 120 ++++++++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 88f0777f57..ec20baa024 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1174,6 +1174,126 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
 #endif
 
 // SColData ========================================
+static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) {
+  int32_t code = 0;
+
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    code = tRealloc((uint8_t **)(&pColData->aOffset), sizeof(int32_t) * (pColData->nVal + 1));
+    if (code) goto _exit;
+    pColData->aOffset[pColData->nVal] = pColData->nData;
+
+    // value
+    if ((!pColVal->isNone) && (!pColVal->isNull)) {
+      code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
+      if (code) goto _exit;
+      memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
+      pColData->nData += pColVal->value.nData;
+    }
+  } else {
+    code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes);
+    if (code) goto _exit;
+    pColData->nData += tPutValue(pColData->pData + pColData->nData, &pColVal->value, pColVal->type);
+  }
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) {
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+    pColData->flag = HAS_NONE;
+  } else if (pColVal->isNull) {
+    pColData->flag = HAS_NULL;
+  } else {
+    pColData->flag = HAS_VALUE;
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) {  // HAS_NONE
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+  } else if (pColVal->isNull) {
+  } else {
+  }
+  pColData->nVal++;
+
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) {  // HAS_NULL
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+  } else if (pColVal->isNull) {
+  } else {
+  }
+  pColData->nVal++;
+
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) {  // HAS_NONE | HAS_NULL
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+  } else if (pColVal->isNull) {
+  } else {
+  }
+  pColData->nVal++;
+
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+  } else if (pColVal->isNull) {
+  } else {
+  }
+  pColData->nVal++;
+
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE | HAS_NONE
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+  } else if (pColVal->isNull) {
+  } else {
+  }
+  pColData->nVal++;
+
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE | HAS_NULL
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+  } else if (pColVal->isNull) {
+  } else {
+  }
+  pColData->nVal++;
+
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData,
+                                                 SColVal  *pColVal) {  // HAS_VALUE | HAS_NULL | HAS_NONE
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+  } else if (pColVal->isNull) {
+  } else {
+  }
+  pColData->nVal++;
+
+  return code;
+}
+
 void tColDataDestroy(void *ph) {
   SColData *pColData = (SColData *)ph;
 

From 10749dc607e00e3d1919f5abdc104d32038140b5 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 13 Sep 2022 16:11:24 +0800
Subject: [PATCH 04/11] refact code

---
 source/common/src/tdataformat.c | 125 +++++++++++---------------------
 1 file changed, 44 insertions(+), 81 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index ec20baa024..fb18fb9e0f 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1174,6 +1174,27 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
 #endif
 
 // SColData ========================================
+void tColDataDestroy(void *ph) {
+  SColData *pColData = (SColData *)ph;
+
+  tFree(pColData->pBitMap);
+  tFree((uint8_t *)pColData->aOffset);
+  tFree(pColData->pData);
+}
+
+void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
+  pColData->cid = cid;
+  pColData->type = type;
+  pColData->smaOn = smaOn;
+  tColDataReset(pColData);
+}
+
+void tColDataReset(SColData *pColData) {
+  pColData->nVal = 0;
+  pColData->flag = 0;
+  pColData->nData = 0;
+}
+
 static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) {
   int32_t code = 0;
 
@@ -1198,7 +1219,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa
 _exit:
   return code;
 }
-static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) {
+static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) {  // 0
   int32_t code = 0;
 
   if (pColVal->isNone) {
@@ -1220,7 +1241,11 @@ static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pC
 
   if (pColVal->isNone) {
   } else if (pColVal->isNull) {
+    pColData->flag |= HAS_NULL;
+    // TODO
   } else {
+    pColData->flag |= HAS_VALUE;
+    // TODO
   }
   pColData->nVal++;
 
@@ -1230,8 +1255,12 @@ static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pC
   int32_t code = 0;
 
   if (pColVal->isNone) {
+    pColData->flag |= HAS_NONE;
+    // TODO
   } else if (pColVal->isNull) {
   } else {
+    pColData->flag |= HAS_VALUE;
+    // TODO
   }
   pColData->nVal++;
 
@@ -1293,78 +1322,25 @@ static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData,
 
   return code;
 }
-
-void tColDataDestroy(void *ph) {
-  SColData *pColData = (SColData *)ph;
-
-  tFree(pColData->pBitMap);
-  tFree((uint8_t *)pColData->aOffset);
-  tFree(pColData->pData);
-}
-
-void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
-  pColData->cid = cid;
-  pColData->type = type;
-  pColData->smaOn = smaOn;
-  tColDataReset(pColData);
-}
-
-void tColDataReset(SColData *pColData) {
-  pColData->nVal = 0;
-  pColData->flag = 0;
-  pColData->nData = 0;
-}
+static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = {
+    tColDataAppendValue0,  // 0
+    tColDataAppendValue1,  // HAS_NONE
+    tColDataAppendValue2,  // HAS_NULL
+    tColDataAppendValue3,  // HAS_NULL | HAS_NONE
+    tColDataAppendValue4,  // HAS_VALUE
+    tColDataAppendValue5,  // HAS_VALUE | HAS_NONE
+    tColDataAppendValue6,  // HAS_VALUE | HAS_NULL
+    tColDataAppendValue7   // HAS_VALUE | HAS_NULL | HAS_NONE
+};
 
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
   int32_t code = 0;
-  int64_t size;
-  SValue  value = {0};
-  SValue *pValue = &value;
 
-  ASSERT(pColVal->cid == pColData->cid);
-  ASSERT(pColVal->type == pColData->type);
+  ASSERT(pColData->cid == pColVal->cid);
+  ASSERT(pColData->type == pColVal->type);
 
-  // realloc bitmap
-  size = BIT2_SIZE(pColData->nVal + 1);
-  code = tRealloc(&pColData->pBitMap, size);
+  code = tColDataAppendValueImpl[pColData->flag](pColData, pColVal);
   if (code) goto _exit;
-  if ((pColData->nVal & 3) == 0) {
-    pColData->pBitMap[pColData->nVal >> 2] = 0;
-  }
-
-  // put value
-  if (pColVal->isNone) {
-    pColData->flag |= HAS_NONE;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
-  } else if (pColVal->isNull) {
-    pColData->flag |= HAS_NULL;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
-  } else {
-    pColData->flag |= HAS_VALUE;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
-    pValue = &pColVal->value;
-  }
-
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-    code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1));
-    if (code) goto _exit;
-    pColData->aOffset[pColData->nVal] = pColData->nData;
-
-    // value
-    if ((!pColVal->isNone) && (!pColVal->isNull)) {
-      code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
-      if (code) goto _exit;
-      memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
-      pColData->nData += pColVal->value.nData;
-    }
-  } else {
-    code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type));
-    if (code) goto _exit;
-    pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type);
-  }
-
-  pColData->nVal++;
 
 _exit:
   return code;
@@ -1450,17 +1426,4 @@ int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
 
 _exit:
   return code;
-}
-
-static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) {
-  SColData *pColData1 = (SColData *)p1;
-  SColData *pColData2 = (SColData *)p2;
-
-  if (pColData1->cid < pColData2->cid) {
-    return -1;
-  } else if (pColData1->cid > pColData2->cid) {
-    return 1;
-  }
-
-  return 0;
-}
+}
\ No newline at end of file

From b6daab624b6fc62481b449d95b9797fb1a984e65 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 13 Sep 2022 17:30:20 +0800
Subject: [PATCH 05/11] more code

---
 include/common/tdataformat.h           |   4 +-
 source/common/src/tdataformat.c        | 137 +++++++++++++++++--------
 source/dnode/vnode/src/tsdb/tsdbUtil.c |   2 +-
 3 files changed, 96 insertions(+), 47 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 4e5ecd90dc..726e0eba6b 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -107,10 +107,10 @@ int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, voi
 
 // SColData ================================
 void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
-void    tColDataReset(SColData *pColData);
+void    tColDataClear(SColData *pColData);
 void    tColDataDestroy(void *ph);
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
-int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
+void    tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
 int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
 int32_t tPutColData(uint8_t *p, SColData *pColData);
 int32_t tGetColData(uint8_t *p, SColData *pColData);
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index fb18fb9e0f..ee68e53e18 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1186,10 +1186,10 @@ void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
   pColData->cid = cid;
   pColData->type = type;
   pColData->smaOn = smaOn;
-  tColDataReset(pColData);
+  tColDataClear(pColData);
 }
 
-void tColDataReset(SColData *pColData) {
+void tColDataClear(SColData *pColData) {
   pColData->nVal = 0;
   pColData->flag = 0;
   pColData->nData = 0;
@@ -1346,6 +1346,97 @@ _exit:
   return code;
 }
 
+static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NONE
+  *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+}
+static FORCE_INLINE void tColDataGetValue2(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NULL
+  *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+}
+static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NULL|HAS_NONE
+  switch (GET_BIT1(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      break;
+    case 1:
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_VALUE
+  SValue value;
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    if (iVal + 1 < pColData->nVal) {
+      value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
+    } else {
+      value.nData = pColData->nData - pColData->aOffset[iVal];
+    }
+    value.pData = pColData->pData + pColData->aOffset[iVal];
+  } else {
+    tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type);
+  }
+  *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value);
+}
+static FORCE_INLINE void tColDataGetValue5(SColData *pColData, int32_t iVal,
+                                           SColVal *pColVal) {  // HAS_VALUE|HAS_NONE
+  switch (GET_BIT1(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      break;
+    case 1:
+      tColDataGetValue4(pColData, iVal, pColVal);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static FORCE_INLINE void tColDataGetValue6(SColData *pColData, int32_t iVal,
+                                           SColVal *pColVal) {  // HAS_VALUE|HAS_NULL
+  switch (GET_BIT1(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      break;
+    case 1:
+      tColDataGetValue4(pColData, iVal, pColVal);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static FORCE_INLINE void tColDataGetValue7(SColData *pColData, int32_t iVal,
+                                           SColVal *pColVal) {  // HAS_VALUE|HAS_NULL|HAS_NONE
+  switch (GET_BIT2(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      break;
+    case 1:
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      break;
+    case 2:
+      tColDataGetValue4(pColData, iVal, pColVal);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal *pColVal) = {
+    NULL,               // 0
+    tColDataGetValue1,  // HAS_NONE
+    tColDataGetValue2,  // HAS_NULL
+    tColDataGetValue3,  // HAS_NULL | HAS_NONE
+    tColDataGetValue4,  // HAS_VALUE
+    tColDataGetValue5,  // HAS_VALUE | HAS_NONE
+    tColDataGetValue6,  // HAS_VALUE | HAS_NULL
+    tColDataGetValue7   // HAS_VALUE | HAS_NULL | HAS_NONE
+};
+void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
+  ASSERT(iVal >= 0 && iVal < pColData->nVal);
+  ASSERT(pColData->flag);
+
+  tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal);
+}
+
 int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
   int32_t code = 0;
   int32_t size;
@@ -1382,48 +1473,6 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
   if (code) goto _exit;
   memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);
 
-_exit:
-  return code;
-}
-
-int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
-  int32_t code = 0;
-
-  ASSERT(iVal < pColData->nVal);
-  ASSERT(pColData->flag);
-
-  if (pColData->flag == HAS_NONE) {
-    *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
-    goto _exit;
-  } else if (pColData->flag == HAS_NULL) {
-    *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
-    goto _exit;
-  } else if (pColData->flag != HAS_VALUE) {
-    uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
-    if (v == 0) {
-      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
-      goto _exit;
-    } else if (v == 1) {
-      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
-      goto _exit;
-    }
-  }
-
-  // get value
-  SValue value;
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    if (iVal + 1 < pColData->nVal) {
-      value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
-    } else {
-      value.nData = pColData->nData - pColData->aOffset[iVal];
-    }
-
-    value.pData = pColData->pData + pColData->aOffset[iVal];
-  } else {
-    tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type);
-  }
-  *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value);
-
 _exit:
   return code;
 }
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 0a78085db2..32fc7b82a1 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -1009,7 +1009,7 @@ void tBlockDataClear(SBlockData *pBlockData) {
   pBlockData->nRow = 0;
   for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
     SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
-    tColDataReset(pColData);
+    tColDataClear(pColData);
   }
 }
 

From c497b7b77d67db3a03458f0f34d081d4f5f031c9 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 13 Sep 2022 18:17:24 +0800
Subject: [PATCH 06/11] refact more code

---
 include/common/tdataformat.h    |   6 +-
 source/common/src/tdataformat.c | 190 +++++++++++++++++++++++++++-----
 2 files changed, 163 insertions(+), 33 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 726e0eba6b..c51eff6a48 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -38,9 +38,9 @@ typedef struct STagVal       STagVal;
 typedef struct STag          STag;
 typedef struct SColData      SColData;
 
-#define HAS_NONE  ((int8_t)0x1)
-#define HAS_NULL  ((int8_t)0x2)
-#define HAS_VALUE ((int8_t)0x4)
+#define HAS_NONE  ((uint8_t)0x1)
+#define HAS_NULL  ((uint8_t)0x2)
+#define HAS_VALUE ((uint8_t)0x4)
 
 // bitmap ================================
 const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0},
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index ee68e53e18..6affa2bf50 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1239,87 +1239,218 @@ _exit:
 static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) {  // HAS_NONE
   int32_t code = 0;
 
-  if (pColVal->isNone) {
-  } else if (pColVal->isNull) {
-    pColData->flag |= HAS_NULL;
-    // TODO
-  } else {
-    pColData->flag |= HAS_VALUE;
-    // TODO
+  if (!pColVal->isNone) {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT1(pColData->pBitMap, iVal, 0);
+    }
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+    if (pColVal->isNull) {
+      pColData->flag |= HAS_NULL;
+    } else {
+      pColData->flag |= HAS_VALUE;
+      code = tColDataPutValue(pColData, pColVal);
+      if (code) goto _exit;
+    }
   }
   pColData->nVal++;
 
+_exit:
   return code;
 }
 static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) {  // HAS_NULL
   int32_t code = 0;
 
-  if (pColVal->isNone) {
-    pColData->flag |= HAS_NONE;
-    // TODO
-  } else if (pColVal->isNull) {
-  } else {
-    pColData->flag |= HAS_VALUE;
-    // TODO
+  if (!pColVal->isNull) {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    if (pColVal->isNone) {
+      pColData->flag |= HAS_NONE;
+
+      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+        SET_BIT1(pColData->pBitMap, iVal, 1);
+      }
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+    } else {
+      pColData->flag |= HAS_VALUE;
+
+      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+        SET_BIT1(pColData->pBitMap, iVal, 0);
+      }
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+      code = tColDataPutValue(pColData, pColVal);
+      if (code) goto _exit;
+    }
   }
   pColData->nVal++;
 
+_exit:
   return code;
 }
-static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) {  // HAS_NONE | HAS_NULL
+static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) {  // HAS_NULL|HAS_NONE
   int32_t code = 0;
 
   if (pColVal->isNone) {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
   } else if (pColVal->isNull) {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
   } else {
+    uint8_t *pBitMap = NULL;
+    code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal));
+    }
+    SET_BIT2(pBitMap, pColData->nVal, 2);
+
+    pColData->flag |= HAS_VALUE;
+
+    tFree(pColData->pBitMap);
+    pColData->pBitMap = pBitMap;
+
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
   }
   pColData->nVal++;
 
+_exit:
   return code;
 }
 static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE
   int32_t code = 0;
 
   if (pColVal->isNone) {
+    pColData->flag |= HAS_NONE;
+
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT1(pColData->pBitMap, iVal, 1);
+    }
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
   } else if (pColVal->isNull) {
+    pColData->flag |= HAS_NULL;
+
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT1(pColData->pBitMap, iVal, 1);
+    }
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
   } else {
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
   }
   pColData->nVal++;
 
+_exit:
   return code;
 }
-static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE | HAS_NONE
+static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE|HAS_NONE
+  int32_t code = 0;
+
+  if (pColVal->isNull) {
+    pColData->flag |= HAS_NULL;
+
+    uint8_t *pBitMap = NULL;
+    code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nData; iVal++) {
+      SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0);
+    }
+    SET_BIT2(pBitMap, pColData->nVal, 1);
+
+    tFree(pColData->pBitMap);
+    pColData->pBitMap = pBitMap;
+  } else {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    if (pColVal->isNone) {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+    } else {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+      code = tColDataPutValue(pColData, pColVal);
+      if (code) goto _exit;
+    }
+  }
+
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE|HAS_NULL
   int32_t code = 0;
 
   if (pColVal->isNone) {
-  } else if (pColVal->isNull) {
-  } else {
-  }
-  pColData->nVal++;
-
-  return code;
-}
-static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE | HAS_NULL
-  int32_t code = 0;
-
-  if (pColVal->isNone) {
-  } else if (pColVal->isNull) {
+    pColData->flag |= HAS_NONE;
+
+    uint8_t *pBitMap = NULL;
+    code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nData; iVal++) {
+      SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
+    }
+    SET_BIT2(pBitMap, pColData->nVal, 0);
+
+    tFree(pColData->pBitMap);
+    pColData->pBitMap = pBitMap;
   } else {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    if (pColVal->isNull) {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+    } else {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+      code = tColDataPutValue(pColData, pColVal);
+      if (code) goto _exit;
+    }
   }
   pColData->nVal++;
 
+_exit:
   return code;
 }
 static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData,
-                                                 SColVal  *pColVal) {  // HAS_VALUE | HAS_NULL | HAS_NONE
+                                                 SColVal  *pColVal) {  // HAS_VALUE|HAS_NULL|HAS_NONE
   int32_t code = 0;
 
+  code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
+  if (code) goto _exit;
+
   if (pColVal->isNone) {
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
   } else if (pColVal->isNull) {
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
   } else {
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
+
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
   }
   pColData->nVal++;
 
+_exit:
   return code;
 }
 static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = {
@@ -1332,7 +1463,6 @@ static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal
     tColDataAppendValue6,  // HAS_VALUE | HAS_NULL
     tColDataAppendValue7   // HAS_VALUE | HAS_NULL | HAS_NONE
 };
-
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
   int32_t code = 0;
 

From 0e1a4212075a8f87557acec8d82a0734120e89e0 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 14 Sep 2022 09:35:57 +0800
Subject: [PATCH 07/11] refact

---
 source/common/src/tdataformat.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 6affa2bf50..ed2d68961f 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1457,11 +1457,11 @@ static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal
     tColDataAppendValue0,  // 0
     tColDataAppendValue1,  // HAS_NONE
     tColDataAppendValue2,  // HAS_NULL
-    tColDataAppendValue3,  // HAS_NULL | HAS_NONE
+    tColDataAppendValue3,  // HAS_NULL|HAS_NONE
     tColDataAppendValue4,  // HAS_VALUE
-    tColDataAppendValue5,  // HAS_VALUE | HAS_NONE
-    tColDataAppendValue6,  // HAS_VALUE | HAS_NULL
-    tColDataAppendValue7   // HAS_VALUE | HAS_NULL | HAS_NONE
+    tColDataAppendValue5,  // HAS_VALUE|HAS_NONE
+    tColDataAppendValue6,  // HAS_VALUE|HAS_NULL
+    tColDataAppendValue7   // HAS_VALUE|HAS_NULL|HAS_NONE
 };
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
   int32_t code = 0;

From 45acc3420bfc337d2af03bd5f6bcd6ec72f4a4b6 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 14 Sep 2022 18:05:57 +0800
Subject: [PATCH 08/11] more code

---
 include/common/tdataformat.h    |   4 +-
 source/common/src/tdataformat.c | 104 +++++++++++++++++++++-----------
 2 files changed, 70 insertions(+), 38 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index c51eff6a48..159279329b 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -106,14 +106,12 @@ void    debugPrintSTag(STag *pTag, const char *tag, int32_t ln);  // TODO: remov
 int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf);
 
 // SColData ================================
+void    tColDataDestroy(void *ph);
 void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
 void    tColDataClear(SColData *pColData);
-void    tColDataDestroy(void *ph);
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
 void    tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
 int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
-int32_t tPutColData(uint8_t *p, SColData *pColData);
-int32_t tGetColData(uint8_t *p, SColData *pColData);
 
 // STRUCT ================================
 struct STColumn {
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index ed2d68961f..dd2667b302 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1203,8 +1203,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa
     if (code) goto _exit;
     pColData->aOffset[pColData->nVal] = pColData->nData;
 
-    // value
-    if ((!pColVal->isNone) && (!pColVal->isNull)) {
+    if (pColVal->value.nData) {
       code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
       if (code) goto _exit;
       memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
@@ -1240,18 +1239,33 @@ static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pC
   int32_t code = 0;
 
   if (!pColVal->isNone) {
-    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+
+    code = tRealloc(&pColData->pBitMap, nBit);
     if (code) goto _exit;
 
-    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
-      SET_BIT1(pColData->pBitMap, iVal, 0);
-    }
+    memset(pColData->pBitMap, 0, nBit);
     SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
 
     if (pColVal->isNull) {
       pColData->flag |= HAS_NULL;
     } else {
       pColData->flag |= HAS_VALUE;
+
+      if (pColData->nVal) {
+        if (IS_VAR_DATA_TYPE(pColData->type)) {
+          int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+          code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+          if (code) goto _exit;
+          memset(pColData->aOffset, 0, nOffset);
+        } else {
+          pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+          code = tRealloc(&pColData->pData, pColData->nData);
+          if (code) goto _exit;
+          memset(pColData->pData, 0, pColData->nData);
+        }
+      }
+
       code = tColDataPutValue(pColData, pColVal);
       if (code) goto _exit;
     }
@@ -1265,24 +1279,35 @@ static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pC
   int32_t code = 0;
 
   if (!pColVal->isNull) {
-    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+    code = tRealloc(&pColData->pBitMap, nBit);
     if (code) goto _exit;
 
     if (pColVal->isNone) {
       pColData->flag |= HAS_NONE;
 
-      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
-        SET_BIT1(pColData->pBitMap, iVal, 1);
-      }
+      memset(pColData->pBitMap, 255, nBit);
       SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
     } else {
       pColData->flag |= HAS_VALUE;
 
-      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
-        SET_BIT1(pColData->pBitMap, iVal, 0);
-      }
+      memset(pColData->pBitMap, 0, nBit);
       SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
 
+      if (pColData->nVal) {
+        if (IS_VAR_DATA_TYPE(pColData->type)) {
+          int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+          code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+          if (code) goto _exit;
+          memset(pColData->aOffset, 0, nOffset);
+        } else {
+          pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+          code = tRealloc(&pColData->pData, pColData->nData);
+          if (code) goto _exit;
+          memset(pColData->pData, 0, pColData->nData);
+        }
+      }
+
       code = tColDataPutValue(pColData, pColVal);
       if (code) goto _exit;
     }
@@ -1320,6 +1345,20 @@ static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pC
     tFree(pColData->pBitMap);
     pColData->pBitMap = pBitMap;
 
+    if (pColData->nVal) {
+      if (IS_VAR_DATA_TYPE(pColData->type)) {
+        int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+        code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+        if (code) goto _exit;
+        memset(pColData->aOffset, 0, nOffset);
+      } else {
+        pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+        code = tRealloc(&pColData->pData, pColData->nData);
+        if (code) goto _exit;
+        memset(pColData->pData, 0, pColData->nData);
+      }
+    }
+
     code = tColDataPutValue(pColData, pColVal);
     if (code) goto _exit;
   }
@@ -1341,6 +1380,9 @@ static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pC
       SET_BIT1(pColData->pBitMap, iVal, 1);
     }
     SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+
+    code = tColDataPutValue(pColData, &COL_VAL_VALUE(pColData->cid, pColData->type, (SValue){0}));
+    if (code) goto _exit;
   } else if (pColVal->isNull) {
     pColData->flag |= HAS_NULL;
 
@@ -1351,6 +1393,9 @@ static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pC
       SET_BIT1(pColData->pBitMap, iVal, 1);
     }
     SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+
+    code = tColDataPutValue(pColData, &COL_VAL_VALUE(pColData->cid, pColData->type, (SValue){0}));
+    if (code) goto _exit;
   } else {
     code = tColDataPutValue(pColData, pColVal);
     if (code) goto _exit;
@@ -1385,11 +1430,10 @@ static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pC
       SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
     } else {
       SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
-
-      code = tColDataPutValue(pColData, pColVal);
-      if (code) goto _exit;
     }
   }
+  code = tColDataPutValue(pColData, pColVal);
+  if (code) goto _exit;
 
   pColData->nVal++;
 
@@ -1421,11 +1465,11 @@ static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pC
       SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
     } else {
       SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
-
-      code = tColDataPutValue(pColData, pColVal);
-      if (code) goto _exit;
     }
   }
+  code = tColDataPutValue(pColData, pColVal);
+  if (code) goto _exit;
+
   pColData->nVal++;
 
 _exit:
@@ -1444,10 +1488,10 @@ static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData,
     SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
   } else {
     SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
-
-    code = tColDataPutValue(pColData, pColVal);
-    if (code) goto _exit;
   }
+  code = tColDataPutValue(pColData, pColVal);
+  if (code) goto _exit;
+
   pColData->nVal++;
 
 _exit:
@@ -1464,16 +1508,8 @@ static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal
     tColDataAppendValue7   // HAS_VALUE|HAS_NULL|HAS_NONE
 };
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
-  int32_t code = 0;
-
-  ASSERT(pColData->cid == pColVal->cid);
-  ASSERT(pColData->type == pColVal->type);
-
-  code = tColDataAppendValueImpl[pColData->flag](pColData, pColVal);
-  if (code) goto _exit;
-
-_exit:
-  return code;
+  ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
+  return tColDataAppendValueImpl[pColData->flag](pColData, pColVal);
 }
 
 static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NONE
@@ -1561,9 +1597,7 @@ static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal
     tColDataGetValue7   // HAS_VALUE | HAS_NULL | HAS_NONE
 };
 void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
-  ASSERT(iVal >= 0 && iVal < pColData->nVal);
-  ASSERT(pColData->flag);
-
+  ASSERT(iVal >= 0 && iVal < pColData->nVal && pColData->flag);
   tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal);
 }
 

From dcbdc43ff5a22cecad48cded12b6b0668d60cc82 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 14 Sep 2022 22:38:29 +0800
Subject: [PATCH 09/11] fix some code

---
 source/common/src/tdataformat.c | 39 +++++++++++++--------------------
 1 file changed, 15 insertions(+), 24 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index dd2667b302..fc7d211ab3 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1210,6 +1210,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa
       pColData->nData += pColVal->value.nData;
     }
   } else {
+    ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal);
     code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes);
     if (code) goto _exit;
     pColData->nData += tPutValue(pColData->pData + pColData->nData, &pColVal->value, pColVal->type);
@@ -1331,6 +1332,8 @@ static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pC
 
     SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
   } else {
+    pColData->flag |= HAS_VALUE;
+
     uint8_t *pBitMap = NULL;
     code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
     if (code) goto _exit;
@@ -1340,8 +1343,6 @@ static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pC
     }
     SET_BIT2(pBitMap, pColData->nVal, 2);
 
-    pColData->flag |= HAS_VALUE;
-
     tFree(pColData->pBitMap);
     pColData->pBitMap = pBitMap;
 
@@ -1370,31 +1371,21 @@ _exit:
 static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE
   int32_t code = 0;
 
-  if (pColVal->isNone) {
-    pColData->flag |= HAS_NONE;
+  if (pColVal->isNone || pColVal->isNull) {
+    if (pColVal->isNone) {
+      pColData->flag |= HAS_NONE;
+    } else {
+      pColData->flag |= HAS_NULL;
+    }
 
-    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+    code = tRealloc(&pColData->pBitMap, nBit);
     if (code) goto _exit;
 
-    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
-      SET_BIT1(pColData->pBitMap, iVal, 1);
-    }
+    memset(pColData->pBitMap, 255, nBit);
     SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
 
-    code = tColDataPutValue(pColData, &COL_VAL_VALUE(pColData->cid, pColData->type, (SValue){0}));
-    if (code) goto _exit;
-  } else if (pColVal->isNull) {
-    pColData->flag |= HAS_NULL;
-
-    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
-    if (code) goto _exit;
-
-    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
-      SET_BIT1(pColData->pBitMap, iVal, 1);
-    }
-    SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
-
-    code = tColDataPutValue(pColData, &COL_VAL_VALUE(pColData->cid, pColData->type, (SValue){0}));
+    code = tColDataPutValue(pColData, pColVal);
     if (code) goto _exit;
   } else {
     code = tColDataPutValue(pColData, pColVal);
@@ -1415,7 +1406,7 @@ static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pC
     code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
     if (code) goto _exit;
 
-    for (int32_t iVal = 0; iVal < pColData->nData; iVal++) {
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
       SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0);
     }
     SET_BIT2(pBitMap, pColData->nVal, 1);
@@ -1450,7 +1441,7 @@ static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pC
     code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
     if (code) goto _exit;
 
-    for (int32_t iVal = 0; iVal < pColData->nData; iVal++) {
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
       SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
     }
     SET_BIT2(pBitMap, pColData->nVal, 0);

From 3b947f17adcaab5f2b77b06a12eccb3ce2c1d7b2 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 14 Sep 2022 22:47:44 +0800
Subject: [PATCH 10/11] more code

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 88 ++++----------------------
 1 file changed, 13 insertions(+), 75 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 32fc7b82a1..64c150484b 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -1744,47 +1744,16 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
   int32_t size = 0;
   // bitmap
   if (pColData->flag != HAS_VALUE) {
-    uint8_t *pBitMap = pColData->pBitMap;
-    int32_t  szBitMap = BIT2_SIZE(pColData->nVal);
-
-    // BIT2 to BIT1
-    if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+    int32_t szBitMap;
+    if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+      szBitMap = BIT2_SIZE(pColData->nVal);
+    } else {
       szBitMap = BIT1_SIZE(pColData->nVal);
-      pBitMap = taosMemoryCalloc(1, szBitMap);
-      if (pBitMap == NULL) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _exit;
-      }
-
-      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
-        uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
-        switch (pColData->flag) {
-          case (HAS_NULL | HAS_NONE):
-            SET_BIT1(pBitMap, iVal, v);
-            break;
-          case (HAS_VALUE | HAS_NONE):
-            if (v) {
-              SET_BIT1(pBitMap, iVal, 1);
-            } else {
-              SET_BIT1(pBitMap, iVal, 0);
-            }
-            break;
-          case (HAS_VALUE | HAS_NULL):
-            SET_BIT1(pBitMap, iVal, v - 1);
-            break;
-          default:
-            ASSERT(0);
-        }
-      }
     }
 
-    code = tsdbCmprData(pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, &pBlockCol->szBitmap,
-                        ppBuf);
+    code = tsdbCmprData(pColData->pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size,
+                        &pBlockCol->szBitmap, ppBuf);
     if (code) goto _exit;
-
-    if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
-      taosMemoryFree(pBitMap);
-    }
   }
   size += pBlockCol->szBitmap;
 
@@ -1822,46 +1791,15 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
   uint8_t *p = pIn;
   // bitmap
   if (pBlockCol->szBitmap) {
-    if (pBlockCol->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
-      uint8_t *pBitMap = NULL;
-      code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pBitMap,
-                            BIT1_SIZE(pColData->nVal), ppBuf);
-      if (code) goto _exit;
-
-      code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal));
-      if (code) {
-        tFree(pBitMap);
-        goto _exit;
-      }
-
-      // BIT1 to BIT2
-      for (int32_t iVal = 0; iVal < nVal; iVal++) {
-        uint8_t v = GET_BIT1(pBitMap, iVal);
-        switch (pBlockCol->flag) {
-          case (HAS_NULL | HAS_NONE):
-            SET_BIT2(pColData->pBitMap, iVal, v);
-            break;
-          case (HAS_VALUE | HAS_NONE):
-            if (v) {
-              SET_BIT2(pColData->pBitMap, iVal, 2);
-            } else {
-              SET_BIT2(pColData->pBitMap, iVal, 0);
-            }
-            break;
-          case (HAS_VALUE | HAS_NULL):
-            SET_BIT2(pColData->pBitMap, iVal, v + 1);
-            break;
-          default:
-            ASSERT(0);
-        }
-      }
-
-      tFree(pBitMap);
+    int32_t szBitMap;
+    if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+      szBitMap = BIT2_SIZE(pColData->nVal);
     } else {
-      code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap,
-                            BIT2_SIZE(pColData->nVal), ppBuf);
-      if (code) goto _exit;
+      szBitMap = BIT1_SIZE(pColData->nVal);
     }
+
+    code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, szBitMap, ppBuf);
+    if (code) goto _exit;
   }
   p += pBlockCol->szBitmap;
 

From 26f740f26b544fde7a2f12235c07b5c0c11282a0 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 15 Sep 2022 09:56:56 +0800
Subject: [PATCH 11/11] fix more

---
 include/common/tdataformat.h           |  3 ++-
 source/common/src/tdataformat.c        | 32 ++++++++++++++++++++++++++
 source/dnode/vnode/src/tsdb/tsdbRead.c |  4 ++--
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 159279329b..5b49560175 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -110,7 +110,8 @@ void    tColDataDestroy(void *ph);
 void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
 void    tColDataClear(SColData *pColData);
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
-void    tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
+void    tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
+uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal);
 int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
 
 // STRUCT ================================
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index fc7d211ab3..15a369fe40 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1592,6 +1592,38 @@ void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
   tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal);
 }
 
+uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal) {
+  uint8_t v;
+  switch (pColData->flag) {
+    case HAS_NONE:
+      v = 0;
+      break;
+    case HAS_NULL:
+      v = 1;
+      break;
+    case (HAS_NULL | HAS_NONE):
+      v = GET_BIT1(pColData->pBitMap, iVal);
+      break;
+    case HAS_VALUE:
+      v = 2;
+      break;
+    case (HAS_VALUE | HAS_NONE):
+      v = GET_BIT1(pColData->pBitMap, iVal);
+      if (v) v = 2;
+      break;
+    case (HAS_VALUE | HAS_NULL):
+      v = GET_BIT1(pColData->pBitMap, iVal) + 1;
+      break;
+    case (HAS_VALUE | HAS_NULL | HAS_NONE):
+      v = GET_BIT2(pColData->pBitMap, iVal);
+      break;
+    default:
+      ASSERT(0);
+      break;
+  }
+  return v;
+}
+
 int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
   int32_t code = 0;
   int32_t size;
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 60d967681b..baa985fb24 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -903,7 +903,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
           // null value exists, check one-by-one
           if (pData->flag != HAS_VALUE) {
             for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) {
-              uint8_t v = GET_BIT2(pData->pBitMap, j);
+              uint8_t v = tColDataGetBitValue(pData, j);
               if (v == 0 || v == 1) {
                 colDataSetNull_f(pColData->nullbitmap, rowIndex);
               }
@@ -4078,4 +4078,4 @@ void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap) {
   }
 
   tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode));
-}
\ No newline at end of file
+}